Merge branch 'main' into add-helper-text

CHANGELOG.md

@@ -10,8 +10,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
 - Answer responses for Forms
 - Language_code field on imports and exports
+- Remove whatsapp body hidden characters
+- Fix for assessment import for multiple languages
+- Add locale filtering to assessments API
+- Consistent labelling on import forms
 ### Removed
-- Locale field on exports 
+- Locale field on exports
 -->
 
 ## v1.4.0 - 2024-12-18

home/api.py

@@ -324,6 +324,7 @@ class AssessmentViewSet(BaseAPIViewSet):
 
     def get_queryset(self):
         qa = self.request.query_params.get("qa")
+        locale_code = self.request.query_params.get("locale")
 
         if qa:
             # return the latest revision for each Assessment
@@ -355,12 +356,16 @@ def get_queryset(self):
                 "last_published_at"
             )
 
+        if locale_code:
+            queryset = queryset.filter(locale__language_code=locale_code)
+
         tag = self.request.query_params.get("tag")
         if tag is not None:
             ids = []
             for t in AssessmentTag.objects.filter(tag__name__iexact=tag):
                 ids.append(t.content_object_id)
             queryset = queryset.filter(id__in=ids)
+
         return queryset
 
 

home/forms.py

@@ -3,7 +3,7 @@
 
 
 class UploadFileForm(forms.Form):
-    FILE_CHOICES = (("CSV", "CSV file"), ("XLSX", "Excel File"))
+    FILE_CHOICES = (("CSV", "CSV File"), ("XLSX", "Excel File"))
     file = forms.FileField()
     file_type = forms.ChoiceField(choices=FILE_CHOICES)
 

home/import_assessments.py

@@ -264,23 +264,23 @@ def add_field_values_to_assessment(self, assessment: Assessment) -> None:
         assessment.high_result_page_id = (
             None
             if self.high_result_page == ""
-            else get_content_page_id_from_slug(self.high_result_page)
+            else get_content_page_id_from_slug(self.high_result_page, self.locale)
         )
         assessment.medium_inflection = self.medium_inflection
         assessment.medium_result_page_id = (
             None
             if self.medium_result_page == ""
-            else get_content_page_id_from_slug(self.medium_result_page)
+            else get_content_page_id_from_slug(self.medium_result_page, self.locale)
         )
         assessment.low_result_page_id = (
             None
             if self.low_result_page == ""
-            else get_content_page_id_from_slug(self.low_result_page)
+            else get_content_page_id_from_slug(self.low_result_page, self.locale)
         )
         assessment.skip_high_result_page_id = (
             None
             if self.skip_high_result_page == ""
-            else get_content_page_id_from_slug(self.skip_high_result_page)
+            else get_content_page_id_from_slug(self.skip_high_result_page, self.locale)
         )
         assessment.skip_threshold = self.skip_threshold
         assessment.generic_error = self.generic_error
@@ -370,6 +370,7 @@ def from_flat(cls, row: dict[str, str], row_num: int) -> "AssessmentRow":
         high_inflection = row.get("high_inflection")
         medium_inflection = row.get("medium_inflection")
         check_punctuation(high_inflection, medium_inflection, row_num)
+        check_score_type(high_inflection, medium_inflection, row_num)
 
         row = {
             key: value for key, value in row.items() if value and key in cls.fields()
@@ -398,13 +399,13 @@ def from_flat(cls, row: dict[str, str], row_num: int) -> "AssessmentRow":
             )
 
 
-def get_content_page_id_from_slug(slug: str) -> int:
+def get_content_page_id_from_slug(slug: str, locale: Locale) -> int:
     try:
-        page = ContentPage.objects.get(slug=slug)
+        page = ContentPage.objects.get(slug=slug, locale=locale)
     except ObjectDoesNotExist:
         raise ImportAssessmentException(
             f"You are trying to add an assessment, where one of the result pages "
-            f"references the content page with slug {slug} which does not exist. "
+            f"references the content page with slug {slug} and locale {locale} which does not exist. "
             "Please create the content page first."
         )
     return page.id
@@ -433,6 +434,29 @@ def check_punctuation(
             )
 
 
+def check_score_type(
+    high_inflection: Any | None, medium_inflection: Any | None, row_num: int
+) -> None:
+    if high_inflection is not None and high_inflection != "":
+        try:
+            float(high_inflection)
+        except ValueError:
+            raise ImportAssessmentException(
+                "Invalid number format for high inflection. "
+                "The score value allows only numbers",
+                row_num,
+            )
+    if medium_inflection is not None and medium_inflection != "":
+        try:
+            float(medium_inflection)
+        except ValueError:
+            raise ImportAssessmentException(
+                "Invalid number format for medium inflection. "
+                "The score value allows only numbers",
+                row_num,
+            )
+
+
 def deserialise_list(value: str) -> list[str]:
     """
     Takes a comma separated value serialised by the CSV library, and returns it as a

home/models.py

@@ -22,7 +22,11 @@
     TitleFieldPanel,
 )
 from wagtail.api import APIField
-from wagtail.blocks import StreamBlockValidationError, StructBlockValidationError
+from wagtail.blocks import (
+    StreamBlockValidationError,
+    StreamValue,
+    StructBlockValidationError,
+)
 from wagtail.contrib.settings.models import BaseSiteSetting, register_setting
 from wagtail.documents.blocks import DocumentChooserBlock
 from wagtail.fields import StreamField
@@ -973,6 +977,19 @@ def clean(self):
         result = super().clean(Page)
         errors = {}
 
+        # Clean the whatsapp body to remove hidden characters
+        if self.whatsapp_body and isinstance(self.whatsapp_body, StreamValue):
+            for block in self.whatsapp_body:
+                if block.block_type == "Whatsapp_Message":
+                    message = block.value["message"]
+                    cleaned_message = "".join(
+                        char
+                        for char in message
+                        if char.isprintable() or char in "\n\r\t"
+                    ).strip()
+
+                    block.value["message"] = cleaned_message
+
         # The WA title is needed for all templates to generate a name for the template
         if self.is_whatsapp_template and not self.whatsapp_title:
             errors.setdefault("whatsapp_title", []).append(

home/tests/import-export-data/bad_form_score.csv

@@ -0,0 +1,3 @@
+title,question_type,tags,slug,version,locale,high_result_page,high_inflection,medium_result_page,medium_inflection,low_result_page,skip_threshold,skip_high_result_page,generic_error,question,explainer,error,min,max,answers,scores,answer_semantic_ids,question_semantic_id,answer_responses
+Titles,age_question,,titles,,en,,Text,,,,0.0,,Generic,What is your age,,,,,,,,test,
+title,age_question,,title,,en,,,,,,0.0,,yes,asg,,,,,,,,id,

home/tests/import-export-data/bad_medium_score.csv

@@ -0,0 +1,3 @@
+title,question_type,tags,slug,version,locale,high_result_page,high_inflection,medium_result_page,medium_inflection,low_result_page,skip_threshold,skip_high_result_page,generic_error,question,explainer,error,min,max,answers,scores,answer_semantic_ids,question_semantic_id,answer_responses
+Titles,age_question,,titles,,en,,,,Text123,,0.0,,Generic,What is your age,,,,,,,,test,
+title,age_question,,title,,en,,,,,,0.0,,yes,asg,,,,,,,,id,

home/tests/import-export-data/content2.csv

@@ -24,12 +24,12 @@ Reply with a number to take part in a *free* self-help program created by WHO.
 
 1. Quit tobacco 🚭
 _Stop smoking with the help of a guided, 42-day program._
-2. Manage your stress 🧘🏽‍♀️
+2. Manage your stress
 _Learn how to cope with stress and improve your wellbeing._",,UTILITY,[],,,,[],,,,,self-help,"*Self-help programs* 🌬️
 
 Reply with a number to take part in a *free* self-help program created by WHO.
 
 1. Quit tobacco 🚭
 _Stop smoking with the help of a guided, 42-day program._
-2. Manage your stress 🧘🏽‍♀️
+2. Manage your stress
 _Learn how to cope with stress and improve your wellbeing._",,,3e5d77f7-4d34-430d-aad7-d9ca01f79732,self_help,,,,[],,,,,,en

home/tests/import-export-data/multiple_language.csv

@@ -0,0 +1,3 @@
+title,question_type,tags,slug,version,locale,high_result_page,high_inflection,medium_result_page,medium_inflection,low_result_page,skip_threshold,skip_high_result_page,generic_error,question,explainer,error,min,max,answers,scores,answer_semantic_ids,question_semantic_id,answer_responses
+Test min max range,integer_question,test-min-max-range,test-min-max-range,v1.0,en,,5.0,,3.0,,0.0,,This is a generic error,Lowest temeprature you're experienced,We need to know some things,This is an error message,0,30,,,,lowest-temperature,
+Weather Trivia,integer_question,weather-trivia,weather-trivia,v1.0,en,high-inflection,5.0,medium-score,1.0,low-score,0.0,,"Sorry, we didn't quite get that.",What's the coldest weather you're experienced?,We need to know some things,Your reply should be between {min} and {max},50,70,,,,coldest-weather,

home/tests/import-export-data/test_special_chars.csv

@@ -0,0 +1,25 @@
+structure,message,page_id,Slug,parent,web_title,web_subtitle,web_body,whatsapp_title,whatsapp_body,whatsapp_template_name,whatsapp_template_category,example_values,variation_title,variation_body,list_title,list_items,sms_title,sms_body,ussd_title,ussd_body,messenger_title,messenger_body,viber_title,viber_body,translation_tag,tags,quick_replies,triggers,locale,next_prompt,buttons,image_link,doc_link,media_link,related_pages,footer,language_code
+Menu 1,0,4,main-menu,,Main Menu,,,,,,,,,,,,,,,,,,,,a0b85075-d01b-46bf-8997-8591e87ba171,,,,English,,,,,,,,
+Sub 1.1,1,5,main-menu-first-time-user,Main Menu,main menu first time user,,,main menu first time user,"🟩🟩🟩🟩🟩🟩🟩🟩
+
+You have completed your registration! 🎉 🌟 ⭐
+
+*Pregnancy info*🤰🏽
+{pregnancy_info_status}
+
+*Basic info* 👤
+{basic_info_status}
+
+*Personal info* 🗝️
+{personal_info_status}
+
+*Daily life*☀️
+{DMA_status}
+
+You can always edit it or add info in `My Profile` on the main menu.

What do you want to do next? 👇🏾",,UTILITY,[],,,,[],,,,,main menu first time user,"Welcome to HealthAlert 🌍
+
+This is a messaging service created by the World Health Organization (WHO) that provides information on COVID-19 as well as emergency resources for disease outbreaks, natural, and man-made disasters.
+
+You can return to this main menu at any time by replying 🏠
+
+Choose what you'd like to know more about by tapping a button below ⬇️",,,5892bccd-8025-419d-9a8e-a6a37b755dbf,menu,"Self-help🌬️, Settings⚙️, Health Info🏥",Main menu🏠,English,,[],,,,,,

home/tests/test_api.py

@@ -2810,3 +2810,54 @@ def test_assessment_detail_endpoint_filter_by_tag(self, uclient):
         response = uclient.get("/api/v2/assessment/?tag=tag3")
         content = json.loads(response.content)
         assert content["count"] == 0
+
+
+@pytest.mark.django_db
+class TestAssessmentLocaleFilterAPI:
+    @pytest.fixture(autouse=True)
+    def create_test_data(self):
+        """
+        Create the content that the tests in this class will use.
+        """
+        self.locale_fr, _ = Locale.objects.get_or_create(language_code="fr")
+        self.locale_en, _ = Locale.objects.get_or_create(language_code="en")
+
+        self.assessment_fr = Assessment.objects.create(
+            title="French Assessment",
+            slug="french-assessment",
+            version="1.0",
+            locale=self.locale_fr,
+            live=True,
+        )
+
+        self.assessment_en = Assessment.objects.create(
+            title="English Assessment",
+            slug="english-assessment",
+            version="1.0",
+            locale=self.locale_en,
+            live=True,
+        )
+
+    def test_assessment_locale_filter_fr(self, admin_client):
+        """
+        Ensure that only French assessment is returned
+        """
+        response_fr = admin_client.get("/api/v2/assessment/", {"locale": "fr"})
+
+        assert response_fr.status_code == 200
+        response_data_fr = response_fr.json()
+
+        assert len(response_data_fr["results"]) == 1
+        assert response_data_fr["results"][0]["locale"] == "fr"
+        assert response_data_fr["results"][0]["title"] == "French Assessment"
+
+    def test_assessment_locale_filter_en(self, admin_client):
+        """
+        Ensure that only English assessment is returned
+        """
+        response_en = admin_client.get("/api/v2/assessment/", {"locale": "en"})
+        assert response_en.status_code == 200
+        response_data_en = response_en.json()
+        assert len(response_data_en["results"]) == 1
+        assert response_data_en["results"][0]["locale"] == "en"
+        assert response_data_en["results"][0]["title"] == "English Assessment"

home/tests/test_assessment_import_export.py

@@ -152,6 +152,24 @@ def normalise_nested_ids(assessment: DbDict) -> DbDict:
 ]
 
 
+def create_content_page(title: str, slug: str, locale_code: str = "en") -> ContentPage:
+    locale = Locale.objects.get(language_code=locale_code)
+    home_page = HomePage.objects.get(slug="home")
+    page = ContentPage(
+        title=title,
+        slug=slug,
+        locale=locale,
+    )
+    home_page.add_child(instance=page)
+    page.save_revision().publish()
+    return page
+
+
+def create_locale_if_not_exists(locale_code: str) -> None:
+    if not Locale.objects.filter(language_code=locale_code).exists():
+        Locale.objects.create(language_code=locale_code)
+
+
 @dataclass
 class ImportExport:
     admin_client: Any
@@ -431,11 +449,11 @@ def test_missing_related_pages(self, csv_impexp: ImportExport) -> None:
         assert (
             e.value.message
             == "You are trying to add an assessment, where one of the result pages "
-            "references the content page with slug fake-page which does not exist. "
+            "references the content page with slug fake-page and locale English which does not exist. "
             "Please create the content page first."
         )
 
-    def test_import_error(elf, csv_impexp: ImportExport) -> None:
+    def test_import_error(self, csv_impexp: ImportExport) -> None:
         """
         Importing an invalid CSV file leaves the db as-is.
 
@@ -573,3 +591,66 @@ def test_mismatched_length_answers(self, csv_impexp: ImportExport) -> None:
             "answer responses (5) do not match."
         )
         assert e.value.row_num == 2
+
+    def test_invalid_high_score(self, csv_impexp: ImportExport) -> None:
+        """
+        Importing a CSV with invalid data in the high inflection value should
+        return an intuitive error message
+        """
+        with pytest.raises(ImportAssessmentException) as e:
+            csv_impexp.import_file("bad_form_score.csv")
+        assert (
+            e.value.message == "Invalid number format for high inflection. "
+            "The score value allows only numbers"
+        )
+        assert e.value.row_num == 2
+
+    def test_invalid_medium_score(self, csv_impexp: ImportExport) -> None:
+        """
+        Importing a CSV with invalid data in the medium inflection value should
+        return an intuitive error message
+        """
+        with pytest.raises(ImportAssessmentException) as e:
+            csv_impexp.import_file("bad_medium_score.csv")
+        assert (
+            e.value.message == "Invalid number format for medium inflection. "
+            "The score value allows only numbers"
+        )
+        assert e.value.row_num == 2
+
+
+@pytest.mark.usefixtures("result_content_pages")
+@pytest.mark.django_db()
+class TestImportMultipleLanguages:
+
+    def test_create_content_page(self, csv_impexp: ImportExport) -> None:
+        """
+        Importing a csv with a results page that has the same slug
+        in more than one locale should pass.
+
+        This test uses high-inflection as slug and high-result page in English and French locale
+        The English high-inflection is already created by the result_content_pages fixture
+        The French high_inflection is created below
+        """
+
+        high_inflection_page_en = ContentPage.objects.get(
+            slug="high-inflection", locale__language_code="en"
+        )
+
+        create_locale_if_not_exists("fr")
+        high_inflection_page_fr = create_content_page(
+            "High Inflection", "high-inflection", locale_code="fr"
+        )
+
+        assert high_inflection_page_en.title == "High Inflection"
+        assert high_inflection_page_en.locale.language_code == "en"
+        assert high_inflection_page_en.live is True
+
+        assert high_inflection_page_fr.title == "High Inflection"
+        assert high_inflection_page_fr.locale.language_code == "fr"
+        assert high_inflection_page_fr.live is True
+
+        csv_bytes = csv_impexp.import_file("multiple_language.csv")
+        content = csv_impexp.export_assessment()
+        src, dst = csv_impexp.csvs2dicts(csv_bytes, content)
+        assert dst == src