Feature/sentry fixes

.github/workflows/ci.yaml

@@ -66,7 +66,7 @@ jobs:
         env:
           DOCKER_IMAGE_BACKEND: ${{ steps.prep.outputs.tagged_image }}
         run: |
-          docker-compose -f ./gh-docker-compose.yml run --rm web bash -c 'wait-for-it db:5432 && ./manage.py graphql_schema --out /ci-share/schema-latest.graphql' &&
+          docker compose -f ./gh-docker-compose.yml run --rm web bash -c 'wait-for-it db:5432 && ./manage.py graphql_schema --out /ci-share/schema-latest.graphql' &&
           cmp --silent schema.graphql ./ci-share/schema-latest.graphql || {
             echo 'The schema.graphql is not up to date with the latest changes. Please update and push latest';
             diff schema.graphql ./ci-share/schema-latest.graphql;
@@ -77,15 +77,15 @@ jobs:
         env:
           DOCKER_IMAGE_BACKEND: ${{ steps.prep.outputs.tagged_image }}
         run: |
-          docker-compose -f ./gh-docker-compose.yml run --rm web bash -c 'wait-for-it db:5432 && ./manage.py makemigrations --check --dry-run' || {
+          docker compose -f ./gh-docker-compose.yml run --rm web bash -c 'wait-for-it db:5432 && ./manage.py makemigrations --check --dry-run' || {
             echo 'There are some changes to be reflected in the migration. Make sure to run makemigrations';
             exit 1;
           }
 
       - name: 🤞 Run Test 🧪 & Publish coverage to code climate
         env:
           DOCKER_IMAGE_BACKEND: ${{ steps.prep.outputs.tagged_image }}
-        run: docker-compose -f gh-docker-compose.yml run --rm web /code/scripts/run_tests.sh
+        run: docker compose -f gh-docker-compose.yml run --rm web /code/scripts/run_tests.sh
 
         # Temp fix
         # https://github.com/docker/build-push-action/blob/master/docs/advanced/cache.md#github-cache

Dockerfile

@@ -12,6 +12,8 @@ RUN apt-get update -y \
     && apt-get install -y --no-install-recommends \
         # Build required packages
         gcc libc-dev gdal-bin libproj-dev \
+        # Django translation
+        gettext \
         # Helper packages
         wait-for-it \
     # Upgrade pip and install python packages for code

README.md

@@ -242,3 +242,14 @@ Start celery worker and scheduler on deployment:
 celery multi start w1 -A main -l info
 celery -A main beat --detach -l info
 ```
+
+Translation
+```
+# Static translation
+docker compose exec web ./manage.py makemessages -l es -l ar -l fr --ignore legacy --ignore main
+
+# Dynamic translation
+# -- Run after adding new field to translations
+:wa
+docker compose exec web ./manage.py update_translation_fields
+```

apps/cap_feed/admin.py

@@ -1,5 +1,6 @@
 from admin_auto_filters.filters import AutocompleteFilterFactory
 from django.contrib import admin
+from modeltranslation.admin import TranslationAdmin
 
 from .models import (
     Admin1,
@@ -78,6 +79,7 @@ class AlertInfoInline(admin.StackedInline):
     extra = 0
 
 
+@admin.register(Alert)
 class AlertAdmin(admin.ModelAdmin):
     list_display = ['url', 'country', 'feed', 'sent', 'status', 'msg_type', 'scope']
     list_filter = (
@@ -112,7 +114,14 @@ class AlertAdmin(admin.ModelAdmin):
     inlines = [AlertInfoInline]
 
 
-class CountryAdmin(admin.ModelAdmin):
+@admin.register(Region)
+class RegionAdmin(TranslationAdmin):
+    list_display = ['name']
+    search_fields = ['name']
+
+
+@admin.register(Country)
+class CountryAdmin(TranslationAdmin):
     list_display = ['name', 'iso3', 'region', 'continent']
     list_filter = (
         'region',
@@ -121,6 +130,7 @@ class CountryAdmin(admin.ModelAdmin):
     search_fields = ['name', 'iso3']
 
 
+@admin.register(Admin1)
 class Admin1Admin(admin.ModelAdmin):
     list_display = ['name', 'country']
     list_filter = (AutocompleteFilterFactory('Country', 'country'),)
@@ -137,6 +147,7 @@ def get_formset(self, *args, **kwargs):
         return super().get_formset(validate_min=self.validate_min, *args, **kwargs)
 
 
+@admin.register(Feed)
 class FeedAdmin(admin.ModelAdmin):
     list_display = ['name', 'country', 'url', 'format', 'polling_interval']
     list_filter = (
@@ -159,6 +170,7 @@ def name(self, obj):
         return feed_name
 
 
+@admin.register(FeedLog)
 class FeedLogAdmin(admin.ModelAdmin):
     list_display = ['exception', 'feed', 'description', 'alert_url', 'timestamp']
     list_filter = (
@@ -172,6 +184,7 @@ class FeedLogAdmin(admin.ModelAdmin):
     ]
 
 
+@admin.register(AlertAdmin1)
 class AlertAdmin1Admin(admin.ModelAdmin):
     list_display = ['alert', 'admin1']
     list_filter = (
@@ -181,18 +194,11 @@ class AlertAdmin1Admin(admin.ModelAdmin):
     search_fields = ['alert__url', 'admin1__name']
 
 
+@admin.register(ProcessedAlert)
 class ProcessedAlertAdmin(admin.ModelAdmin):
     list_display = ['url', 'feed']
     list_filter = (AutocompleteFilterFactory('Feed', 'feed'),)
     search_fields = ['url']
 
 
-admin.site.register(ProcessedAlert, ProcessedAlertAdmin)
-admin.site.register(Alert, AlertAdmin)
 admin.site.register(Continent)
-admin.site.register(Region)
-admin.site.register(Country, CountryAdmin)
-admin.site.register(Admin1, Admin1Admin)
-admin.site.register(AlertAdmin1, AlertAdmin1Admin)
-admin.site.register(Feed, FeedAdmin)
-admin.site.register(FeedLog, FeedLogAdmin)

apps/cap_feed/data_injector/geo.py

@@ -2,13 +2,15 @@
 import json
 import os
 import re
+from collections import defaultdict
 
 import requests
 from django.conf import settings
 from django.contrib.gis.gdal import DataSource
 from django.contrib.gis.geos import GEOSGeometry
 from django.core.management.base import BaseCommand
 from django.db import models, transaction
+from modeltranslation.utils import build_localized_fieldname
 
 from apps.cap_feed.models import Admin1, Country, Region
 from main.managers import BulkUpdateManager
@@ -116,6 +118,9 @@ def log_error(self, *messages: str):
 
     @staticmethod
     def clean_name(name: str | None) -> str | None:
+        """
+        Removes excessive spaces and newlines
+        """
         if name is None:
             return
 
@@ -126,8 +131,9 @@ def clean_name(name: str | None) -> str | None:
         ).strip()
 
     def handle_pagination(self, url, **requests_kwargs):
-        _url = f'{self.GO_DOMAIN}{url}?limit=50'
-        self.log_info('Fetching data from IFRC-GO:', _url)
+        _url = f'{self.GO_DOMAIN}{url}'
+        _lang = requests_kwargs.get('headers', {}).get('Accept-Language', 'en')
+        self.log_info(f'Fetching data from IFRC-GO: ({_lang})', _url)
         # TODO: Add some check to avoid infinite run
         while True:
             resp = requests.get(_url, **requests_kwargs).json()
@@ -145,6 +151,19 @@ def inject_continents(self):
     def inject_regions(self):
         go_data = self.handle_pagination('/api/v2/region/')
 
+        lang_map = defaultdict(dict)
+        for lang, _ in settings.LANGUAGES:
+            if lang == 'en':
+                continue
+            lang_go_data = self.handle_pagination(
+                '/api/v2/region/',
+                headers={"Accept-Language": lang},
+            )
+            for region_data in lang_go_data:
+                ifrc_go_id = region_data['id']
+                if translated_name := self.clean_name(region_data['region_name']):
+                    lang_map[ifrc_go_id][lang] = translated_name
+
         for region_data in go_data:
             ifrc_go_id = region_data['id']
             region_name = self.clean_name(region_data['region_name'])
@@ -153,12 +172,19 @@ def inject_regions(self):
                 models.Q(ifrc_go_id=ifrc_go_id) | models.Q(name__iexact=region_name),
             ).get_or_create(
                 defaults={
-                    # 'name': region_name,
+                    'name': region_name,
                 },
             )
 
             region.ifrc_go_id = ifrc_go_id
+
             region.name = region_name
+            for lang, _ in settings.LANGUAGES:
+                translated_name = lang_map[ifrc_go_id].get(lang)
+                if lang == 'en' or translated_name == region.name:
+                    continue
+                setattr(region, build_localized_fieldname('name', lang), translated_name)
+
             region.bbox = GEOSGeometry(str(region_data['bbox']))
             region.save()
             self.region_map[region.ifrc_go_id] = region
@@ -168,14 +194,38 @@ def inject_regions(self):
                 self.log_success(f'Update region: {region}')
 
     def inject_countries(self):
+        fetch_params = {
+            'is_independent': True,
+            'is_deprecated': False,
+            'limit': 300,
+        }
         go_data = self.handle_pagination(
             '/api/v2/country/',
-            params={
-                'is_independent': True,
-                'is_deprecated': False,
-            },
+            params=fetch_params,
+        )
+
+        lang_map = defaultdict(dict)
+        for lang, _ in settings.LANGUAGES:
+            if lang == 'en':
+                continue
+            lang_go_data = self.handle_pagination(
+                '/api/v2/country/',
+                params=fetch_params,
+                headers={"Accept-Language": lang},
+            )
+            for country_data in lang_go_data:
+                ifrc_go_id = country_data['id']
+                if translated_name := self.clean_name(country_data['name']):
+                    lang_map[ifrc_go_id][lang] = translated_name
+
+        mgr = BulkUpdateManager(
+            update_fields=[
+                'ifrc_go_id',
+                'region',
+                'bbox',
+                *[build_localized_fieldname('name', lang) for lang, _ in settings.LANGUAGES],
+            ]
         )
-        mgr = BulkUpdateManager(update_fields=['ifrc_go_id', 'name', 'region', 'bbox'])
 
         CUSTOM_COUNTRY_BBOX = get_custom_country_bbox()
 
@@ -215,7 +265,14 @@ def inject_countries(self):
             )
 
             country.ifrc_go_id = ifrc_go_id
+
             country.name = country_name
+            for lang, _ in settings.LANGUAGES:
+                translated_name = lang_map[ifrc_go_id].get(lang)
+                if lang == 'en' or translated_name == country.name:
+                    continue
+                setattr(country, build_localized_fieldname('name', lang), translated_name)
+
             country.region = region
             if iso3 in CUSTOM_COUNTRY_BBOX:
                 country.bbox = GEOSGeometry(json.dumps(CUSTOM_COUNTRY_BBOX[iso3]))
@@ -294,15 +351,15 @@ def get_geojson_data():
         self.log_success(str(mgr.summary()))
 
     @transaction.atomic
-    def sync(self):
+    def sync(
+        self,
+        # NOTE: Admin1s sync takes a lot of time
+        skip_admin1s_sync: bool = False,
+    ):
         # NOTE: Inject order matters here
         self.inject_continents()
         self.inject_regions()
         self.inject_countries()
-        self.inject_admin1s()
+        if not skip_admin1s_sync:
+            self.inject_admin1s()
         # TODO: Show change summary
-
-
-# inject region and country data if not already present
-def inject_geographical_data():
-    IfrcGoGeoInjector().sync()