Service usage cache and submissions suspended flag

kobo/apps/openrosa/apps/api/tests/viewsets/test_xform_submission_api.py

@@ -387,9 +387,9 @@ def test_post_submission_json_without_submission_key(self):
         )
 
     def test_submission_blocking_flag(self):
-        # Set 'submissions_suspended' True in the profile metadata to test if
+        # Set 'submissions_suspended' True in the profile to test if
         # submission do fail with the flag set
-        self.xform.user.profile.metadata['submissions_suspended'] = True
+        self.xform.user.profile.submissions_suspended = True
         self.xform.user.profile.save()
 
         # No need auth for this test
@@ -430,7 +430,7 @@ def test_submission_blocking_flag(self):
                 f.seek(0)
 
                 # check that users can submit data again when flag is removed
-                self.xform.user.profile.metadata['submissions_suspended'] = False
+                self.xform.user.profile.submissions_suspended = False
                 self.xform.user.profile.save()
 
                 request = self.factory.post(

kobo/apps/openrosa/apps/logger/management/commands/populate_submission_counters.py

@@ -212,15 +212,15 @@ def suspend_submissions_for_user(self, user: settings.AUTH_USER_MODEL):
             user_profile.metadata = {}
 
         # Set the flag `submissions_suspended` to true if it is not already.
-        if not user_profile.metadata.get('submissions_suspended'):
+        if not user_profile.submissions_suspended:
             # We are using the flag `submissions_suspended` to prevent
             # new submissions from coming in while the
             # counters are being calculated.
-            user_profile.metadata['submissions_suspended'] = True
-            user_profile.save(update_fields=['metadata'])
+            user_profile.submissions_suspended = True
+            user_profile.save(update_fields=['submissions_suspended'])
 
     def release_old_locks(self):
-        updates = {'submissions_suspended': False}
+        updates = {}
 
         if self._force:
             updates['counters_updates_status'] = 'not-complete'
@@ -231,12 +231,12 @@ def release_old_locks(self):
                 'metadata',
                 updates=updates,
             ),
+            submissions_suspended=False
         )
 
     def update_user_profile(self, user: settings.AUTH_USER_MODEL):
         # Update user's profile (and lock the related row)
         updates = {
-            'submissions_suspended': False,
             'counters_updates_status': 'complete',
         }
         UserProfile.objects.filter(
@@ -246,4 +246,5 @@ def update_user_profile(self, user: settings.AUTH_USER_MODEL):
                 'metadata',
                 updates=updates,
             ),
+            submissions_suspended=False,
         )

kobo/apps/openrosa/apps/logger/management/commands/update_attachment_storage_bytes.py

@@ -199,24 +199,19 @@ def _lock_user_profile(self, user: settings.AUTH_USER_MODEL):
             user_profile.metadata = {}
 
         # Set the flag to true if it was never set.
-        if not user_profile.metadata.get('submissions_suspended'):
+        if not user_profile.submissions_suspended:
             # We are using the flag `submissions_suspended` to prevent
             # new submissions from coming in while the
             # `attachment_storage_bytes` is being calculated.
-            user_profile.metadata['submissions_suspended'] = True
-            user_profile.save(update_fields=['metadata'])
+            user_profile.submissions_suspended = True
+            user_profile.save(update_fields=['submissions_suspended'])
 
     def _release_locks(self):
         # Release any locks on the users' profile from getting submissions
         if self._verbosity > 1:
             self.stdout.write('Releasing submission locks…')
 
-        UserProfile.objects.all().update(
-            metadata=ReplaceValues(
-                'metadata',
-                updates={'submissions_suspended': False},
-            ),
-        )
+        UserProfile.objects.all().update(submissions_suspended=False)
 
     def _reset_user_profile_counters(self):
 
@@ -251,7 +246,6 @@ def _update_user_profile(self, user: settings.AUTH_USER_MODEL):
 
         # Update user's profile (and lock the related row)
         updates = {
-            'submissions_suspended': False,
             'attachments_counting_status': 'complete',
         }
 
@@ -271,4 +265,5 @@ def _update_user_profile(self, user: settings.AUTH_USER_MODEL):
                 'metadata',
                 updates=updates,
             ),
+            submissions_suspended=False,
         )

kobo/apps/openrosa/apps/logger/migrations/0029_populate_daily_xform_counters_for_year.py

@@ -32,7 +32,7 @@ class Migration(migrations.Migration):
 
     dependencies = [
         ('logger', '0028_add_user_to_daily_submission_counters'),
-        ('main', '0012_add_validate_password_flag_to_profile'),
+        ('main', '0017_userprofile_submissions_suspended'),
     ]
 
     # We don't do anything when migrating in reverse

kobo/apps/openrosa/apps/logger/models/instance.py

@@ -135,7 +135,7 @@ def check_active(self, force):
         #   wrapped in try/except
         UserProfile = apps.get_model('main', 'UserProfile')  # noqa - Avoid circular imports
         if profile := UserProfile.objects.filter(user=self.xform.user).first():
-            if profile.metadata.get('submissions_suspended', False):
+            if profile.submissions_suspended:
                 raise TemporarilyUnavailableError()
         return
 

kobo/apps/openrosa/apps/main/migrations/0017_userprofile_submissions_suspended.py

@@ -0,0 +1,21 @@
+# Generated by Django 4.2.15 on 2024-09-02 21:41
+
+from django.db import migrations, models
+import kobo.apps.openrosa.apps.main.models.meta_data
+import kpi.deployment_backends.kc_access.storage
+import kpi.fields.file
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('main', '0016_drop_old_restservice_tables'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='userprofile',
+            name='submissions_suspended',
+            field=models.BooleanField(default=False),
+        ),
+    ]

kobo/apps/openrosa/apps/main/models/user_profile.py

@@ -37,6 +37,7 @@ class UserProfile(models.Model):
     metadata = models.JSONField(default=dict, blank=True)
     is_mfa_active = LazyDefaultBooleanField(default=False)
     validated_password = models.BooleanField(default=True)
+    submissions_suspended = models.BooleanField(default=False)
 
     class Meta:
         app_label = 'main'

kobo/apps/organizations/models.py

@@ -27,9 +27,6 @@
 
 class Organization(AbstractOrganization):
     id = KpiUidField(uid_prefix='org', primary_key=True)
-    asr_seconds_limit = models.PositiveIntegerField(blank=True, null=True, default=None)
-    mt_character_limit = models.PositiveIntegerField(blank=True, null=True, default=None)
-    usage_updated = models.DateTimeField(blank=True, null=True, default=None)
 
     @property
     def email(self):
@@ -84,27 +81,34 @@ def canceled_subscription_billing_cycle_anchor(self):
 
         return None
 
-    def update_usage_cache(self, service_usage: dict):
+    def get_cached_usage(self, usage_key: str):
         if not (billing_details := self.active_subscription_billing_details()):
             return None
+
         interval = billing_details['recurring_interval']
-        self.asr_seconds_limit = service_usage['total_nlp_usage'][f'asr_seconds_current_{interval}']
-        self.mt_character_limit = service_usage['total_nlp_usage'][f'mt_characters_current_{interval}']
-        self.usage_updated = timezone.now()
-        return self.save()
+        service_usage = ServiceUsageSerializer(
+            get_database_user(request.user),
+            context=context,
+        ).data
+
+        cached_usage = {
+            'asr_seconds': service_usage['total_nlp_usage'][f'asr_seconds_current_{interval}'],
+            'mt_character': service_usage['total_nlp_usage'][f'mt_characters_current_{interval}'],
+        }
+
+        return cached_usage[usage_key]
 
     @cache_for_request
     def is_organization_over_plan_limit(self, limit_type: UsageType) -> Union[bool, None]:
         """
         Check if an organization is over their plan's limit for a given usage type
         Returns None if Stripe isn't enabled or the limit status couldn't be determined
         """
+
         if not settings.STRIPE_ENABLED:
             return None
-        if timezone.now() - self.usage_updated > ORGANIZATION_USAGE_MAX_CACHE_AGE:
-            # TODO: re-fetch service usage data if stale
-            pass
-        cached_usage = self.serializable_value(f'{USAGE_LIMIT_MAP[limit_type]}_limit')
+
+        cached_usage = self.get_cached_usage(USAGE_LIMIT_MAP[limit_type])
         stripe_key = f'{USAGE_LIMIT_MAP_STRIPE[limit_type]}_limit'
         current_limit = Organization.objects.filter(
             id=self.id,

kobo/apps/organizations/tests/test_organizations_api.py

@@ -1,4 +1,8 @@
+import time
+from datetime import datetime
+
 from django.urls import reverse
+from django.utils.http import parse_http_date
 from model_bakery import baker
 from rest_framework import status
 
@@ -72,3 +76,12 @@ def test_update(self):
         org_user = self.organization.add_user(user=user)
         res = self.client.patch(self.url_detail, data)
         self.assertEqual(res.status_code, 403)
+
+    def test_service_usage_date_header(self):
+        self._insert_data()
+        self.client.get(self.url_detail + 'service_usage/')
+        time.sleep(3)
+        response = self.client.get(self.url_detail + 'service_usage/')
+        last_updated_date = parse_http_date(response.headers["Date"])
+        now = datetime.now().timestamp()
+        assert (now - last_updated_date) > 3

kobo/apps/organizations/views.py

@@ -2,6 +2,7 @@
 from django.contrib.postgres.aggregates import ArrayAgg
 from django.db.models import QuerySet
 from django.utils.decorators import method_decorator
+from django.utils.http import http_date
 from django.views.decorators.cache import cache_page
 from django_dont_vary_on.decorators import only_vary_on
 from kpi import filters
@@ -24,10 +25,6 @@
 from ..stripe.constants import ACTIVE_STRIPE_STATUSES
 
 
-@method_decorator(cache_page(settings.ENDPOINT_CACHE_DURATION), name='service_usage')
-# django uses the Vary header in its caching, and each middleware can potentially add more Vary headers
-# we use this decorator to remove any Vary headers except 'origin' (we don't want to cache between different installs)
-@method_decorator(only_vary_on('Origin'), name='service_usage')
 class OrganizationViewSet(viewsets.ModelViewSet):
     """
     Organizations are groups of users with assigned permissions and configurations
@@ -88,6 +85,7 @@ def service_usage(self, request, pk=None, *args, **kwargs):
         >           "current_month_start": {string (date), ISO format},
         >           "current_year_start": {string (date), ISO format},
         >           "billing_period_end": {string (date), ISO format}|{None},
+        >           "last_updated": {string (date), ISO format},
         >       }
         ### CURRENT ENDPOINT
         """
@@ -101,11 +99,15 @@ def service_usage(self, request, pk=None, *args, **kwargs):
             get_database_user(request.user),
             context=context,
         )
-        response = Response(data=serializer.data)
+        response = Response(
+            data=serializer.data,
+            headers={
+                'Date': http_date(
+                    serializer.calculator.get_last_updated().timestamp()
+                )
+            },
+        )
 
-        # update the cached usage data only if we successfully got results for this organization
-        if (response.status_code == 200) and (organization := context.get('organization')):
-            organization.update_usage_cache(serializer.data)
         return response
 
     @action(detail=True, methods=['get'])

kobo/apps/project_ownership/tests/api/v2/test_api.py

@@ -3,6 +3,7 @@
 from constance.test import override_config
 from django.conf import settings
 from django.contrib.auth import get_user_model
+from django.core.cache import cache
 from django.utils import timezone
 from mock import patch, MagicMock
 from rest_framework import status
@@ -13,7 +14,6 @@
     InviteStatusChoices,
     Transfer,
 )
-from kobo.apps.project_ownership.tests.utils import MockServiceUsageSerializer
 from kobo.apps.trackers.utils import update_nlp_counter
 
 from kpi.constants import PERM_VIEW_ASSET
@@ -432,6 +432,7 @@ def test_account_usage_transferred_to_new_user(self):
         assert response.status_code == status.HTTP_201_CREATED
 
         # someuser should have no usage reported anymore
+        cache.clear()
         response = self.client.get(service_usage_url)
         assert response.data['total_nlp_usage'] == expected_empty_data['total_nlp_usage']
         assert response.data['total_storage_bytes'] == expected_empty_data['total_storage_bytes']