Stop processing when the quota is finished

backend/google_api/exceptions.py

@@ -0,0 +1,2 @@
+class NoGoogleCloudQuotaLeftError(Exception):
+    pass

backend/google_api/models.py

@@ -60,7 +60,8 @@ def get_available_credentials_token(
         service: str, min_quota: int
     ) -> Optional["GoogleCloudToken"]:
         credential = (
-            GoogleCloudOAuthCredential.objects.with_quota_left(service)
+            GoogleCloudOAuthCredential.objects.has_token()
+            .with_quota_left(service)
             .annotate(
                 has_token=models.Exists(
                     GoogleCloudToken.objects.filter(

backend/google_api/sdk.py

@@ -1,4 +1,5 @@
 import inspect
+from google_api.exceptions import NoGoogleCloudQuotaLeftError
 from google_api.models import GoogleCloudOAuthCredential, UsedRequestQuota
 from googleapiclient.discovery import build
 from apiclient.http import MediaFileUpload
@@ -12,6 +13,10 @@ def get_available_credentials(service, min_quota):
     token = GoogleCloudOAuthCredential.get_available_credentials_token(
         service=service, min_quota=min_quota
     )
+
+    if not token:
+        raise NoGoogleCloudQuotaLeftError()
+
     return Credentials.from_authorized_user_info(
         {
             "token": token.token,

backend/pycon/celery.py

@@ -27,6 +27,6 @@ def setup_periodic_tasks(sender, **kwargs):
         add = sender.add_periodic_task
 
         add(timedelta(minutes=5), check_association_membership_subscriptions)
-        add(timedelta(minutes=10), process_schedule_items_videos_to_upload)
+        add(timedelta(minutes=30), process_schedule_items_videos_to_upload)
     except Exception:
         logger.exception("setup_periodic_tasks")

backend/schedule/admin.py

@@ -705,6 +705,14 @@ def retry_video_upload(modeladmin, request, queryset):
     )
 
 
+@admin.action(description="Mark as failed")
+def mark_as_failed(modeladmin, request, queryset):
+    queryset.update(status=ScheduleItemSentForVideoUpload.Status.failed)
+    messages.add_message(
+        request, messages.INFO, f"Marked {queryset.count()} videos as failed"
+    )
+
+
 @admin.register(ScheduleItemSentForVideoUpload)
 class ScheduleItemSentForVideoUploadAdmin(admin.ModelAdmin):
     list_display = (
@@ -718,4 +726,4 @@ class ScheduleItemSentForVideoUploadAdmin(admin.ModelAdmin):
     list_filter = ("status", "schedule_item__conference")
     search_fields = ("schedule_item__title",)
     autocomplete_fields = ("schedule_item",)
-    actions = [retry_video_upload]
+    actions = [retry_video_upload, mark_as_failed]

backend/schedule/tasks.py

@@ -1,4 +1,5 @@
 from django.db.models import Q
+from google_api.exceptions import NoGoogleCloudQuotaLeftError
 from googleapiclient.errors import HttpError
 from google_api.sdk import youtube_videos_insert, youtube_videos_set_thumbnail
 from integrations import plain
@@ -384,6 +385,17 @@ def process_schedule_items_videos_to_upload():
             upload_schedule_item_video(
                 sent_for_video_upload_state_id=sent_for_video_upload_state.id
             )
+        except NoGoogleCloudQuotaLeftError:
+            logger.info(
+                "No google cloud quota left to upload the schedule item %s. Moving back to pending and stopping processing.",
+                sent_for_video_upload_state.schedule_item.id,
+            )
+            sent_for_video_upload_state.status = (
+                ScheduleItemSentForVideoUpload.Status.pending
+            )
+            sent_for_video_upload_state.failed_reason = "No Google Cloud Quota Left"
+            sent_for_video_upload_state.save(update_fields=["status", "failed_reason"])
+            break
         except Exception as e:
             logger.exception(
                 "Error processing schedule item %s video upload: %s",

backend/schedule/tests/test_tasks.py

@@ -1,3 +1,4 @@
+from google_api.exceptions import NoGoogleCloudQuotaLeftError
 from googleapiclient.errors import HttpError
 import numpy as np
 from io import BytesIO
@@ -421,6 +422,36 @@ def test_process_schedule_items_videos_to_upload(mocker):
     )
 
 
+def test_process_schedule_items_videos_stops_processing_when_the_quota_is_finished(
+    mocker,
+):
+    mock_process = mocker.patch(
+        "schedule.tasks.upload_schedule_item_video",
+        side_effect=NoGoogleCloudQuotaLeftError(),
+    )
+
+    sent_for_upload_1 = ScheduleItemSentForVideoUploadFactory(
+        last_attempt_at=None,
+        status=ScheduleItemSentForVideoUpload.Status.pending,
+    )
+
+    ScheduleItemSentForVideoUploadFactory(
+        last_attempt_at=None,
+        status=ScheduleItemSentForVideoUpload.Status.pending,
+    )
+
+    with time_machine.travel("2020-01-01 10:30:00Z", tick=False):
+        process_schedule_items_videos_to_upload()
+
+    mock_process.assert_called_once_with(
+        sent_for_video_upload_state_id=sent_for_upload_1.id
+    )
+    mock_process.reset()
+
+    sent_for_upload_1.refresh_from_db()
+    assert sent_for_upload_1.status == ScheduleItemSentForVideoUpload.Status.pending
+
+
 def test_failing_to_process_schedule_items_videos_to_upload_sets_failed_status(mocker):
     mock_process = mocker.patch(
         "schedule.tasks.upload_schedule_item_video",