remove celery-heimdall

backend/pycon/settings/base.py

@@ -368,3 +368,5 @@
 }
 
 X_FRAME_OPTIONS = "SAMEORIGIN"
+
+CELERY_TASK_IGNORE_RESULT = True

backend/pycon/settings/test.py

@@ -1,3 +1,4 @@
+from .base import env
 from .base import *  # noqa
 
 SECRET_KEY = "this-key-should-only-be-used-for-tests"
@@ -35,3 +36,8 @@
 
 CELERY_TASK_ALWAYS_EAGER = True
 CELERY_TASK_EAGER_PROPAGATES = True
+
+PASSWORD_HASHERS = ("django.contrib.auth.hashers.MD5PasswordHasher",)
+
+CELERY_BROKER_URL = env("CELERY_BROKER_URL")
+CELERY_RESULT_BACKEND = env("CELERY_RESULT_BACKEND")

backend/pyproject.toml

@@ -83,7 +83,7 @@ dependencies = [
     "psycopg2<3.0.0,>=2.9.5",
     "django-imagekit<5.0.0,>=4.1.0",
     "pillow<11.0.0,>=10.0.1",
-    "redis[hiredis]<5.0.0,>=4.5.5",
+    "redis[hiredis]==5.0.2",
     "google-api-python-client<3.0.0,>=2.94.0",
     "google-auth<3.0.0,>=2.22.0",
     "google-auth-oauthlib<2.0.0,>=1.0.0",
@@ -98,7 +98,6 @@ dependencies = [
     "wagtail-localize==1.5.2",
     "celery>=5.3.6",
     "wagtail-headless-preview>=0.7.0",
-    "celery-heimdall>=1.0.0",
 ]
 name = "backend"
 version = "0.1.0"

backend/schedule/tasks.py

@@ -1,6 +1,5 @@
 from django.db.models import Q
 from googleapiclient.errors import HttpError
-from celery_heimdall import HeimdallTask
 from google_api.sdk import youtube_videos_insert, youtube_videos_set_thumbnail
 from integrations import plain
 from pythonit_toolkit.emails.utils import mark_safe
@@ -269,8 +268,8 @@ def send_schedule_invitation_plain_message(*, schedule_item_id, message):
     schedule_item.save(update_fields=["plain_thread_id"])
 
 
-@app.task
-def upload_schedule_item_video(*, sent_for_video_upload_state_id):
+@app.task()
+def upload_schedule_item_video(*, sent_for_video_upload_state_id: int):
     sent_for_video_upload = ScheduleItemSentForVideoUpload.objects.get(
         id=sent_for_video_upload_state_id
     )
@@ -367,12 +366,7 @@ def upload_schedule_item_video(*, sent_for_video_upload_state_id):
     sent_for_video_upload.save(update_fields=["status"])
 
 
-@app.task(
-    base=HeimdallTask,
-    heimdall={
-        "unique": True,
-    },
-)
+@app.task()
 def process_schedule_items_videos_to_upload():
     statuses = (
         ScheduleItemSentForVideoUpload.objects.filter(
@@ -384,7 +378,8 @@ def process_schedule_items_videos_to_upload():
         .to_upload()
         .order_by("last_attempt_at")
     )
-    for sent_for_video_upload_state in statuses:
+
+    for sent_for_video_upload_state in statuses.iterator():
         try:
             upload_schedule_item_video(
                 sent_for_video_upload_state_id=sent_for_video_upload_state.id

backend/schedule/tests/factories.py

@@ -13,6 +13,7 @@
     ScheduleItem,
     ScheduleItemAdditionalSpeaker,
     ScheduleItemAttendee,
+    ScheduleItemSentForVideoUpload,
     Slot,
 )
 from submissions.tests.factories import SubmissionFactory
@@ -112,3 +113,11 @@ class ScheduleItemAttendeeFactory(DjangoModelFactory):
 
     class Meta:
         model = ScheduleItemAttendee
+
+
+@register
+class ScheduleItemSentForVideoUploadFactory(DjangoModelFactory):
+    schedule_item = factory.SubFactory(ScheduleItemFactory)
+
+    class Meta:
+        model = ScheduleItemSentForVideoUpload

backend/schedule/tests/test_tasks.py

@@ -1,3 +1,4 @@
+from unittest import mock
 from conferences.tests.factories import SpeakerVoucherFactory
 from i18n.strings import LazyI18nString
 from datetime import datetime, timezone
@@ -6,18 +7,22 @@
 
 from schedule.tasks import (
     notify_new_schedule_invitation_answer_slack,
+    process_schedule_items_videos_to_upload,
     send_schedule_invitation_email,
     send_schedule_invitation_plain_message,
     send_speaker_communication_email,
     send_speaker_voucher_email,
     send_submission_time_slot_changed_email,
 )
-from schedule.tests.factories import ScheduleItemFactory
+from schedule.tests.factories import (
+    ScheduleItemFactory,
+    ScheduleItemSentForVideoUploadFactory,
+)
 from submissions.tests.factories import SubmissionFactory
 import time_machine
 from conferences.models.speaker_voucher import SpeakerVoucher
 from users.tests.factories import UserFactory
-from schedule.models import ScheduleItem
+from schedule.models import ScheduleItem, ScheduleItemSentForVideoUpload
 from pythonit_toolkit.emails.templates import EmailTemplate
 
 import pytest
@@ -366,3 +371,44 @@ def test_send_schedule_invitation_plain_message_with_existing_thread(mocker):
 
     schedule_item.refresh_from_db()
     assert schedule_item.plain_thread_id == "thread_id"
+
+
+def test_process_schedule_items_videos_to_upload(mocker):
+    mock_process = mocker.patch("schedule.tasks.upload_schedule_item_video")
+
+    ScheduleItemSentForVideoUploadFactory(
+        last_attempt_at=None,
+        status=ScheduleItemSentForVideoUpload.Status.failed,
+    )
+
+    # Never attempted - should be scheduled
+    sent_for_upload_1 = ScheduleItemSentForVideoUploadFactory(
+        last_attempt_at=None,
+        status=ScheduleItemSentForVideoUpload.Status.pending,
+    )
+
+    # Recently attempted
+    sent_for_upload_2 = ScheduleItemSentForVideoUploadFactory(
+        last_attempt_at=datetime(2020, 1, 1, 10, 0, 0, tzinfo=timezone.utc),
+        status=ScheduleItemSentForVideoUpload.Status.pending,
+    )
+
+    with time_machine.travel("2020-01-01 10:30:00Z", tick=False):
+        process_schedule_items_videos_to_upload()
+
+    mock_process.assert_called_once_with(
+        sent_for_video_upload_state_id=sent_for_upload_1.id
+    )
+    mock_process.reset()
+
+    # it has been an hour since upload 2 attempt, so it should be rescheduled
+    with time_machine.travel("2020-01-01 11:20:00Z", tick=False):
+        process_schedule_items_videos_to_upload()
+
+    mock_process.assert_has_calls(
+        [
+            mock.call(sent_for_video_upload_state_id=sent_for_upload_1.id),
+            mock.call(sent_for_video_upload_state_id=sent_for_upload_2.id),
+        ],
+        any_order=True,
+    )

docker-compose.yml

@@ -24,7 +24,6 @@ x-defaults:
     <<: *enviroment_defaults
     CACHE_URL: redis://redis:6379/0
     DATABASE_URL: psql://pycon:pycon@backend-db/pycon
-    DJANGO_SETTINGS_MODULE: pycon.settings.dev
     ALLOWED_HOSTS: "*"
     PRETIX_API_TOKEN: ${PRETIX_API_TOKEN}
     MAILCHIMP_SECRET_KEY: ${MAILCHIMP_SECRET_KEY}
@@ -53,7 +52,8 @@ services:
       dockerfile: ../Dockerfile.python.local
     networks: [pycon_net]
     entrypoint: ""
-    command: sh -c "pdm install &&
+    command: sh -c "export DJANGO_SETTINGS_MODULE=pycon.settings.dev &&
+      pdm install &&
       pdm run python manage.py migrate &&
       pdm run python manage.py create_admin &&
       touch /.ready &&