remove unpublished resources with duplicate readable_ids (#2478)

* adding task to remove duplicates

* adding recurring task

* adding test

* adding generate_embeddings task after removal

* fixing test

Author: shanbady

learning_resources/tasks.py

@@ -9,7 +9,7 @@
 import boto3
 import celery
 from django.conf import settings
-from django.db.models import Q
+from django.db.models import Count, Q
 from django.utils import timezone
 
 from learning_resources.content_summarizer import ContentSummarizer
@@ -35,6 +35,7 @@
     load_course_blocklist,
     resource_unpublished_actions,
 )
+from learning_resources_search.constants import COURSE_TYPE
 from learning_resources_search.exceptions import RetryError
 from main.celery import app
 from main.constants import ISOFORMAT
@@ -43,6 +44,34 @@
 log = logging.getLogger(__name__)
 
 
+@app.task(bind=True)
+def remove_duplicate_resources(self):
+    """Remove duplicate unpublished resources"""
+    from vector_search.tasks import generate_embeddings
+
+    duplicates = (
+        LearningResource.objects.values("readable_id")
+        .annotate(count_id=Count("id"))
+        .filter(count_id__gt=1)
+    )
+    embed_tasks = []
+    for duplicate in duplicates:
+        unpublished_resources = LearningResource.objects.filter(
+            readable_id=duplicate["readable_id"],
+            published=False,
+        ).values_list("id", flat=True)
+        published_resources = LearningResource.objects.filter(
+            readable_id=duplicate["readable_id"],
+            published=False,
+        ).values_list("id", flat=True)
+        # keep the most recently created resource, delete the rest
+        LearningResource.objects.filter(id__in=unpublished_resources).delete()
+        embed_tasks.append(
+            generate_embeddings.si(published_resources, COURSE_TYPE, overwrite=True)
+        )
+    self.replace(celery.chain(*embed_tasks))
+
+
 @app.task
 def update_next_start_date_and_prices():
     """Update expired next start dates and prices"""

learning_resources/tasks_test.py

@@ -23,6 +23,7 @@
     get_youtube_data,
     get_youtube_transcripts,
     marketing_page_for_resources,
+    remove_duplicate_resources,
     scrape_marketing_pages,
     sync_canvas_courses,
     update_next_start_date_and_prices,
@@ -676,3 +677,18 @@ def test_sync_canvas_courses(settings, mocker, django_assert_num_queries, canvas
         assert mock_ingest_course.call_count == 1
     else:
         assert mock_ingest_course.call_count == 2
+
+
+def test_remove_duplicate_resources(mocker, mocked_celery):
+    """
+    Test that remove_duplicate_resources removes duplicate unpublished resources
+    while keeping the most recently created resource.
+    """
+    duplicate_id = "duplicate_id"
+
+    LearningResourceFactory.create_batch(3, readable_id=duplicate_id, published=False)
+    LearningResourceFactory.create(readable_id=duplicate_id)
+    assert LearningResource.objects.filter(readable_id=duplicate_id).count() == 4
+    with pytest.raises(mocked_celery.replace_exception_class):
+        remove_duplicate_resources()
+    assert LearningResource.objects.filter(readable_id=duplicate_id).count() == 1

main/settings_celery.py

@@ -161,6 +161,10 @@
             "SCRAPE_MARKETING_PAGES_SCHEDULE_SECONDS", 60 * 60 * 12
         ),  # default is every 12 hours
     },
+    "remove-duplicate-courses-every-6-hours": {
+        "task": "learning_resources.tasks.remove_duplicate_resources",
+        "schedule": crontab(minute=0, hour=9),  # 5:00am EST
+    },
 }