feat: add optional inclusion date arg to course reindex command (#35830)

* feat: add optional inclusion date arg to course reindex command

* fix: pylint

cms/djangoapps/contentstore/management/commands/reindex_course.py

@@ -4,9 +4,10 @@
 import logging
 from textwrap import dedent
 from time import time
-from datetime import date
+from datetime import date, datetime
 
 from django.core.management import BaseCommand, CommandError
+from django.conf import settings
 from elasticsearch import exceptions
 from opaque_keys import InvalidKeyError
 from opaque_keys.edx.keys import CourseKey
@@ -42,6 +43,10 @@ def add_arguments(self, parser):
         parser.add_argument('--active',
                             action='store_true',
                             help='Reindex active courses only')
+        parser.add_argument('--from_inclusion_date',
+                            action='store_true',
+                            help='Reindex courses with a start date greater than COURSEWARE_SEARCH_INCLUSION_DATE'
+                            )
         parser.add_argument('--setup',
                             action='store_true',
                             help='Reindex all courses on developers stack setup')
@@ -70,15 +75,17 @@ def handle(self, *args, **options):  # pylint: disable=too-many-statements
         course_ids = options['course_ids']
         all_option = options['all']
         active_option = options['active']
+        inclusion_date_option = options['from_inclusion_date']
         setup_option = options['setup']
         readable_option = options['warning']
         index_all_courses_option = all_option or setup_option
 
-        if ((not course_ids and not (index_all_courses_option or active_option)) or
-                (course_ids and (index_all_courses_option or active_option))):
+        course_option_flag_option = index_all_courses_option or active_option or inclusion_date_option
+
+        if (not course_ids and not course_option_flag_option) or (course_ids and course_option_flag_option):
             raise CommandError((
                 "reindex_course requires one or more <course_id>s"
-                " OR the --all, --active or --setup flags."
+                " OR the --all, --active, --setup, or --from_inclusion_date flags."
             ))
 
         store = modulestore()
@@ -129,6 +136,20 @@ def handle(self, *args, **options):  # pylint: disable=too-many-statements
             course_keys = list(map(lambda course: course.id, active_courses))
 
             logging.warning(f'Selected {len(course_keys)} active courses over a total of {len(all_courses)}.')
+        elif inclusion_date_option:
+            # in case of --from_inclusion_date, we get the list of course keys from all courses
+            # that are stored in modulestore and filter out courses with a start date less than
+            # the settings defined COURSEWARE_SEARCH_INCLUSION_DATE
+            all_courses = modulestore().get_courses()
+
+            inclusion_date = datetime.strptime(
+                settings.FEATURES.get('COURSEWARE_SEARCH_INCLUSION_DATE', '2020-01-01'),
+                '%Y-%m-%d'
+            )
+
+            # We keep the courses that has a start date and the start date is greater than the inclusion date
+            active_courses = filter(lambda course: course.start and (course.start >= inclusion_date), all_courses)
+            course_keys = list(map(lambda course: course.id, active_courses))
 
         else:
             # in case course keys are provided as arguments

cms/djangoapps/contentstore/management/commands/tests/test_reindex_courses.py

@@ -40,6 +40,9 @@ def setUp(self):
         self.third_course = CourseFactory.create(
             org="test", course="course3", display_name="run1", start=None, end=None
         )
+        self.fourth_course = CourseFactory.create(
+            org="test", course="course4", display_name="run1", start=datetime.min.today() - timedelta(weeks=60)
+        )
 
     REINDEX_PATH_LOCATION = (
         'cms.djangoapps.contentstore.management.commands.reindex_course.CoursewareSearchIndexer.do_course_reindex'
@@ -111,7 +114,9 @@ def test_given_all_key_prompts_and_reindexes_all_courses(self):
                 call_command('reindex_course', all=True)
 
                 patched_yes_no.assert_called_once_with(ReindexCommand.CONFIRMATION_PROMPT, default='no')
-                expected_calls = self._build_calls(self.first_course, self.second_course, self.third_course)
+                expected_calls = self._build_calls(
+                    self.first_course, self.second_course, self.third_course, self.fourth_course
+                )
                 self.assertCountEqual(patched_index.mock_calls, expected_calls)
 
     def test_given_all_key_prompts_and_reindexes_all_courses_cancelled(self):
@@ -134,5 +139,21 @@ def test_given_active_key_prompt(self):
                 mock.patch(self.MODULESTORE_PATCH_LOCATION, mock.Mock(return_value=self.store)):
             call_command('reindex_course', active=True)
 
-            expected_calls = self._build_calls(self.first_course)
+            expected_calls = self._build_calls(self.first_course, self.fourth_course)
+            self.assertCountEqual(patched_index.mock_calls, expected_calls)
+
+    @mock.patch.dict(
+        'django.conf.settings.FEATURES',
+        {'COURSEWARE_SEARCH_INCLUSION_DATE': (datetime.min.today() - timedelta(weeks=52)).strftime('%Y-%m-%d')}
+    )
+    def test_given_from_inclusion_date_key_prompt(self):
+        """
+            Test that reindexes all courses that have a start date after a defined inclusion date
+            when --from_inclusion_date key is given.
+        """
+        with mock.patch(self.REINDEX_PATH_LOCATION) as patched_index, \
+                mock.patch(self.MODULESTORE_PATCH_LOCATION, mock.Mock(return_value=self.store)):
+            call_command('reindex_course', from_inclusion_date=True)
+
+            expected_calls = self._build_calls(self.first_course, self.second_course)
             self.assertCountEqual(patched_index.mock_calls, expected_calls)