Merge pull request #827 from open-craft/alfred/enable-couse-block-str…

…ucture-caching

[FAL-2123] Ocim Add playbook and block structure caching configuration

instance/models/mixins/openedx_config.py

@@ -66,6 +66,29 @@ def _get_configuration_variables(self):
             "COMMON_JWT_SECRET_KEY": "{{ EDXAPP_JWT_SECRET_KEY }}",
             "COMMON_JWT_AUDIENCE": 'lms-key',
 
+            # edxapp Course Block Structures, to enable Course Block Structure caching
+            "EDXAPP_BLOCK_STRUCTURES_SETTINGS": {
+                # Delay, in seconds, after a new edit of a course is published before updating the block structures
+                # cache. This is needed for a better chance at getting the latest changes when there are secondary
+                # reads in sharded MongoDB clusters. See TNL-5041 for more info.
+                "COURSE_PUBLISH_TASK_DELAY": 30,
+                # Delay, in seconds, between retry attempts if a task fails.
+                "TASK_DEFAULT_RETRY_DELAY": 30,
+                # Maximum number of retries per task.
+                "TASK_MAX_RETRIES": 5,
+                # DEPRECATED: this should be a default behavior in Lilac, along with the Waffle Switches mentioned in
+                # SE-4088. Delete previous file versions.
+                "PRUNING_ACTIVE": True,
+                # S3 tiering cache for speeding up course-outline
+                "STORAGE_CLASS": "{{ EDXAPP_DEFAULT_FILE_STORAGE }}",
+                "STORAGE_KWARGS": {
+                    "AWS_ACCESS_KEY_ID": "{{ EDXAPP_AWS_ACCESS_KEY_ID }}",
+                    "AWS_SECRET_ACCESS_KEY": "{{ EDXAPP_AWS_SECRET_ACCESS_KEY }}",
+                    "AWS_STORAGE_BUCKET_NAME": "{{ EDXAPP_AWS_STORAGE_BUCKET_NAME }}",
+                },
+                "DIRECTORY_PREFIX": "courses/"
+            },
+
             # edxapp
             "EDXAPP_PLATFORM_NAME": self.instance.name,
             "EDXAPP_SITE_NAME": self.instance.domain,

instance/models/openedx_appserver.py

@@ -413,6 +413,18 @@ def enable_bulk_emails_playbook(self):
             variables=self.configuration_settings,
         )
 
+    def enable_course_block_structure_caching(self):
+        """
+        Return a Playbook instance for enabling the Course Block Structure caching
+        """
+        return Playbook(
+            version=None,
+            source_repo=os.path.join(settings.SITE_ROOT, 'playbooks/enable_course_block_structure_caching'),
+            requirements_path='requirements.txt',
+            playbook_path='enable_course_block_structure_caching.yml',
+            variables=self.configuration_settings
+        )
+
     def get_playbooks(self):
         """
         Get the ansible playbooks used to provision this AppServer
@@ -422,6 +434,7 @@ def get_playbooks(self):
             playbooks.append(self.lms_user_creation_playbook())
         if not self.instance.successfully_provisioned:
             playbooks.append(self.enable_bulk_emails_playbook())
+        playbooks.append(self.enable_course_block_structure_caching())
         return playbooks + super().get_playbooks()
 
     def merge_site_configuration_settings(self, confvars):

instance/tests/models/test_openedx_appserver.py

@@ -85,17 +85,18 @@ def test_get_playbooks(self, mock_consul):
         # - enable bulk emails playbook,
         # - and the OCIM service ansible playbook.
         playbooks = appserver.get_playbooks()
-        self.assertEqual(len(playbooks), 4)
+        self.assertEqual(len(playbooks), 5)
         self.assertEqual(playbooks[0], appserver.default_playbook())
         self.assertEqual(playbooks[1], appserver.lms_user_creation_playbook())
         self.assertEqual(playbooks[2], appserver.enable_bulk_emails_playbook())
-        self.assertEqual(playbooks[3].source_repo, settings.ANSIBLE_APPSERVER_REPO)
-        self.assertEqual(playbooks[3].playbook_path, settings.ANSIBLE_APPSERVER_PLAYBOOK)
+        self.assertEqual(playbooks[3], appserver.enable_course_block_structure_caching())
+        self.assertEqual(playbooks[4].source_repo, settings.ANSIBLE_APPSERVER_REPO)
+        self.assertEqual(playbooks[4].playbook_path, settings.ANSIBLE_APPSERVER_PLAYBOOK)
         # Once the instance has been successfully provisioned, the "enable bulk emails" playbooks is no longer run.
         instance.successfully_provisioned = True
         instance.save()
         playbooks = appserver.get_playbooks()
-        self.assertEqual(len(playbooks), 3)
+        self.assertEqual(len(playbooks), 4)
         self.assertTrue(appserver.enable_bulk_emails_playbook() not in playbooks)
 
     @patch_services

playbooks/enable_course_block_structure_caching/enable_course_block_structure_caching.yml

@@ -0,0 +1,21 @@
+---
+# This playbook enable course block structure caching (more detail in FAL-2033)
+
+- hosts: all
+  tasks:
+    - name: Create BlockStructureConfiguration
+      shell: |
+       echo 'from openedx.core.djangoapps.content.block_structure.config.models import BlockStructureConfiguration
+       if BlockStructureConfiguration.objects.count() == 0 : BlockStructureConfiguration.objects.create(enabled=True, num_versions_to_keep=1, cache_timeout_in_seconds=None)' |  /edx/bin/edxapp-shell-lms
+
+    - name: Generate cache
+      shell: "sudo -Hsu www-data bash -c '. /edx/app/edxapp/edxapp_env && python /edx/bin/manage.edxapp lms generate_course_blocks --all_courses --with_storage --force_update'"
+
+    - name: Enable waffle switch block_structure.raise_error_when_not_found
+      shell: "sudo -Hsu www-data bash -c '. /edx/app/edxapp/edxapp_env && /edx/app/edxapp/venvs/edxapp/bin/python /edx/bin/manage.edxapp lms waffle_switch --create block_structure.raise_error_when_not_found off'"
+
+    - name: Enable waffle switch block_structure.invalidate_cache_on_publish
+      shell: "sudo -Hsu www-data bash -c '. /edx/app/edxapp/edxapp_env && /edx/app/edxapp/venvs/edxapp/bin/python /edx/bin/manage.edxapp lms waffle_switch --create block_structure.invalidate_cache_on_publish off'"
+
+    - name: Enable waffle switch block_structure.storage_backing_for_cache
+      shell: "sudo -Hsu www-data bash -c '. /edx/app/edxapp/edxapp_env && /edx/app/edxapp/venvs/edxapp/bin/python /edx/bin/manage.edxapp lms waffle_switch --create block_structure.storage_backing_for_cache on'"

playbooks/enable_course_block_structure_caching/requirements.txt

@@ -0,0 +1 @@
+ansible==2.8.17