From dd8ccf48223bb909f5bf0a0310c788d4b6327a0f Mon Sep 17 00:00:00 2001 From: Alfred Chavez Date: Thu, 22 Jul 2021 18:41:45 -0500 Subject: [PATCH] Add playbook and block structure caching configuration --- instance/models/mixins/openedx_config.py | 23 +++++++++++++++++++ instance/models/openedx_appserver.py | 13 +++++++++++ .../tests/models/test_openedx_appserver.py | 9 ++++---- .../enable_course_block_structure_caching.yml | 21 +++++++++++++++++ .../requirements.txt | 1 + 5 files changed, 63 insertions(+), 4 deletions(-) create mode 100644 playbooks/enable_course_block_structure_caching/enable_course_block_structure_caching.yml create mode 100644 playbooks/enable_course_block_structure_caching/requirements.txt diff --git a/instance/models/mixins/openedx_config.py b/instance/models/mixins/openedx_config.py index 3f34ff93b..bc894dcc7 100644 --- a/instance/models/mixins/openedx_config.py +++ b/instance/models/mixins/openedx_config.py @@ -66,6 +66,29 @@ def _get_configuration_variables(self): "COMMON_JWT_SECRET_KEY": "{{ EDXAPP_JWT_SECRET_KEY }}", "COMMON_JWT_AUDIENCE": 'lms-key', + # edxapp Course Block Structures, to enable Course Block Structure caching + "EDXAPP_BLOCK_STRUCTURES_SETTINGS": { + # Delay, in seconds, after a new edit of a course is published before updating the block structures + # cache. This is needed for a better chance at getting the latest changes when there are secondary + # reads in sharded MongoDB clusters. See TNL-5041 for more info. + "COURSE_PUBLISH_TASK_DELAY": 30, + # Delay, in seconds, between retry attempts if a task fails. + "TASK_DEFAULT_RETRY_DELAY": 30, + # Maximum number of retries per task. + "TASK_MAX_RETRIES": 5, + # DEPRECATED: this should be a default behavior in Lilac, along with the Waffle Switches mentioned in + # SE-4088. Delete previous file versions. + "PRUNING_ACTIVE": True, + # S3 tiering cache for speeding up course-outline + "STORAGE_CLASS": "{{ EDXAPP_DEFAULT_FILE_STORAGE }}", + "STORAGE_KWARGS": { + "AWS_ACCESS_KEY_ID": "{{ EDXAPP_AWS_ACCESS_KEY_ID }}", + "AWS_SECRET_ACCESS_KEY": "{{ EDXAPP_AWS_SECRET_ACCESS_KEY }}", + "AWS_STORAGE_BUCKET_NAME": "{{ EDXAPP_AWS_STORAGE_BUCKET_NAME }}", + }, + "DIRECTORY_PREFIX": "courses/" + }, + # edxapp "EDXAPP_PLATFORM_NAME": self.instance.name, "EDXAPP_SITE_NAME": self.instance.domain, diff --git a/instance/models/openedx_appserver.py b/instance/models/openedx_appserver.py index 07ee7d55b..dda695cd3 100644 --- a/instance/models/openedx_appserver.py +++ b/instance/models/openedx_appserver.py @@ -413,6 +413,18 @@ def enable_bulk_emails_playbook(self): variables=self.configuration_settings, ) + def enable_course_block_structure_caching(self): + """ + Return a Playbook instance for enabling the Course Block Structure caching + """ + return Playbook( + version=None, + source_repo=os.path.join(settings.SITE_ROOT, 'playbooks/enable_course_block_structure_caching'), + requirements_path='requirements.txt', + playbook_path='enable_course_block_structure_caching.yml', + variables=self.configuration_settings + ) + def get_playbooks(self): """ Get the ansible playbooks used to provision this AppServer @@ -422,6 +434,7 @@ def get_playbooks(self): playbooks.append(self.lms_user_creation_playbook()) if not self.instance.successfully_provisioned: playbooks.append(self.enable_bulk_emails_playbook()) + playbooks.append(self.enable_course_block_structure_caching()) return playbooks + super().get_playbooks() def merge_site_configuration_settings(self, confvars): diff --git a/instance/tests/models/test_openedx_appserver.py b/instance/tests/models/test_openedx_appserver.py index 4f8ba442e..4573ff2d4 100644 --- a/instance/tests/models/test_openedx_appserver.py +++ b/instance/tests/models/test_openedx_appserver.py @@ -85,17 +85,18 @@ def test_get_playbooks(self, mock_consul): # - enable bulk emails playbook, # - and the OCIM service ansible playbook. playbooks = appserver.get_playbooks() - self.assertEqual(len(playbooks), 4) + self.assertEqual(len(playbooks), 5) self.assertEqual(playbooks[0], appserver.default_playbook()) self.assertEqual(playbooks[1], appserver.lms_user_creation_playbook()) self.assertEqual(playbooks[2], appserver.enable_bulk_emails_playbook()) - self.assertEqual(playbooks[3].source_repo, settings.ANSIBLE_APPSERVER_REPO) - self.assertEqual(playbooks[3].playbook_path, settings.ANSIBLE_APPSERVER_PLAYBOOK) + self.assertEqual(playbooks[3], appserver.enable_course_block_structure_caching()) + self.assertEqual(playbooks[4].source_repo, settings.ANSIBLE_APPSERVER_REPO) + self.assertEqual(playbooks[4].playbook_path, settings.ANSIBLE_APPSERVER_PLAYBOOK) # Once the instance has been successfully provisioned, the "enable bulk emails" playbooks is no longer run. instance.successfully_provisioned = True instance.save() playbooks = appserver.get_playbooks() - self.assertEqual(len(playbooks), 3) + self.assertEqual(len(playbooks), 4) self.assertTrue(appserver.enable_bulk_emails_playbook() not in playbooks) @patch_services diff --git a/playbooks/enable_course_block_structure_caching/enable_course_block_structure_caching.yml b/playbooks/enable_course_block_structure_caching/enable_course_block_structure_caching.yml new file mode 100644 index 000000000..a5491416a --- /dev/null +++ b/playbooks/enable_course_block_structure_caching/enable_course_block_structure_caching.yml @@ -0,0 +1,21 @@ +--- +# This playbook enable course block structure caching (more detail in FAL-2033) + +- hosts: all + tasks: + - name: Create BlockStructureConfiguration + shell: | + echo 'from openedx.core.djangoapps.content.block_structure.config.models import BlockStructureConfiguration + if BlockStructureConfiguration.objects.count() == 0 : BlockStructureConfiguration.objects.create(enabled=True, num_versions_to_keep=1, cache_timeout_in_seconds=None)' | /edx/bin/edxapp-shell-lms + + - name: Generate cache + shell: "sudo -Hsu www-data bash -c '. /edx/app/edxapp/edxapp_env && python /edx/bin/manage.edxapp lms generate_course_blocks --all_courses --with_storage --force_update'" + + - name: Enable waffle switch block_structure.raise_error_when_not_found + shell: "sudo -Hsu www-data bash -c '. /edx/app/edxapp/edxapp_env && /edx/app/edxapp/venvs/edxapp/bin/python /edx/bin/manage.edxapp lms waffle_switch --create block_structure.raise_error_when_not_found off'" + + - name: Enable waffle switch block_structure.invalidate_cache_on_publish + shell: "sudo -Hsu www-data bash -c '. /edx/app/edxapp/edxapp_env && /edx/app/edxapp/venvs/edxapp/bin/python /edx/bin/manage.edxapp lms waffle_switch --create block_structure.invalidate_cache_on_publish off'" + + - name: Enable waffle switch block_structure.storage_backing_for_cache + shell: "sudo -Hsu www-data bash -c '. /edx/app/edxapp/edxapp_env && /edx/app/edxapp/venvs/edxapp/bin/python /edx/bin/manage.edxapp lms waffle_switch --create block_structure.storage_backing_for_cache on'" diff --git a/playbooks/enable_course_block_structure_caching/requirements.txt b/playbooks/enable_course_block_structure_caching/requirements.txt new file mode 100644 index 000000000..c721abf38 --- /dev/null +++ b/playbooks/enable_course_block_structure_caching/requirements.txt @@ -0,0 +1 @@ +ansible==2.8.17 \ No newline at end of file