From 00b1ce28b957bf9ca7df1998968d7e2bf3a79e5c Mon Sep 17 00:00:00 2001 From: connorhaugh <49422820+connorhaugh@users.noreply.github.com> Date: Mon, 10 Jul 2023 15:45:18 -0400 Subject: [PATCH] feat: add library copy management command (#32598) This PR introduces the "copy" management command, which copies v1 libraries into v2 libraries. --- .../commands/copy_libraries_from_v1_to_v2.py | 125 +++++++++++++ cms/djangoapps/contentstore/tasks.py | 167 +++++++++++++++++- .../core/djangoapps/content_libraries/api.py | 11 +- setup.cfg | 9 +- 4 files changed, 303 insertions(+), 9 deletions(-) create mode 100644 cms/djangoapps/contentstore/management/commands/copy_libraries_from_v1_to_v2.py diff --git a/cms/djangoapps/contentstore/management/commands/copy_libraries_from_v1_to_v2.py b/cms/djangoapps/contentstore/management/commands/copy_libraries_from_v1_to_v2.py new file mode 100644 index 000000000000..c0a866115127 --- /dev/null +++ b/cms/djangoapps/contentstore/management/commands/copy_libraries_from_v1_to_v2.py @@ -0,0 +1,125 @@ +"""A Command to Copy or uncopy V1 Content Libraries entires to be stored as v2 content libraries.""" + +import logging +from textwrap import dedent + +from django.core.management import BaseCommand, CommandError + +from opaque_keys.edx.keys import CourseKey +from opaque_keys.edx.locator import LibraryLocator + +from xmodule.modulestore.django import modulestore + + +from celery import group + +from cms.djangoapps.contentstore.tasks import create_v2_library_from_v1_library, delete_v2_library_from_v1_library + +from .prompt import query_yes_no + +log = logging.getLogger(__name__) + + +class Command(BaseCommand): + """ + Copy or uncopy V1 Content Libraries (default all) entires to be stored as v2 content libraries. + First Specify the uuid for the collection to store the content libraries in. + Specfiy --all for all libraries, library ids for specific libraries, + and -- file followed by the path for a list of libraries from a file. + + Example usage: + + $ ./manage.py cms copy_libraries_from_v1_to_v2 'collection_uuid' --all + $ ./manage.py cms copy_libraries_from_v1_to_v2 + library-v1:edX+DemoX+Demo_Library' 'library-v1:edX+DemoX+Better_Library' -c 'collection_uuid' + $ ./manage.py cms copy_libraries_from_v1_to_v2 --all --uncopy + $ ./manage.py cms copy_libraries_from_v1_to_v2 'library-v1:edX+DemoX+Better_Library' --uncopy + $ ./manage.py cms copy_libraries_from_v1_to_v2 + '11111111-2111-4111-8111-111111111111' + './list_of--library-locators- --file + + Note: + This Command Also produces an "output file" which contains the mapping of locators and the status of the copy. + """ + + help = dedent(__doc__) + CONFIRMATION_PROMPT = "Reindexing all libraries might be a time consuming operation. Do you want to continue?" + + def add_arguments(self, parser): + """arguements for command""" + + parser.add_argument( + '-collection_uuid', + '-c', + nargs=1, + type=str, + help='the uuid for the collection to create the content library in.' + ) + parser.add_argument( + 'library_ids', + nargs='*', + help='a space-seperated list of v1 library ids to copy' + ) + parser.add_argument( + '--all', + action='store_true', + dest='all', + help='Copy all libraries' + ) + parser.add_argument( + '--uncopy', + action='store_true', + dest='uncopy', + help='Delete libraries specified' + ) + + parser.add_argument( + 'output_csv', + nargs='?', + default=None, + help='a file path to write the tasks output to. Without this the result is simply logged.' + ) + + def _parse_library_key(self, raw_value): + """ Parses library key from string """ + result = CourseKey.from_string(raw_value) + + if not isinstance(result, LibraryLocator): + raise CommandError(f"Argument {raw_value} is not a library key") + return result + + def handle(self, *args, **options): # lint-amnesty, pylint: disable=unused-argument + """Parse args and generate tasks for copying content.""" + print(options) + + if (not options['library_ids'] and not options['all']) or (options['library_ids'] and options['all']): + raise CommandError("copy_libraries_from_v1_to_v2 requires one or more s or the --all flag.") + + if (not options['library_ids'] and not options['all']) or (options['library_ids'] and options['all']): + raise CommandError("copy_libraries_from_v1_to_v2 requires one or more s or the --all flag.") + + if options['all']: + store = modulestore() + if query_yes_no(self.CONFIRMATION_PROMPT, default="no"): + v1_library_keys = [ + library.location.library_key.replace(branch=None) for library in store.get_libraries() + ] + else: + return + else: + v1_library_keys = list(map(self._parse_library_key, options['library_ids'])) + + create_library_task_group = group([ + delete_v2_library_from_v1_library.s(str(v1_library_key), options['collection_uuid'][0]) + if options['uncopy'] + else create_v2_library_from_v1_library.s(str(v1_library_key), options['collection_uuid'][0]) + for v1_library_key in v1_library_keys + ]) + + group_result = create_library_task_group.apply_async().get() + if options['output_csv']: + with open(options['output_csv'][0], 'w', encoding='utf-8', newline='') as output_writer: + output_writer.writerow("v1_library_id", "v2_library_id", "status", "error_msg") + for result in group_result: + output_writer.write(result.keys()) + log.info(group_result) diff --git a/cms/djangoapps/contentstore/tasks.py b/cms/djangoapps/contentstore/tasks.py index a2c536227f8f..d452b2f004c1 100644 --- a/cms/djangoapps/contentstore/tasks.py +++ b/cms/djangoapps/contentstore/tasks.py @@ -19,6 +19,7 @@ from django.contrib.auth import get_user_model from django.core.exceptions import SuspiciousOperation from django.core.files import File +from django.db.transaction import atomic from django.test import RequestFactory from django.utils.text import get_valid_filename from edx_django_utils.monitoring import ( @@ -30,9 +31,10 @@ from olxcleaner.exceptions import ErrorLevel from olxcleaner.reporting import report_error_summary, report_errors from opaque_keys.edx.keys import CourseKey -from opaque_keys.edx.locator import LibraryLocator +from opaque_keys.edx.locator import LibraryLocator, LibraryLocatorV2 from organizations.api import add_organization_course, ensure_organization -from organizations.models import OrganizationCourse +from organizations.exceptions import InvalidOrganizationException +from organizations.models import Organization, OrganizationCourse from path import Path as path from pytz import UTC from user_tasks.models import UserTaskArtifact, UserTaskStatus @@ -47,13 +49,17 @@ from cms.djangoapps.contentstore.storage import course_import_export_storage from cms.djangoapps.contentstore.utils import initialize_permissions, reverse_usage_url, translation_language from cms.djangoapps.models.settings.course_metadata import CourseMetadata + from common.djangoapps.course_action_state.models import CourseRerunState from common.djangoapps.student.auth import has_course_author_access +from common.djangoapps.student.roles import CourseInstructorRole, CourseStaffRole, LibraryUserRole from common.djangoapps.util.monitoring import monitor_import_failure from openedx.core.djangoapps.content.learning_sequences.api import key_supports_outlines +from openedx.core.djangoapps.content_libraries import api as v2contentlib_api from openedx.core.djangoapps.course_apps.toggles import exams_ida_enabled from openedx.core.djangoapps.discussions.tasks import update_unit_discussion_state_from_discussion_blocks from openedx.core.djangoapps.embargo.models import CountryAccessRule, RestrictedCourse +from openedx.core.lib.blockstore_api import get_collection from openedx.core.lib.extract_tar import safetar_extractall from xmodule.contentstore.django import contentstore # lint-amnesty, pylint: disable=wrong-import-order from xmodule.course_block import CourseFields # lint-amnesty, pylint: disable=wrong-import-order @@ -790,7 +796,6 @@ def get_error_by_type(error_type): def handle_course_import_exception(courselike_key, exception, status, known=True): """ Handle course import exception and fail task status. - Arguments: courselike_key: A locator identifies a course resource. exception: Exception object @@ -808,3 +813,159 @@ def handle_course_import_exception(courselike_key, exception, status, known=True if status.state != UserTaskStatus.FAILED: status.fail(task_fail_message) + + +def _parse_organization(org_name): + """Find a matching organization name, if one does not exist, specify that this is the *unspecfied* organization""" + try: + ensure_organization(org_name) + except InvalidOrganizationException: + return 'None' + return Organization.objects.get(short_name=org_name) + + +def copy_v1_user_roles_into_v2_library(v2_library_key, v1_library_key): + """ + write the access and edit permissions of a v1 library into a v2 library. + """ + + def _get_users_by_access_level(v1_library_key): + """ + Get a permissions object for a library which contains a list of user IDs for every V2 permissions level, + based on V1 library roles. + The following mapping exists for a library: + V1 Library Role -> V2 Permission Level + LibraryUserRole -> READ_LEVEL + CourseStaffRole -> AUTHOR_LEVEL + CourseInstructorRole -> ADMIN_LEVEL + """ + permissions = {} + permissions[v2contentlib_api.AccessLevel.READ_LEVEL] = list(LibraryUserRole(v1_library_key).users_with_role()) + permissions[v2contentlib_api.AccessLevel.AUTHOR_LEVEL] = list(CourseStaffRole(v1_library_key).users_with_role()) + permissions[v2contentlib_api.AccessLevel.ADMIN_LEVEL] = list( + CourseInstructorRole(v1_library_key).users_with_role() + ) + return permissions + + permissions = _get_users_by_access_level(v1_library_key) + for access_level in permissions.keys(): # lint-amnesty, pylint: disable=consider-iterating-dictionary + for user in permissions[access_level]: + v2contentlib_api.set_library_user_permissions(v2_library_key, user, access_level) + + +def _create_copy_content_task(v2_library_key, v1_library_key): + """ + spin up a celery task to import the V1 Library's content into the V2 library. + This utalizes the fact that course and v1 library content is stored almost identically. + """ + return v2contentlib_api.import_blocks_create_task(v2_library_key, v1_library_key) + + +def _create_metadata(v1_library_key, collection_uuid): + """instansiate an index for the V2 lib in the collection""" + + store = modulestore() + v1_library = store.get_library(v1_library_key) + collection = get_collection(collection_uuid).uuid + # To make it easy, all converted libs are complex, meaning they can contain problems, videos, and text + library_type = 'complex' + org = _parse_organization(v1_library.location.library_key.org) + slug = v1_library.location.library_key.library + title = v1_library.display_name + # V1 libraries do not have descriptions. + description = '' + # permssions & license are most restrictive. + allow_public_learning = False + allow_public_read = False + library_license = '' # '' = ALL_RIGHTS_RESERVED + with atomic(): + return v2contentlib_api.create_library( + collection, + library_type, + org, + slug, + title, + description, + allow_public_learning, + allow_public_read, + library_license + ) + + +@shared_task(time_limit=30) +@set_code_owner_attribute +def delete_v2_library_from_v1_library(v1_library_key_string, collection_uuid): + """ + For a V1 Library, delete the matching v2 library, where the library is the result of the copy operation + This method relys on _create_metadata failling for LibraryAlreadyExists in order to obtain the v2 slug. + """ + v1_library_key = CourseKey.from_string(v1_library_key_string) + v2_library_key = LibraryLocatorV2.from_string('lib:' + v1_library_key.org + ':' + v1_library_key.course) + + try: + v2contentlib_api.delete_library(v2_library_key) + return { + "v1_library_id": v1_library_key_string, + "v2_library_id": v2_library_key, + "status": "SUCCESS", + "msg": None + } + except Exception as error: # lint-amnesty, pylint: disable=broad-except + return { + "v1_library_id": v1_library_key_string, + "v2_library_id": v2_library_key, + "status": "FAILED", + "msg": f"Exception: {v2_library_key} did not delete: {error}" + } + + +@shared_task(time_limit=30) +@set_code_owner_attribute +def create_v2_library_from_v1_library(v1_library_key_string, collection_uuid): + """ + write the metadata, permissions, and content of a v1 library into a v2 library in the given collection. + """ + + v1_library_key = CourseKey.from_string(v1_library_key_string) + + LOGGER.info(f"Copy Library task created for library: {v1_library_key}") + + try: + v2_library_metadata = _create_metadata(v1_library_key, collection_uuid) + + except v2contentlib_api.LibraryAlreadyExists: + return { + "v1_library_id": v1_library_key_string, + "v2_library_id": None, + "status": "FAILED", + "msg": f"Exception: LibraryAlreadyExists {v1_library_key_string} aleady exists" + } + + try: + _create_copy_content_task(v2_library_metadata.key, v1_library_key) + except Exception as error: # lint-amnesty, pylint: disable=broad-except + return { + "v1_library_id": v1_library_key_string, + "v2_library_id": str(v2_library_metadata.key), + "status": "FAILED", + "msg": + f"Could not import content from {v1_library_key_string} into {str(v2_library_metadata.key)}: {str(error)}" + } + + try: + copy_v1_user_roles_into_v2_library(v2_library_metadata.key, v1_library_key) + except Exception as error: # lint-amnesty, pylint: disable=broad-except + return { + "v1_library_id": v1_library_key_string, + "v2_library_id": str(v2_library_metadata.key), + "status": "FAILED", + "msg": + f"Could not copy permissions from {v1_library_key_string} into {str(v2_library_metadata.key)}: {str(error)}" + } + + return { + "v1_library_id": v1_library_key_string, + "v2_library_id": str(v2_library_metadata.key), + "status": "SUCCESS", + "msg": None + } diff --git a/openedx/core/djangoapps/content_libraries/api.py b/openedx/core/djangoapps/content_libraries/api.py index 5d4a134b20a9..7b44ec7d51a4 100644 --- a/openedx/core/djangoapps/content_libraries/api.py +++ b/openedx/core/djangoapps/content_libraries/api.py @@ -70,7 +70,13 @@ from elasticsearch.exceptions import ConnectionError as ElasticConnectionError from lxml import etree from opaque_keys.edx.keys import LearningContextKey, UsageKey -from opaque_keys.edx.locator import BundleDefinitionLocator, LibraryLocatorV2, LibraryUsageLocatorV2 +from opaque_keys.edx.locator import ( + BundleDefinitionLocator, + LibraryLocatorV2, + LibraryUsageLocatorV2, + LibraryLocator as LibraryLocatorV1 +) + from organizations.models import Organization from xblock.core import XBlock from xblock.exceptions import XBlockNotFoundError @@ -1160,7 +1166,6 @@ def import_block(self, modulestore_key): """ Import a single modulestore block. """ - block_data = self.get_block_data(modulestore_key) # Get or create the block in the library. @@ -1270,6 +1275,8 @@ def get_export_keys(self, course_key): Retrieve the course from modulestore and traverse its content tree. """ course = self.modulestore.get_course(course_key) + if isinstance(course_key, LibraryLocatorV1): + course = self.modulestore.get_library(course_key) export_keys = set() blocks_q = collections.deque(course.get_children()) while blocks_q: diff --git a/setup.cfg b/setup.cfg index d6328b4af8f6..673bfd6f61d8 100644 --- a/setup.cfg +++ b/setup.cfg @@ -115,11 +115,12 @@ ignore_imports = # cms.djangoapps.export_course_metadata.tasks # -> openedx.core.djangoapps.schedules.content_highlights # -> lms.djangoapps.courseware.block_render & lms.djangoapps.courseware.model_data - openedx.core.djangoapps.content_libraries.* -> lms.djangoapps.grades.api - # cms.djangoapps.contentstore.tasks -> openedx.core.djangoapps.content_libraries.models - # -> openedx.core.djangoapps.content_libraries.apps - # -> openedx.core.djangoapps.content_libraries.signal_handlers + openedx.core.djangoapps.content_libraries.* -> lms.djangoapps.*.* + # cms.djangoapps.contentstore.tasks -> openedx.core.djangoapps.content_libraries.[various] # -> lms.djangoapps.grades.api + openedx.core.djangoapps.xblock.*.* -> lms.djangoapps.*.* + # cms.djangoapps.contentstore.tasks -> openedx.core.djangoapps.content_libraries.[various] -> openedx.core.djangoapps.xblock.[various] + # -> lms.djangoapps.courseware & lms.djangoapps.courseware.grades openedx.core.djangoapps.schedules.content_highlights -> lms.djangoapps.courseware.* # cms.djangoapps.contentstore.[various] # -> openedx.core.lib.gating.api