feat: add library copy management command (#32598)

This PR introduces the "copy" management command, which copies v1 libraries into v2 libraries.

cms/djangoapps/contentstore/management/commands/copy_libraries_from_v1_to_v2.py

@@ -0,0 +1,125 @@
+"""A Command to  Copy or uncopy V1 Content Libraries entires to be stored as v2 content libraries."""
+
+import logging
+from textwrap import dedent
+
+from django.core.management import BaseCommand, CommandError
+
+from opaque_keys.edx.keys import CourseKey
+from opaque_keys.edx.locator import LibraryLocator
+
+from xmodule.modulestore.django import modulestore
+
+
+from celery import group
+
+from cms.djangoapps.contentstore.tasks import create_v2_library_from_v1_library, delete_v2_library_from_v1_library
+
+from .prompt import query_yes_no
+
+log = logging.getLogger(__name__)
+
+
+class Command(BaseCommand):
+    """
+    Copy or uncopy V1 Content Libraries (default all) entires to be stored as v2 content libraries.
+    First Specify the uuid for the collection to store the content libraries in.
+    Specfiy --all for all libraries, library ids for specific libraries,
+    and -- file followed by the path for a list of libraries from a file.
+
+    Example usage:
+
+        $ ./manage.py cms copy_libraries_from_v1_to_v2 'collection_uuid' --all
+        $ ./manage.py cms copy_libraries_from_v1_to_v2
+            library-v1:edX+DemoX+Demo_Library'  'library-v1:edX+DemoX+Better_Library' -c 'collection_uuid'
+        $ ./manage.py cms copy_libraries_from_v1_to_v2 --all --uncopy
+        $ ./manage.py cms copy_libraries_from_v1_to_v2 'library-v1:edX+DemoX+Better_Library' --uncopy
+        $ ./manage.py cms copy_libraries_from_v1_to_v2
+            '11111111-2111-4111-8111-111111111111'
+            './list_of--library-locators- --file
+
+    Note:
+       This Command Also produces an "output file" which contains the mapping of locators and the status of the copy.
+    """
+
+    help = dedent(__doc__)
+    CONFIRMATION_PROMPT = "Reindexing all libraries might be a time consuming operation. Do you want to continue?"
+
+    def add_arguments(self, parser):
+        """arguements for command"""
+
+        parser.add_argument(
+            '-collection_uuid',
+            '-c',
+            nargs=1,
+            type=str,
+            help='the uuid for the collection to create the content library in.'
+        )
+        parser.add_argument(
+            'library_ids',
+            nargs='*',
+            help='a space-seperated list of v1 library ids to copy'
+        )
+        parser.add_argument(
+            '--all',
+            action='store_true',
+            dest='all',
+            help='Copy all libraries'
+        )
+        parser.add_argument(
+            '--uncopy',
+            action='store_true',
+            dest='uncopy',
+            help='Delete libraries specified'
+        )
+
+        parser.add_argument(
+            'output_csv',
+            nargs='?',
+            default=None,
+            help='a file path to write the tasks output to. Without this the result is simply logged.'
+        )
+
+    def _parse_library_key(self, raw_value):
+        """ Parses library key from string """
+        result = CourseKey.from_string(raw_value)
+
+        if not isinstance(result, LibraryLocator):
+            raise CommandError(f"Argument {raw_value} is not a library key")
+        return result
+
+    def handle(self, *args, **options):  # lint-amnesty, pylint: disable=unused-argument
+        """Parse args and generate tasks for copying content."""
+        print(options)
+
+        if (not options['library_ids'] and not options['all']) or (options['library_ids'] and options['all']):
+            raise CommandError("copy_libraries_from_v1_to_v2 requires one or more <library_id>s or the --all flag.")
+
+        if (not options['library_ids'] and not options['all']) or (options['library_ids'] and options['all']):
+            raise CommandError("copy_libraries_from_v1_to_v2 requires one or more <library_id>s or the --all flag.")
+
+        if options['all']:
+            store = modulestore()
+            if query_yes_no(self.CONFIRMATION_PROMPT, default="no"):
+                v1_library_keys = [
+                    library.location.library_key.replace(branch=None) for library in store.get_libraries()
+                ]
+            else:
+                return
+        else:
+            v1_library_keys = list(map(self._parse_library_key, options['library_ids']))
+
+        create_library_task_group = group([
+            delete_v2_library_from_v1_library.s(str(v1_library_key), options['collection_uuid'][0])
+            if options['uncopy']
+            else create_v2_library_from_v1_library.s(str(v1_library_key), options['collection_uuid'][0])
+            for v1_library_key in v1_library_keys
+        ])
+
+        group_result = create_library_task_group.apply_async().get()
+        if options['output_csv']:
+            with open(options['output_csv'][0], 'w', encoding='utf-8', newline='') as output_writer:
+                output_writer.writerow("v1_library_id", "v2_library_id", "status", "error_msg")
+                for result in group_result:
+                    output_writer.write(result.keys())
+        log.info(group_result)

cms/djangoapps/contentstore/tasks.py

@@ -19,6 +19,7 @@
 from django.contrib.auth import get_user_model
 from django.core.exceptions import SuspiciousOperation
 from django.core.files import File
+from django.db.transaction import atomic
 from django.test import RequestFactory
 from django.utils.text import get_valid_filename
 from edx_django_utils.monitoring import (
@@ -30,9 +31,10 @@
 from olxcleaner.exceptions import ErrorLevel
 from olxcleaner.reporting import report_error_summary, report_errors
 from opaque_keys.edx.keys import CourseKey
-from opaque_keys.edx.locator import LibraryLocator
+from opaque_keys.edx.locator import LibraryLocator, LibraryLocatorV2
 from organizations.api import add_organization_course, ensure_organization
-from organizations.models import OrganizationCourse
+from organizations.exceptions import InvalidOrganizationException
+from organizations.models import Organization, OrganizationCourse
 from path import Path as path
 from pytz import UTC
 from user_tasks.models import UserTaskArtifact, UserTaskStatus
@@ -47,13 +49,17 @@
 from cms.djangoapps.contentstore.storage import course_import_export_storage
 from cms.djangoapps.contentstore.utils import initialize_permissions, reverse_usage_url, translation_language
 from cms.djangoapps.models.settings.course_metadata import CourseMetadata
+
 from common.djangoapps.course_action_state.models import CourseRerunState
 from common.djangoapps.student.auth import has_course_author_access
+from common.djangoapps.student.roles import CourseInstructorRole, CourseStaffRole, LibraryUserRole
 from common.djangoapps.util.monitoring import monitor_import_failure
 from openedx.core.djangoapps.content.learning_sequences.api import key_supports_outlines
+from openedx.core.djangoapps.content_libraries import api as v2contentlib_api
 from openedx.core.djangoapps.course_apps.toggles import exams_ida_enabled
 from openedx.core.djangoapps.discussions.tasks import update_unit_discussion_state_from_discussion_blocks
 from openedx.core.djangoapps.embargo.models import CountryAccessRule, RestrictedCourse
+from openedx.core.lib.blockstore_api import get_collection
 from openedx.core.lib.extract_tar import safetar_extractall
 from xmodule.contentstore.django import contentstore  # lint-amnesty, pylint: disable=wrong-import-order
 from xmodule.course_block import CourseFields  # lint-amnesty, pylint: disable=wrong-import-order
@@ -790,7 +796,6 @@ def get_error_by_type(error_type):
 def handle_course_import_exception(courselike_key, exception, status, known=True):
     """
     Handle course import exception and fail task status.
-
     Arguments:
         courselike_key: A locator identifies a course resource.
         exception: Exception object
@@ -808,3 +813,159 @@ def handle_course_import_exception(courselike_key, exception, status, known=True
 
     if status.state != UserTaskStatus.FAILED:
         status.fail(task_fail_message)
+
+
+def _parse_organization(org_name):
+    """Find a matching organization name, if one does not exist, specify that this is the *unspecfied* organization"""
+    try:
+        ensure_organization(org_name)
+    except InvalidOrganizationException:
+        return 'None'
+    return Organization.objects.get(short_name=org_name)
+
+
+def copy_v1_user_roles_into_v2_library(v2_library_key, v1_library_key):
+    """
+    write the access and edit permissions of a v1 library into a v2 library.
+    """
+
+    def _get_users_by_access_level(v1_library_key):
+        """
+        Get a permissions object for a library which contains a list of user IDs for every V2 permissions level,
+        based on V1 library roles.
+        The following mapping exists for a library:
+        V1 Library Role -> V2 Permission Level
+        LibraryUserRole -> READ_LEVEL
+        CourseStaffRole -> AUTHOR_LEVEL
+        CourseInstructorRole -> ADMIN_LEVEL
+        """
+        permissions = {}
+        permissions[v2contentlib_api.AccessLevel.READ_LEVEL] = list(LibraryUserRole(v1_library_key).users_with_role())
+        permissions[v2contentlib_api.AccessLevel.AUTHOR_LEVEL] = list(CourseStaffRole(v1_library_key).users_with_role())
+        permissions[v2contentlib_api.AccessLevel.ADMIN_LEVEL] = list(
+            CourseInstructorRole(v1_library_key).users_with_role()
+        )
+        return permissions
+
+    permissions = _get_users_by_access_level(v1_library_key)
+    for access_level in permissions.keys():  # lint-amnesty, pylint: disable=consider-iterating-dictionary
+        for user in permissions[access_level]:
+            v2contentlib_api.set_library_user_permissions(v2_library_key, user, access_level)
+
+
+def _create_copy_content_task(v2_library_key, v1_library_key):
+    """
+    spin up a celery task to import the V1 Library's content into the V2 library.
+    This utalizes the fact that course and v1 library content is stored almost identically.
+    """
+    return v2contentlib_api.import_blocks_create_task(v2_library_key, v1_library_key)
+
+
+def _create_metadata(v1_library_key, collection_uuid):
+    """instansiate an index for the V2 lib in the collection"""
+
+    store = modulestore()
+    v1_library = store.get_library(v1_library_key)
+    collection = get_collection(collection_uuid).uuid
+    # To make it easy, all converted libs are complex, meaning they can contain problems, videos, and text
+    library_type = 'complex'
+    org = _parse_organization(v1_library.location.library_key.org)
+    slug = v1_library.location.library_key.library
+    title = v1_library.display_name
+    #  V1 libraries do not have descriptions.
+    description = ''
+    #  permssions & license are most restrictive.
+    allow_public_learning = False
+    allow_public_read = False
+    library_license = ''  # '' = ALL_RIGHTS_RESERVED
+    with atomic():
+        return v2contentlib_api.create_library(
+            collection,
+            library_type,
+            org,
+            slug,
+            title,
+            description,
+            allow_public_learning,
+            allow_public_read,
+            library_license
+        )
+
+
+@shared_task(time_limit=30)
+@set_code_owner_attribute
+def delete_v2_library_from_v1_library(v1_library_key_string, collection_uuid):
+    """
+    For a V1 Library, delete the matching v2 library, where the library is the result of the copy operation
+    This method relys on _create_metadata failling for LibraryAlreadyExists in order to obtain the v2 slug.
+    """
+    v1_library_key = CourseKey.from_string(v1_library_key_string)
+    v2_library_key = LibraryLocatorV2.from_string('lib:' + v1_library_key.org + ':' + v1_library_key.course)
+
+    try:
+        v2contentlib_api.delete_library(v2_library_key)
+        return {
+            "v1_library_id": v1_library_key_string,
+            "v2_library_id": v2_library_key,
+            "status": "SUCCESS",
+            "msg": None
+        }
+    except Exception as error:  # lint-amnesty, pylint: disable=broad-except
+        return {
+            "v1_library_id": v1_library_key_string,
+            "v2_library_id": v2_library_key,
+            "status": "FAILED",
+            "msg": f"Exception: {v2_library_key} did not delete: {error}"
+        }
+
+
+@shared_task(time_limit=30)
+@set_code_owner_attribute
+def create_v2_library_from_v1_library(v1_library_key_string, collection_uuid):
+    """
+    write the metadata, permissions, and content of a v1 library into a v2 library in the given collection.
+    """
+
+    v1_library_key = CourseKey.from_string(v1_library_key_string)
+
+    LOGGER.info(f"Copy Library task created for library: {v1_library_key}")
+
+    try:
+        v2_library_metadata = _create_metadata(v1_library_key, collection_uuid)
+
+    except v2contentlib_api.LibraryAlreadyExists:
+        return {
+            "v1_library_id": v1_library_key_string,
+            "v2_library_id": None,
+            "status": "FAILED",
+            "msg": f"Exception: LibraryAlreadyExists {v1_library_key_string} aleady exists"
+        }
+
+    try:
+        _create_copy_content_task(v2_library_metadata.key, v1_library_key)
+    except Exception as error:  # lint-amnesty, pylint: disable=broad-except
+        return {
+            "v1_library_id": v1_library_key_string,
+            "v2_library_id": str(v2_library_metadata.key),
+            "status": "FAILED",
+            "msg":
+            f"Could not import content from {v1_library_key_string} into {str(v2_library_metadata.key)}: {str(error)}"
+        }
+
+    try:
+        copy_v1_user_roles_into_v2_library(v2_library_metadata.key, v1_library_key)
+    except Exception as error:  # lint-amnesty, pylint: disable=broad-except
+        return {
+            "v1_library_id": v1_library_key_string,
+            "v2_library_id": str(v2_library_metadata.key),
+            "status": "FAILED",
+            "msg":
+            f"Could not copy permissions from {v1_library_key_string} into {str(v2_library_metadata.key)}: {str(error)}"
+        }
+
+    return {
+        "v1_library_id": v1_library_key_string,
+        "v2_library_id": str(v2_library_metadata.key),
+        "status": "SUCCESS",
+        "msg": None
+    }

openedx/core/djangoapps/content_libraries/api.py

@@ -70,7 +70,13 @@
 from elasticsearch.exceptions import ConnectionError as ElasticConnectionError
 from lxml import etree
 from opaque_keys.edx.keys import LearningContextKey, UsageKey
-from opaque_keys.edx.locator import BundleDefinitionLocator, LibraryLocatorV2, LibraryUsageLocatorV2
+from opaque_keys.edx.locator import (
+    BundleDefinitionLocator,
+    LibraryLocatorV2,
+    LibraryUsageLocatorV2,
+    LibraryLocator as LibraryLocatorV1
+)
+
 from organizations.models import Organization
 from xblock.core import XBlock
 from xblock.exceptions import XBlockNotFoundError
@@ -1160,7 +1166,6 @@ def import_block(self, modulestore_key):
         """
         Import a single modulestore block.
         """
-
         block_data = self.get_block_data(modulestore_key)
 
         # Get or create the block in the library.
@@ -1270,6 +1275,8 @@ def get_export_keys(self, course_key):
         Retrieve the course from modulestore and traverse its content tree.
         """
         course = self.modulestore.get_course(course_key)
+        if isinstance(course_key, LibraryLocatorV1):
+            course = self.modulestore.get_library(course_key)
         export_keys = set()
         blocks_q = collections.deque(course.get_children())
         while blocks_q:

setup.cfg

@@ -115,11 +115,12 @@ ignore_imports =
     # cms.djangoapps.export_course_metadata.tasks
     #  -> openedx.core.djangoapps.schedules.content_highlights
     #  -> lms.djangoapps.courseware.block_render & lms.djangoapps.courseware.model_data
-    openedx.core.djangoapps.content_libraries.* -> lms.djangoapps.grades.api
-    # cms.djangoapps.contentstore.tasks -> openedx.core.djangoapps.content_libraries.models
-    # -> openedx.core.djangoapps.content_libraries.apps
-    # -> openedx.core.djangoapps.content_libraries.signal_handlers
+    openedx.core.djangoapps.content_libraries.* -> lms.djangoapps.*.*
+    # cms.djangoapps.contentstore.tasks -> openedx.core.djangoapps.content_libraries.[various]
     # -> lms.djangoapps.grades.api
+    openedx.core.djangoapps.xblock.*.* -> lms.djangoapps.*.*
+    # cms.djangoapps.contentstore.tasks -> openedx.core.djangoapps.content_libraries.[various] -> openedx.core.djangoapps.xblock.[various]
+    # -> lms.djangoapps.courseware & lms.djangoapps.courseware.grades
     openedx.core.djangoapps.schedules.content_highlights -> lms.djangoapps.courseware.*
     # cms.djangoapps.contentstore.[various]
     #  -> openedx.core.lib.gating.api