Skip to content

Commit

Permalink
Merge branch 'master' into testing-python-version
Browse files Browse the repository at this point in the history
  • Loading branch information
awais786 committed Feb 5, 2024
2 parents 8a7e4fc + 6edc0c2 commit 247596c
Show file tree
Hide file tree
Showing 224 changed files with 5,321 additions and 1,694 deletions.
13 changes: 4 additions & 9 deletions .github/workflows/migrations-check.yml
Original file line number Diff line number Diff line change
Expand Up @@ -16,16 +16,11 @@ jobs:
os: [ ubuntu-20.04 ]
python-version: [ '3.9' ]
# 'pinned' is used to install the latest patch version of Django
# within the global constraint i.e. Django==3.2.21 in current case
# because we have global constraint of Django<4.2
django-version: ["pinned", "4.2"]
# within the global constraint i.e. Django==4.2.8 in current case
# because we have global constraint of Django<4.2
django-version: ["pinned"]
mongo-version: ["4"]
mysql-version: ["5.7", "8"]
# excluding mysql5.7 with Django 4.2 since Django 4.2 has
# dropped support for MySQL<8
exclude:
- django-version: "4.2"
mysql-version: "5.7"
mysql-version: ["8"]
services:
mongo:
image: mongo:${{ matrix.mongo-version }}
Expand Down
1 change: 0 additions & 1 deletion .github/workflows/unit-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@ jobs:
- "3.9"
django-version:
- "pinned"
- "4.2"
# When updating the shards, remember to make the same changes in
# .github/workflows/unit-tests-gh-hosted.yml
shard_name:
Expand Down
11 changes: 10 additions & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -56,8 +56,14 @@ endif
push_translations: ## push source strings to Transifex for translation
i18n_tool transifex push

pull_plugin_translations: ## Pull translations from Transifex for edx_django_utils.plugins for both lms and cms
rm -rf conf/plugins-locale/plugins # Clean up existing atlas translations
mkdir -p conf/plugins-locale/plugins
python manage.py lms pull_plugin_translations --verbose $(ATLAS_OPTIONS)
python manage.py lms compile_plugin_translations

pull_xblock_translations: ## pull xblock translations via atlas
rm -rf conf/plugins-locale # Clean up existing atlas translations
rm -rf conf/plugins-locale/xblock.v1 # Clean up existing atlas translations
rm -rf lms/static/i18n/xblock.v1 cms/static/i18n/xblock.v1 # Clean up existing xblock compiled translations
mkdir -p conf/plugins-locale/xblock.v1/ lms/static/js/xblock.v1-i18n cms/static/js
python manage.py lms pull_xblock_translations --verbose $(ATLAS_OPTIONS)
Expand All @@ -76,6 +82,7 @@ ifeq ($(OPENEDX_ATLAS_PULL),)
i18n_tool validate --verbose
else
make pull_xblock_translations
make pull_plugin_translations
find conf/locale -mindepth 1 -maxdepth 1 -type d -exec rm -r {} \;
atlas pull $(ATLAS_OPTIONS) translations/edx-platform/conf/locale:conf/locale
i18n_tool generate
Expand Down Expand Up @@ -150,6 +157,8 @@ compile-requirements: pre-requirements $(COMMON_CONSTRAINTS_TXT) ## Re-compile *
@# time someone tries to use the outputs.
sed '/^django-simple-history==/d' requirements/common_constraints.txt > requirements/common_constraints.tmp
mv requirements/common_constraints.tmp requirements/common_constraints.txt
sed 's/Django<4.0//g' requirements/common_constraints.txt > requirements/common_constraints.tmp
mv requirements/common_constraints.tmp requirements/common_constraints.txt
pip-compile -v --allow-unsafe ${COMPILE_OPTS} -o requirements/pip.txt requirements/pip.in
pip install -r requirements/pip.txt

Expand Down
3 changes: 3 additions & 0 deletions cms/djangoapps/contentstore/course_info_model.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
from django.http import HttpResponseBadRequest
from django.utils.translation import gettext as _

from cms.djangoapps.contentstore.utils import track_course_update_event
from openedx.core.lib.xblock_utils import get_course_update_items
from xmodule.html_block import CourseInfoBlock # lint-amnesty, pylint: disable=wrong-import-order
from xmodule.modulestore.django import modulestore # lint-amnesty, pylint: disable=wrong-import-order
Expand Down Expand Up @@ -85,6 +86,8 @@ def update_course_updates(location, update, passed_id=None, user=None):

# update db record
save_course_update_items(location, course_updates, course_update_items, user)
# track course update event
track_course_update_event(location.course_key, user, course_update_dict)
# remove status key
if "status" in course_update_dict:
del course_update_dict["status"]
Expand Down
60 changes: 48 additions & 12 deletions cms/djangoapps/contentstore/management/commands/reindex_course.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
import logging
from textwrap import dedent
from time import time
from datetime import date

from django.core.management import BaseCommand, CommandError
from elasticsearch import exceptions
Expand Down Expand Up @@ -38,6 +39,9 @@ def add_arguments(self, parser):
parser.add_argument('--all',
action='store_true',
help='Reindex all courses')
parser.add_argument('--active',
action='store_true',
help='Reindex active courses only')
parser.add_argument('--setup',
action='store_true',
help='Reindex all courses on developers stack setup')
Expand All @@ -58,19 +62,24 @@ def _parse_course_key(self, raw_value):

return result

def handle(self, *args, **options):
def handle(self, *args, **options): # pylint: disable=too-many-statements
"""
By convention set by Django developers, this method actually executes command's actions.
So, there could be no better docstring than emphasize this once again.
"""
course_ids = options['course_ids']
all_option = options['all']
active_option = options['active']
setup_option = options['setup']
readable_option = options['warning']
index_all_courses_option = all_option or setup_option

if (not len(course_ids) and not index_all_courses_option) or (len(course_ids) and index_all_courses_option): # lint-amnesty, pylint: disable=len-as-condition
raise CommandError("reindex_course requires one or more <course_id>s OR the --all or --setup flags.")
if ((not course_ids and not (index_all_courses_option or active_option)) or
(course_ids and (index_all_courses_option or active_option))):
raise CommandError((
"reindex_course requires one or more <course_id>s"
" OR the --all, --active or --setup flags."
))

store = modulestore()

Expand All @@ -79,8 +88,8 @@ def handle(self, *args, **options):
logging.warning('Reducing logging to WARNING level for easier progress tracking')

if index_all_courses_option:
index_names = (CoursewareSearchIndexer.INDEX_NAME, CourseAboutSearchIndexer.INDEX_NAME)
if setup_option:
index_names = (CoursewareSearchIndexer.INDEX_NAME, CourseAboutSearchIndexer.INDEX_NAME)
for index_name in index_names:
try:
searcher = SearchEngine.get_search_engine(index_name)
Expand All @@ -104,22 +113,49 @@ def handle(self, *args, **options):
course_keys = [course.id for course in modulestore().get_courses()]
else:
return
elif active_option:
# in case of --active, we get the list of course keys from all courses
# that are stored in the modulestore and filter out the non-active
all_courses = modulestore().get_courses()

today = date.today()
# We keep the courses that has a start date and either don't have an end date
# or the end date is not in the past.
active_courses = filter(lambda course: course.start
and (not course.end or course.end.date() >= today),
all_courses)
course_keys = list(map(lambda course: course.id, active_courses))

logging.warning(f'Selected {len(course_keys)} active courses over a total of {len(all_courses)}.')

else:
# in case course keys are provided as arguments
course_keys = list(map(self._parse_course_key, course_ids))

total = len(course_keys)
logging.warning(f'Reindexing {total} courses')
reindexed = 0
logging.warning(f'Reindexing {total} courses...')
start = time()

count = 0
success = 0
errors = []

for course_key in course_keys:
try:
count += 1
CoursewareSearchIndexer.do_course_reindex(store, course_key)
reindexed += 1
if reindexed % 10 == 0 or reindexed == total:
now = time()
t = now - start
logging.warning(f'{reindexed}/{total} reindexed in {t:.1f} seconds')
success += 1
if count % 10 == 0 or count == total:
t = time() - start
remaining = total - success - len(errors)
logging.warning(f'{success} courses reindexed in {t:.1f} seconds. {remaining} remaining...')
except Exception as exc: # lint-amnesty, pylint: disable=broad-except
logging.exception('Error indexing course %s due to the error: %s', course_key, exc)
errors.append(course_key)
logging.exception('Error indexing course %s due to the error: %s.', course_key, exc)

t = time() - start
logging.warning(f'{success} of {total} courses reindexed succesfully. Total running time: {t:.1f} seconds.')
if errors:
logging.warning('Reindex failed for %s courses:', len(errors))
for course_key in errors:
logging.warning(course_key)
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
from xmodule.modulestore.django import modulestore # lint-amnesty, pylint: disable=wrong-import-order
from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase # lint-amnesty, pylint: disable=wrong-import-order
from xmodule.modulestore.tests.factories import CourseFactory, LibraryFactory # lint-amnesty, pylint: disable=wrong-import-order
from datetime import datetime, timedelta


@ddt.ddt
Expand All @@ -26,11 +27,18 @@ def setUp(self):
org="test", library="lib2", display_name="run2", default_store=ModuleStoreEnum.Type.split
)

yesterday = datetime.min.today() - timedelta(days=1)

self.first_course = CourseFactory.create(
org="test", course="course1", display_name="run1"
org="test", course="course1", display_name="run1", start=yesterday, end=None
)

self.second_course = CourseFactory.create(
org="test", course="course2", display_name="run1"
org="test", course="course2", display_name="run1", start=yesterday, end=yesterday
)

self.third_course = CourseFactory.create(
org="test", course="course3", display_name="run1", start=None, end=None
)

REINDEX_PATH_LOCATION = (
Expand Down Expand Up @@ -103,7 +111,7 @@ def test_given_all_key_prompts_and_reindexes_all_courses(self):
call_command('reindex_course', all=True)

patched_yes_no.assert_called_once_with(ReindexCommand.CONFIRMATION_PROMPT, default='no')
expected_calls = self._build_calls(self.first_course, self.second_course)
expected_calls = self._build_calls(self.first_course, self.second_course, self.third_course)
self.assertCountEqual(patched_index.mock_calls, expected_calls)

def test_given_all_key_prompts_and_reindexes_all_courses_cancelled(self):
Expand All @@ -116,3 +124,15 @@ def test_given_all_key_prompts_and_reindexes_all_courses_cancelled(self):

patched_yes_no.assert_called_once_with(ReindexCommand.CONFIRMATION_PROMPT, default='no')
patched_index.assert_not_called()

def test_given_active_key_prompt(self):
"""
Test that reindexes all active courses when --active key is given
Active courses have a start date but no end date, or the end date is in the future.
"""
with mock.patch(self.REINDEX_PATH_LOCATION) as patched_index, \
mock.patch(self.MODULESTORE_PATCH_LOCATION, mock.Mock(return_value=self.store)):
call_command('reindex_course', active=True)

expected_calls = self._build_calls(self.first_course)
self.assertCountEqual(patched_index.mock_calls, expected_calls)
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
from .course_team import CourseTeamSerializer
from .course_index import CourseIndexSerializer
from .grading import CourseGradingModelSerializer, CourseGradingSerializer
from .home import CourseHomeSerializer, CourseTabSerializer, LibraryTabSerializer
from .home import CourseHomeSerializer, CourseHomeTabSerializer, LibraryTabSerializer
from .proctoring import (
LimitedProctoredExamSettingsSerializer,
ProctoredExamConfigurationSerializer,
Expand All @@ -21,3 +21,4 @@
VideoUsageSerializer,
VideoDownloadSerializer
)
from .vertical_block import ContainerHandlerSerializer
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ class CourseIndexSerializer(serializers.Serializer):
deprecated_blocks_info = serializers.DictField()
discussions_incontext_feedback_url = serializers.CharField()
discussions_incontext_learnmore_url = serializers.CharField()
discussions_settings = serializers.DictField()
initial_state = InitialIndexStateSerializer()
initial_user_clipboard = serializers.DictField()
language_code = serializers.CharField()
Expand All @@ -29,3 +30,5 @@ class CourseIndexSerializer(serializers.Serializer):
proctoring_errors = ProctoringErrorListSerializer(many=True)
reindex_link = serializers.CharField()
rerun_notification_id = serializers.IntegerField()
advance_settings_url = serializers.CharField()
is_custom_relative_dates_active = serializers.BooleanField()
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ class LibraryViewSerializer(serializers.Serializer):
can_edit = serializers.BooleanField()


class CourseTabSerializer(serializers.Serializer):
class CourseHomeTabSerializer(serializers.Serializer):
archived_courses = CourseCommonSerializer(required=False, many=True)
courses = CourseCommonSerializer(required=False, many=True)
in_process_course_actions = UnsucceededCourseSerializer(many=True, required=False, allow_null=True)
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,92 @@
"""
API Serializers for unit page
"""

from django.urls import reverse
from rest_framework import serializers

from cms.djangoapps.contentstore.helpers import (
xblock_studio_url,
xblock_type_display_name,
)


class ChildAncestorSerializer(serializers.Serializer):
"""
Serializer for representing child blocks in the ancestor XBlock.
"""

url = serializers.SerializerMethodField()
display_name = serializers.CharField(source="display_name_with_default")

def get_url(self, obj):
"""
Method to generate studio URL for the child block.
"""
return xblock_studio_url(obj)


class AncestorXBlockSerializer(serializers.Serializer):
"""
Serializer for representing the ancestor XBlock and its children.
"""

children = ChildAncestorSerializer(many=True)
title = serializers.CharField()
is_last = serializers.BooleanField()


class ContainerXBlock(serializers.Serializer):
"""
Serializer for representing XBlock data. Doesn't include all data about XBlock.
"""

display_name = serializers.CharField(source="display_name_with_default")
display_type = serializers.SerializerMethodField()
category = serializers.CharField()

def get_display_type(self, obj):
"""
Method to get the display type name for the container XBlock.
"""
return xblock_type_display_name(obj)


class ContainerHandlerSerializer(serializers.Serializer):
"""
Serializer for container handler
"""

language_code = serializers.CharField()
action = serializers.CharField()
xblock = ContainerXBlock()
is_unit_page = serializers.BooleanField()
is_collapsible = serializers.BooleanField()
position = serializers.IntegerField(min_value=1)
prev_url = serializers.CharField(allow_null=True)
next_url = serializers.CharField(allow_null=True)
new_unit_category = serializers.CharField()
outline_url = serializers.CharField()
ancestor_xblocks = AncestorXBlockSerializer(many=True)
component_templates = serializers.ListField(child=serializers.DictField())
xblock_info = serializers.DictField()
draft_preview_link = serializers.CharField()
published_preview_link = serializers.CharField()
show_unit_tags = serializers.BooleanField()
user_clipboard = serializers.DictField()
is_fullwidth_content = serializers.BooleanField()
assets_url = serializers.SerializerMethodField()
unit_block_id = serializers.CharField(source="unit.location.block_id")
subsection_location = serializers.CharField(source="subsection.location")

def get_assets_url(self, obj):
"""
Method to get the assets URL based on the course id.
"""

context_course = obj.get("context_course", None)
if context_course:
return reverse(
"assets_handler", kwargs={"course_key_string": context_course.id}
)
return None
Loading

0 comments on commit 247596c

Please sign in to comment.