Merge branch 'main' into dependabot/npm_and_yarn/frontend/url-parse-1…

….5.10

devsite/.env.example

@@ -15,8 +15,8 @@ DATABASE_URL=mysql://figures_user:[email protected]:3306/figures-db
 # Set which expected Open edX release mocks for devsite to use.
 # Valid options are: "GINKGO", "HAWTHORN", "JUNIPER"
 #
-# If not specified here, then "HAWTHORN" is used
-OPENEDX_RELEASE=JUNIPER
+# If not specified here, then "JUNIPER" is used
+# OPENEDX_RELEASE=JUNIPER
 
 # Enable/disable Figures multisite mode in devsite
 # This also requires

devsite/devsite/settings.py

@@ -14,6 +14,7 @@
 from figures.settings.lms_production import (
     update_webpack_loader,
     update_celerybeat_schedule,
+    update_celery_routes,
 )
 
 DEVSITE_BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
@@ -22,7 +23,7 @@
 env = environ.Env(
     DEBUG=(bool, True),
     ALLOWED_HOSTS=(list, []),
-    OPENEDX_RELEASE=(str, 'HAWTHORN'),
+    OPENEDX_RELEASE=(str, 'JUNIPER'),
     FIGURES_IS_MULTISITE=(bool, False),
     ENABLE_DEVSITE_CELERY=(bool, True),
     ENABLE_OPENAPI_DOCS=(bool, False),
@@ -89,11 +90,15 @@
 if ENABLE_DEVSITE_CELERY:
     INSTALLED_APPS.append('djcelery')
 
-# certificates app
+
 if OPENEDX_RELEASE == 'GINKGO':
+    # certificates and courseware do NOT use the `lms.djangoapps.` namespace
+    # prefix on Ginkgo. See here: https://github.com/appsembler/figures/issues/433
     INSTALLED_APPS.append('certificates')
-    INSTALLED_APPS.append('lms.djangoapps.courseware')
+    INSTALLED_APPS.append('courseware')
 elif OPENEDX_RELEASE == 'HAWTHORN':
+    # Yes, this is correct, certificates was updated to uses the full namespace
+    # and courseware has not yet been updated
     INSTALLED_APPS.append('lms.djangoapps.certificates')
     INSTALLED_APPS.append('courseware')
 else:
@@ -240,8 +245,16 @@
 # We have an empty dict here to replicate behavior in the LMS
 ENV_TOKENS = {}
 
+PRJ_SETTINGS = {
+    'CELERY_ROUTES': "app.celery.routes"
+}
+
+FIGURES_PIPELINE_TASKS_ROUTING_KEY = ""
+
 update_webpack_loader(WEBPACK_LOADER, ENV_TOKENS)
-update_celerybeat_schedule(CELERYBEAT_SCHEDULE, ENV_TOKENS)
+update_celerybeat_schedule(CELERYBEAT_SCHEDULE, ENV_TOKENS, FIGURES_PIPELINE_TASKS_ROUTING_KEY)
+update_celery_routes(PRJ_SETTINGS, ENV_TOKENS, FIGURES_PIPELINE_TASKS_ROUTING_KEY)
+
 
 # Used by Django Debug Toolbar
 INTERNAL_IPS = [

devsite/devsite/test_settings.py

@@ -28,7 +28,7 @@ def root(*args):
 
 
 env = environ.Env(
-    OPENEDX_RELEASE=(str, 'HAWTHORN'),
+    OPENEDX_RELEASE=(str, 'JUNIPER'),
 )
 
 environ.Env.read_env(join(dirname(dirname(__file__)), '.env'))

figures/course.py

@@ -0,0 +1,115 @@
+"""Course specific module for Figures
+
+## This module defined a `Course` class for data retrieval
+
+Initialy created to do the following:
+
+1. Reduce duplication in Figures "pipeline"
+2. Build stronger course context to make Figures programming easier
+
+## Background summary
+
+A course id is globally unique as it has the identify of the organization and
+organizations are globally unique.
+
+## Design to think about - Enrollment class
+
+Build on Django's lazy eval for querysets to create also an `Enrollment` class
+that provides interfaces for `enrollment.date_for` and abstracts this bit of
+mess that enrollments and student modules do NOT associate and instead we need
+to query back and forth with `user_id` and `course_id`.
+"""
+from __future__ import absolute_import
+from django.db.models import Q
+from figures.compat import CourseEnrollment, StudentModule
+from figures.helpers import (
+    as_course_key,
+    as_date,
+)
+from figures.sites import (
+    get_site_for_course,
+)
+
+
+class Course(object):
+    """Representation of a Course.
+
+    The impetus for this class was dealing with querying for course enrollment
+    and student module records for a specific course and for dates and date
+    ranges for the course
+
+    ## Architecture goal
+
+    **Start simple and don't build the kitchen sink into here right away just
+    because this class exists**
+
+    ## Data under consideration to have this class handle
+
+    * enrollments created on a date, before, after, between. However this would
+    just be a convenience as the `.enrollments` property returns a queryset that
+    can be filtered on `.created`
+    """
+    def __init__(self, course_id):
+        """
+        Initial version, we pass in a course ID and cast to a course key as an
+        instance attribute. Later on, add `CourseLike` to abstract course identity
+        so we can stop worrying about "Is it a string repretentation of a course or
+        is it a CourseKey?"
+        """
+        self.course_key = as_course_key(course_id)
+
+        # Improvement: Consider doing lazy evaluation
+        self.site = get_site_for_course(self.course_id)
+
+    def __repr__(self):
+        return '{}.{} <{}>'.format(self.__module__,
+                                   self.__class__.__name__,
+                                   str(self.course_key))
+
+    def __str__(self):
+        return self.__repr__()
+
+    @property
+    def course_id(self):
+        """Returns string representation of the course id
+        """
+        return str(self.course_key)
+
+    @property
+    def enrollments(self):
+        """Returns CourseEnrollment queryset for the course
+        """
+        return CourseEnrollment.objects.filter(course_id=self.course_key)
+
+    @property
+    def student_modules(self):
+        """Returns StudentModule queryset for enrollments in the course
+        """
+        return StudentModule.objects.filter(course_id=self.course_key)
+
+    def student_modules_active_on_date(self, date_for):
+        """Returns StudentModule queryset active on the date
+        Active is if there was a `created` or `modified` field for the given date
+
+        NOTE: We need to do this instead of simplly `modified__date=date_for`
+        because we still have to support Django 1.8/Ginkgo
+        """
+        date_for = as_date(date_for)
+        q_created = Q(created__year=date_for.year,
+                      created__month=date_for.month,
+                      created__day=date_for.day)
+        q_modified = Q(modified__year=date_for.year,
+                       modified__month=date_for.month,
+                       modified__day=date_for.day)
+        return self.student_modules.filter(q_created | q_modified)
+
+    def enrollments_active_on_date(self, date_for):
+        """Return CourseEnrollment queryset for enrollments active on the date
+
+        Looks for student modules modified on the specified date and returns
+        matching CourseEnrollment records
+        """
+        sm = self.student_modules_active_on_date(date_for)
+        user_ids = sm.values('student_id').distinct()
+        return CourseEnrollment.objects.filter(course_id=self.course_key,
+                                               user_id__in=user_ids)

figures/helpers.py

@@ -1,6 +1,7 @@
 """Helper functions to make data handling and conversions easier
 
 # Figures 0.3.13 - Defining scope of this module
+# Figures 0.4.x - Yep, this module is still handy and the scope hasn't exploded
 
 The purpose of this module is to provide conveniece methods around commonly
 executed statements. These convenience methods should serve as shorthand for
@@ -34,6 +35,8 @@
 
 ## What does not belong here?
 
+* Most importantly, if you have to import from another figures module, it does
+  not belong here!
 * "Feature" functionality does not belong here
 * Long functions do not belong here
 * Code that communicates outside of Figures does not belong here. No database,
@@ -42,7 +45,10 @@
 This is not an exhaustive list. We'll grow it as needed.
 
 An important point is that we should not expect this module to be a permanent
-home for functionality.
+home for the functionality included. As we evolve Figures, we may find functions
+here that have a stronger context with another module. For example, we've got
+a decent set of date oriented functions that are candidatdes for a datetime and
+date handling module.
 """
 
 from __future__ import absolute_import
@@ -188,6 +194,15 @@ def prev_day(val):
     return days_from(val, -1)
 
 
+def utc_yesterday():
+    """Get "yesterday" form the utc datetime
+
+    We primarily use this for the daily metrics collection. However, it proves
+    handy as a convenience function for exploring data in the Django shell
+    """
+    return prev_day(datetime.datetime.utcnow().date())
+
+
 def days_in_month(month_for):
     _, num_days_in_month = calendar.monthrange(month_for.year, month_for.month)
     return num_days_in_month

figures/management/commands/backfill_figures_daily_metrics.py

@@ -70,16 +70,11 @@ def handle(self, *args, **options):
                 del kwargs['site_id']  # not implemented for experimental
             else:
                 metrics_func = populate_daily_metrics
-            # try:
+
             if options['no_delay']:
                 metrics_func(**kwargs)
             else:
                 metrics_func.delay(**kwargs)  # pragma: no cover
-            # except Exception as e:  # pylint: disable=bare-except
-            #     if options['ignore_exceptions']:
-            #         self.print_exc("daily", dt, e.message)
-            #     else:
-            #         raise
 
             print('END: Backfill Figures daily metrics metrics for: {}'.format(dt))