Merge branch 'master' into maxi/hotfix-router-class

.github/issue_template.md

@@ -0,0 +1,17 @@
+## Expected Behavior
+
+
+## Actual Behavior
+
+
+## Steps to Reproduce the Problem
+
+  1.
+  1.
+  1.
+
+## Specifications
+
+  - Version:
+  - Platform:
+  - Subsystem:

.github/pull_request_template.md

@@ -0,0 +1,31 @@
+## Change description
+
+> Description here
+
+## Type of change
+- [ ] Bug fix (fixes an issue)
+- [ ] New feature (adds functionality)
+
+## Related issues
+
+> Fix [#1]() 
+
+## Checklists
+
+### Development
+
+- [ ] Lint rules pass locally
+- [ ] Application changes have been tested thoroughly
+- [ ] Automated tests covering modified code pass
+
+### Security
+
+- [ ] Security impact of change has been considered
+- [ ] Code follows company security practices and guidelines
+
+### Code review 
+
+- [ ] Pull request has a descriptive title and context useful to a reviewer. Screenshots or screencasts are attached as necessary
+- [ ] "Ready for review" label attached and reviewers assigned
+- [ ] Changes have been reviewed by at least one other contributor
+- [ ] Pull request linked to task tracker where applicable

.github/workflows/python-publish.yml

@@ -0,0 +1,44 @@
+# This workflow will upload a Python Package using Twine when a release is created
+# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries
+
+name: Build and upload Python package
+
+on:
+  release:
+    types: [published]
+  pull_request:
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Use Node.js 12.x
+      uses: actions/setup-node@v1
+      with:
+        node-version: 12.x
+    - name: Build frontend
+      run: |
+        cd frontend
+        npm install -g 'yarn@<2'
+        yarn
+        yarn build
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: '3.x'
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install build
+    - name: Build package
+      run: python -m build
+    - name: 'Publish package (only for dev releases)'
+      # For now avoiding publishing production releases:
+      #   - The setuptools guide: https://packaging.python.org/guides/distributing-packages-using-setuptools/#pre-release-versioning
+      uses: pypa/gh-action-pypi-publish@27b31702a0e7fc50959f5ad993c78deac1bdfc29
+      with:
+        user: __token__
+        password: ${{ secrets.PYPI_API_TOKEN }}
+      if: ${{ startsWith(github.ref, 'refs/tags') && contains(github.ref, '.dev') }}

.github/workflows/tests.yml

@@ -15,9 +15,9 @@ jobs:
           - python: 2.7
             tox-env: py27-hawthorn_multisite
           - python: 3.5
-            tox-env: py35-juniper
-          - python: 3.8
-            tox-env: py38-juniper
+            tox-env: py35-juniper_community
+          - python: 3.5
+            tox-env: py35-juniper_multisite
           - python: 3.8
             tox-env: lint
           - python: 3.8

README.rst

@@ -8,10 +8,10 @@ Reporting and data retrieval app for `Open edX <https://open.edx.org/>`__.
 
 .. _notice_section:
 
-TBD Q1 2021 - Figures release 0.4 "Official"
+TBD Q3 2021 - Figures release 0.4 "Official"
 ============================================
 
-**Most recent release**: "0.4.dev8" on January 21, 2021
+**Most recent release**: "0.4.dev10" on June 6, 2021
 
 Figures is iterating through "0.4.dev" development releases on the way to
 an official 0.4 release.

devsite/devsite/settings.py

@@ -73,6 +73,7 @@
     'debug_toolbar',
     'webpack_loader',
     'organizations',
+    'waffle',
     'devsite',
     'figures',
 

devsite/devsite/test_settings.py

@@ -13,6 +13,7 @@
     update_celerybeat_schedule,
     # TODO: https://appsembler.atlassian.net/browse/RED-673
     # update_webpack_loader,
+    update_celery_routes,
 )
 
 
@@ -62,6 +63,7 @@ def root(*args):
     'rest_framework',
     'django_countries',
     'django_filters',
+    'waffle',
     'devsite',
     'webpack_loader',
     'figures',
@@ -148,6 +150,13 @@ def root(*args):
     'FIGURES': {},  # This variable is patched by the Figures' `lms_production.py` settings module.
 }
 
+PRJ_SETTINGS = {
+    'CELERY_ROUTES': "app.celery.routes"
+}
+
+FIGURES_PIPELINE_TASKS_ROUTING_KEY = ""
+
 # TODO: https://appsembler.atlassian.net/browse/RED-673
 # update_webpack_loader(WEBPACK_LOADER, ENV_TOKENS)
-update_celerybeat_schedule(CELERYBEAT_SCHEDULE, ENV_TOKENS)
+update_celerybeat_schedule(CELERYBEAT_SCHEDULE, ENV_TOKENS, FIGURES_PIPELINE_TASKS_ROUTING_KEY)
+update_celery_routes(PRJ_SETTINGS, ENV_TOKENS, FIGURES_PIPELINE_TASKS_ROUTING_KEY)

devsite/requirements/ginkgo.txt

@@ -38,6 +38,7 @@ django-model-utils==2.3.1
 django-environ==0.4.5
 django-celery==3.2.1
 jsonfield==1.0.3  # Version used in Ginkgo. Hawthorn uses version 2.0.2
+django-waffle==0.12.0
 
 ##
 ## Documentation (Sphinx) dependencies

devsite/requirements/hawthorn_base.txt

@@ -21,6 +21,7 @@ pytz==2016.10
 Django==1.11.29
 djangorestframework==3.6.3
 django-countries==4.6.1
+django-waffle==0.12.0 
 
 # edx-platform hawthorn does not use django-extensions
 django-extensions==1.5.9

devsite/requirements/juniper_base.txt

@@ -11,7 +11,7 @@
 
 celery==3.1.26.post2
 django-celery==3.3.1
-six==1.14.0
+six==1.15.0
 
 # Faker is used to seed mock data in devsite
 Faker==4.1.0 
@@ -24,17 +24,21 @@ pytz==2020.1
 ## Django package dependencies
 ##
 
-Django==2.2.15
+Django==2.2.24
 
 djangorestframework==3.9.4
 django-countries==5.5
 django-webpack-loader==0.7.0
 django-model-utils==4.0.0
 django-filter==2.3.0
 django-environ==0.4.5
+django-waffle==0.18.0
 
 jsonfield==2.1.1
 
+# For
+
+
 ##
 ## Documentation (Sphinx) dependencies
 ##
@@ -49,7 +53,6 @@ Sphinx==3.1.2
 edx-opaque-keys[django]==2.1.0
 #edx-drf-extensions==6.0.0
 
-edx-organizations==5.2.0
 
 ##
 ## Devsite 
@@ -75,3 +78,4 @@ pytest-cov==2.8.1
 tox==3.15.0
 freezegun==0.3.12
 edx-lint==1.4.1
+mock==3.0.5

devsite/requirements/juniper_community.txt

@@ -0,0 +1,5 @@
+# Requirements needed for Juniper community environment
+
+-r juniper_base.txt
+
+edx-organizations==5.2.0

devsite/requirements/juniper_multisite.txt

@@ -0,0 +1,6 @@
+# Requirements needed for Juniper multisite environment
+
+-r juniper_base.txt
+
+# Organization/site mapping requires Appsembler's fork
+git+https://github.com/appsembler/[email protected]

figures/backfill.py

@@ -12,15 +12,15 @@
 
 from figures.compat import CourseNotFound
 from figures.sites import (
-  get_course_enrollments_for_site,
-  get_student_modules_for_site
+    get_course_enrollments_for_site,
+    get_student_modules_for_site
 )
 from figures.pipeline.site_monthly_metrics import fill_month
 from figures.models import EnrollmentData
 
 
 def backfill_monthly_metrics_for_site(site, overwrite=False):
-    """Backfill all historical site metrics for the specified site
+    """Backfill specified months' historical site metrics for the specified site
     """
     site_sm = get_student_modules_for_site(site)
     if not site_sm:

figures/helpers.py

@@ -54,7 +54,6 @@
 
 from dateutil.parser import parse as dateutil_parse
 from dateutil.relativedelta import relativedelta
-from dateutil.rrule import rrule, MONTHLY
 
 from opaque_keys.edx.keys import CourseKey
 import six
@@ -194,9 +193,15 @@ def days_in_month(month_for):
     return num_days_in_month
 
 
+def is_past_date(val):
+    return as_date(val) < datetime.date.today()
+
+
 # TODO: Consider changing name to 'months_back_iterator' or similar
 def previous_months_iterator(month_for, months_back):
-    """Iterator returns a year,month tuple for n months including the month_for
+    """Iterator returns a year,month tuple for n months including the month_for.
+    months_back is a misnomer as iteration includes the start month.  The actual
+    number of previous months iterated is months_back minus one.
 
     month_for is either a date, datetime, or tuple with year and month
     months back is the number of months to iterate
@@ -206,9 +211,10 @@ def previous_months_iterator(month_for, months_back):
         # TODO make sure we've got just two values in the tuple
         month_for = datetime.date(year=month_for[0], month=month_for[1], day=1)
     if isinstance(month_for, (datetime.datetime, datetime.date)):
-        start_month = month_for - relativedelta(months=months_back)
+        start_month = month_for - relativedelta(months=(max(0, months_back - 1)))
 
-    for dt in rrule(freq=MONTHLY, dtstart=start_month, until=month_for):
+    for n_months in range(max(1, months_back)):
+        dt = start_month + relativedelta(months=n_months)
         last_day_of_month = days_in_month(month_for=dt)
         yield (dt.year, dt.month, last_day_of_month)
 

figures/management/__init__.py

@@ -0,0 +1,2 @@
+"""
+"""

figures/management/base.py

@@ -0,0 +1,75 @@
+"""
+Management command base classes for Figures.
+"""
+from datetime import datetime
+
+from django.contrib.sites.models import Site
+from django.core.management.base import BaseCommand
+
+from figures import helpers
+from figures.sites import get_sites
+
+
+class BaseBackfillCommand(BaseCommand):
+    '''Base class for Figures backfill management commands with common options.
+    '''
+    def get_site_ids(self, identifier=None):
+        """Quick-n-dirty function to let the caller choose the site id or domain.
+        If no identifier is passed, return all available Sites.
+        Let the 'get' fail if record can't be found from the identifier.
+        Returns Site ids for passing to Celery tasks.
+        Note that at present, none of the tasks handle more than one specified Site.
+        """
+        if not identifier:
+            sites = get_sites()
+        else:
+            try:
+                filter_arg = dict(pk=int(identifier))
+            except ValueError:
+                filter_arg = dict(domain=identifier)
+            sites = Site.objects.filter(**filter_arg)
+        return [site.id for site in sites]
+
+    def get_date(self, date_str=None):
+        '''Return a datetime.date from a string or NoneType.
+        '''
+        try:
+            return helpers.as_date(date_str)
+        except TypeError:
+            return datetime.today().date()
+
+    def add_arguments(self, parser):
+        '''
+        '''
+        # TODO: allow passing the queue to use.  Warn if no_delay specified.
+        parser.add_argument(
+            '--site',
+            help='backfill a specific site. provide numeric id or domain name',
+            default=None
+        )
+        # TODO: handle date start later than date end
+        parser.add_argument(
+            '--date_start',
+            help='date for which we start backfilling data',
+        )
+        parser.add_argument(
+            '--date_end',
+            help='date for which we end backfilling data',
+        )
+        parser.add_argument(
+            '--no-delay',
+            action='store_true',
+            default=False,
+            help='Disable the celery "delay" directive'
+        )
+        parser.add_argument(
+            '--overwrite',
+            action='store_true',
+            default=False,
+            help='Overwrite metrics records if they exist for the given date'
+        )
+
+    def print_exc(self, metrics_type, date, exc_message):
+        print("Could not populate {} for {}. Exception was {}".format(
+            metrics_type, date, exc_message)
+        )

figures/management/commands/__init__.py

@@ -0,0 +1,2 @@
+"""Django management commands for Figures.
+"""