Skip to content
This repository has been archived by the owner on Aug 22, 2022. It is now read-only.

Upgrade requirements for python 3.8 #902

Draft
wants to merge 8 commits into
base: master
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions circle.yml
Original file line number Diff line number Diff line change
Expand Up @@ -145,7 +145,7 @@ jobs:

build:
docker:
- image: circleci/python:3.6-stretch-node-browsers
- image: circleci/python:3.8-buster-node-browsers
environment:
DEBUG: 'true'
DEFAULT_FORK: 'open-craft/edx-platform'
Expand Down Expand Up @@ -241,7 +241,7 @@ jobs:

e2e-tests:
docker:
- image: circleci/python:3.6-stretch-node-browsers
- image: circleci/python:3.8-buster-node-browsers
environment:
DEBUG: 'true'
DEFAULT_FORK: 'open-craft/edx-platform'
Expand Down Expand Up @@ -325,7 +325,7 @@ jobs:

coverage:
docker:
- image: circleci/python:3.6-stretch-node
- image: circleci/python:3.8-buster-node
steps:
- checkout
- restore_cache:
Expand All @@ -338,7 +338,7 @@ jobs:
pip install --upgrade pip
pip install --upgrade virtualenv
sudo apt-get update; sudo apt-get install python2.7-dev
sudo apt-get install mysql-client
sudo apt-get install default-mysql-client
pip install -r requirements.txt
pip install -r cleanup_utils/requirements.txt
sudo apt-get install postgresql-client
Expand All @@ -358,7 +358,7 @@ jobs:

cleanup:
docker:
- image: circleci/python:3.6-stretch
- image: circleci/python:3.8-buster
environment:
LOAD_BALANCER_FRAGMENT_NAME_PREFIX: 'integration-'
DJANGO_SETTINGS_MODULE: 'opencraft.settings'
Expand Down
2 changes: 1 addition & 1 deletion debian_packages.lst
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ libffi-dev
libmysqlclient-dev
libpq-dev
make
mysql-client
default-mysql-client
postgresql-client
python-dev
python2.7-dev
Expand Down
25 changes: 25 additions & 0 deletions instance/migrations/0151_studio_oauth.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
# Generated by Django 2.2.24 on 2022-05-18 14:02

from django.db import migrations, models
import django.utils.crypto
import functools


class Migration(migrations.Migration):

dependencies = [
('instance', '0150_auto_20220309_1120'),
]

operations = [
migrations.AddField(
model_name='openedxinstance',
name='studio_oauth_key',
field=models.CharField(blank=True, default=functools.partial(django.utils.crypto.get_random_string, *(), **{'allowed_chars': 'abcdefghijklmnopqrstuvwxyz', 'length': 6}), max_length=6),
),
migrations.AddField(
model_name='openedxinstance',
name='studio_oauth_secret',
field=models.CharField(blank=True, default=functools.partial(django.utils.crypto.get_random_string, *(), **{'length': 32}), max_length=32),
),
]
7 changes: 7 additions & 0 deletions instance/models/mixins/domain_names.py
Original file line number Diff line number Diff line change
Expand Up @@ -242,6 +242,13 @@ def url(self):
"""
return u'https://{}/'.format(self.domain)

@property
def mfe_url(self):
"""
MFE URL.
"""
return u'https://{}/'.format(self.mfe_domain)

@property
def studio_url(self):
"""
Expand Down
67 changes: 66 additions & 1 deletion instance/models/mixins/openedx_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,12 @@ def _get_configuration_variables(self):
# EDXAPP_ENV_EXTRA as the other two overwrite this.
"EDXAPP_CMS_ENV_EXTRA": {
**extra_config,
# BB-6228 Studio Oauth login on Maple
'SOCIAL_AUTH_EDX_OAUTH2_KEY': self.instance.studio_oauth_key,
'SOCIAL_AUTH_EDX_OAUTH2_SECRET': self.instance.studio_oauth_secret,
'SOCIAL_AUTH_EDX_OAUTH2_URL_ROOT': self.instance.url,
'SOCIAL_AUTH_EDX_OAUTH2_PUBLIC_URL_ROOT': self.instance.url,
'SESSION_COOKIE_NAME': settings.STUDIO_SESSION_COOKIE_NAME,
},
"EDXAPP_LMS_ENV_EXTRA": {
**extra_config,
Expand All @@ -115,6 +121,10 @@ def _get_configuration_variables(self):
# Explicitly set TOS_AND_HONOR link to '#' so that they are not combined.
"TOS_AND_HONOR": "#"
},
# BB-6228 Studio Oauth login on Maple
'IDA_LOGOUT_URI_LIST': [
'{}/logout/'.format(self.instance.studio_domain)
]
},
"EDXAPP_MKTG_URL_LINK_MAP": {
# set the tos and honor pages separately. Required for koa and above.
Expand Down Expand Up @@ -439,6 +449,11 @@ def _get_configuration_variables(self):
"FUNCTION": "retirement_lms_retire",
},
],
# BB-6228 Studio Oauth login on Maple
'EDXAPP_CMS_SOCIAL_AUTH_EDX_OAUTH2_KEY': self.instance.studio_oauth_key,
'EDXAPP_CMS_SOCIAL_AUTH_EDX_OAUTH2_SECRET': self.instance.studio_oauth_secret,
'EDXAPP_CMS_URL_ROOT': self.instance.studio_url,
'EDXAPP_CMS_LOGOUT_URL': '{}/logout/'.format(self.instance.studio_url),
}

if self.smtp_relay_settings:
Expand Down Expand Up @@ -542,7 +557,57 @@ def _get_configuration_variables(self):
"EDXAPP_LMS_WRITABLE_GRADEBOOK_URL": "https://{}/gradebook".format(self.instance.mfe_domain),
"EDXAPP_PROFILE_MICROFRONTEND_URL": "https://{}/profile/u/".format(self.instance.mfe_domain),
})

if self._is_openedx_release_in(['maple']):
template.update({
'EDXAPP_VIDEO_IMAGE_SETTINGS': {
'VIDEO_IMAGE_MAX_BYTES': 2097152,
'VIDEO_IMAGE_MIN_BYTES': 2048,
'STORAGE_KWARGS': {
'location': 'video-images/',
'bucket': '{{ EDXAPP_AWS_STORAGE_BUCKET_NAME }}'
}
},
'EDXAPP_VIDEO_TRANSCRIPTS_SETTINGS': {
'VIDEO_TRANSCRIPTS_MAX_BYTES': 3145728,
'STORAGE_KWARGS': {
'location': 'video-transcripts/',
'bucket': '{{ EDXAPP_AWS_STORAGE_BUCKET_NAME }}'
}
},
'EDXAPP_CELERY_BROKER_TRANSPORT': 'amqp',
'MFE_BASE': self.instance.mfe_domain,
'MFE_SITE_NAME': self.instance.name,
'MFE_BASE_SCHEMA': 'https',
'MFE_DEPLOY_NGINX_PORT': 80,
'MFE_DEPLOY_COMMON_HOSTNAME': self.instance.mfe_domain,
'MFE_DEPLOY_VERSION': self.openedx_release,
'MFES': [
{
'name': 'gradebook',
'repo': 'frontend-app-gradebook',
'public_path': '/gradebook/'
}
],
'EDXAPP_CSRF_COOKIE_SECURE': True,
'EDXAPP_SESSION_COOKIE_SECURE': True,
'EDXAPP_ENABLE_CORS_HEADERS': True,
'EDXAPP_ENABLE_CROSS_DOMAIN_CSRF_COOKIE': True,
'EDXAPP_CROSS_DOMAIN_CSRF_COOKIE_DOMAIN': '.{}'.format(self.instance.domain),
'EDXAPP_CROSS_DOMAIN_CSRF_COOKIE_NAME': 'cross-domain-cookie-mfe',
'EDXAPP_CORS_ORIGIN_WHITELIST': [
self.instance.url.rstrip('/'),
self.instance.lms_preview_url.rstrip('/'),
self.instance.studio_url.rstrip('/'),
self.instance.mfe_url.rstrip('/'),
],
'EDXAPP_LMS_WRITABLE_GRADEBOOK_URL': '{}gradebook'.format(self.instance.mfe_url),
'MFE_FLAGS_SETUP_FLAGS_LIST': [
'grades.writable_gradebook',
'course_home.course_home_use_legacy_frontend',
'courseware.use_legacy_frontend'
],
'EDXAPP_SESSION_COOKIE_DOMAIN': '.{}'.format(self.instance.domain),
})
if settings.OPENEDX_ORACLEJDK_URL:
template["oraclejdk_url"] = settings.OPENEDX_ORACLEJDK_URL

Expand Down
46 changes: 46 additions & 0 deletions instance/models/mixins/openedx_studio_oauth.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
# -*- coding: utf-8 -*-
#
# OpenCraft -- tools to aid developing and hosting free software projects
# Copyright (C) 2015-2019 OpenCraft <[email protected]>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
"""
Open edX studio oauth sso mixin
"""
import functools
import string

from django.db import models
from django.utils.crypto import get_random_string


class OpenEdXStudioOauthMixin(models.Model):
"""
An instance that configures Studio OAuth2 SSO
"""

studio_oauth_key = models.CharField(
max_length=6,
default=functools.partial(get_random_string, length=6, allowed_chars=string.ascii_lowercase),
blank=True,
)
studio_oauth_secret = models.CharField(
max_length=32,
blank=True,
default=functools.partial(get_random_string, length=32),
)

class Meta:
abstract = True
2 changes: 2 additions & 0 deletions instance/models/openedx_instance.py
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@
from instance.models.mixins.openedx_periodic_builds import OpenEdXPeriodicBuildsMixin
from instance.models.mixins.openedx_site_configuration import OpenEdXSiteConfigurationMixin
from instance.models.mixins.openedx_storage import OpenEdXStorageMixin
from instance.models.mixins.openedx_studio_oauth import OpenEdXStudioOauthMixin
from instance.models.mixins.openedx_theme import OpenEdXThemeMixin
from instance.models.mixins.secret_keys import SecretKeyInstanceMixin
from instance.models.openedx_appserver import OpenEdXAppConfiguration
Expand All @@ -65,6 +66,7 @@ class OpenEdXInstance(
OpenEdXDatabaseMixin,
OpenEdXMonitoringMixin,
OpenEdXStorageMixin,
OpenEdXStudioOauthMixin,
OpenEdXThemeMixin,
OpenEdXPeriodicBuildsMixin,
OpenEdXSiteConfigurationMixin,
Expand Down
4 changes: 2 additions & 2 deletions instance/tests/integration/factories/instance.py
Original file line number Diff line number Diff line change
Expand Up @@ -79,8 +79,8 @@ def create(cls, *args, **kwargs): # pylint: disable=arguments-differ
# or a release candidate tag will work. We point both the edx-platform and the configuration
# versions to the branch "integration" in our own forks. These branches are based on the
# corresponding openedx_release versions from upstream, but can contain custom modifications.
openedx_release = 'open-release/lilac.2'
openedx_release = 'open-release/maple.3'
configuration_source_repo_url = 'https://github.com/open-craft/configuration.git'
configuration_version = 'integration-lilac'
edx_platform_repository_url = 'https://github.com/open-craft/edx-platform.git'
edx_platform_commit = 'opencraft-release/lilac.2'
edx_platform_commit = 'opencraft-release/maple.3'
99 changes: 99 additions & 0 deletions instance/tests/models/test_openedx_studio_oauth_mixins.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,99 @@
# -*- coding: utf-8 -*-
#
# OpenCraft -- tools to aid developing and hosting free software projects
# Copyright (C) 2015-2019 OpenCraft <[email protected]>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
"""
OpenEdXInstance Studio Oauth Mixins - Tests
"""

# Imports #####################################################################

from django.conf import settings

import yaml

from instance.tests.base import TestCase
from instance.tests.models.factories.openedx_appserver import make_test_appserver
from instance.tests.models.factories.openedx_instance import OpenEdXInstanceFactory


# Tests #######################################################################


class OpenEdXStudioOauthMixinTestCase(TestCase):
"""
Tests for OpenEdXStudioOauthMixin
"""

def test_studio_oauth_settings(self):
"""
Test that Studio Oauth key and secret are generated for instance
"""
instance = OpenEdXInstanceFactory()
self.assertIsNot(instance.studio_oauth_key, None)
self.assertIsNot(instance.studio_oauth_secret, None)

def _check_generated_settings(self, instance):
"""
Check required settings are rendered on the appserver
"""
appserver = make_test_appserver(instance)
parsed_vars = yaml.load(appserver.configuration_settings, Loader=yaml.SafeLoader)
self.assertEqual(
parsed_vars['EDXAPP_CMS_ENV_EXTRA']['SOCIAL_AUTH_EDX_OAUTH2_KEY'],
instance.studio_oauth_key)
self.assertEqual(
parsed_vars['EDXAPP_CMS_ENV_EXTRA']['SOCIAL_AUTH_EDX_OAUTH2_SECRET'],
instance.studio_oauth_secret)
self.assertEqual(
parsed_vars['EDXAPP_CMS_ENV_EXTRA']['SOCIAL_AUTH_EDX_OAUTH2_URL_ROOT'],
instance.url)
self.assertEqual(
parsed_vars['EDXAPP_CMS_ENV_EXTRA']['SOCIAL_AUTH_EDX_OAUTH2_PUBLIC_URL_ROOT'],
instance.url)
self.assertEqual(
parsed_vars['EDXAPP_CMS_ENV_EXTRA']['SESSION_COOKIE_NAME'],
settings.STUDIO_SESSION_COOKIE_NAME)
self.assertIn(
'{}/logout/'.format(instance.studio_domain),
parsed_vars['EDXAPP_LMS_ENV_EXTRA']['IDA_LOGOUT_URI_LIST'])
self.assertEqual(
parsed_vars['EDXAPP_CMS_SOCIAL_AUTH_EDX_OAUTH2_KEY'],
instance.studio_oauth_key)
self.assertEqual(
parsed_vars['EDXAPP_CMS_SOCIAL_AUTH_EDX_OAUTH2_SECRET'],
instance.studio_oauth_secret)
self.assertEqual(
parsed_vars['EDXAPP_CMS_URL_ROOT'],
instance.studio_url)
self.assertEqual(
parsed_vars['EDXAPP_CMS_LOGOUT_URL'],
'{}/logout/'.format(instance.studio_url))

def test_studio_oauth_master_settings(self):
"""
Test that Studio Oauth key and secret are passed to the appserver on master
"""
instance = OpenEdXInstanceFactory()
self._check_generated_settings(instance)

def test_studio_oauth_production_settings(self):
"""
Test that Studio Oauth key and secret are passed to the appserver on Maple
"""
instance = OpenEdXInstanceFactory(openedx_release=settings.STABLE_EDX_PLATFORM_COMMIT)
self._check_generated_settings(instance)
8 changes: 6 additions & 2 deletions opencraft/settings.py
Original file line number Diff line number Diff line change
Expand Up @@ -448,7 +448,7 @@

# Git ref for stable Open edX release. Used as a default refspec for
# configuration, edx-platform, forum, notifier, xqueue, and certs when creating production instances.
OPENEDX_RELEASE_STABLE_REF = env('OPENEDX_RELEASE_STABLE_REF', default='open-release/lilac.2')
OPENEDX_RELEASE_STABLE_REF = env('OPENEDX_RELEASE_STABLE_REF', default='open-release/maple.3')

# The edx-platform repository used by default for production instances
STABLE_EDX_PLATFORM_REPO_URL = env(
Expand All @@ -460,7 +460,7 @@
STABLE_CONFIGURATION_REPO_URL = env(
'STABLE_CONFIGURATION_REPO_URL', default=DEFAULT_CONFIGURATION_REPO_URL
)
STABLE_CONFIGURATION_VERSION = env('STABLE_CONFIGURATION_VERSION', default='opencraft-release/lilac.2')
STABLE_CONFIGURATION_VERSION = env('STABLE_CONFIGURATION_VERSION', default='opencraft-release/maple.3')

# The name of the security group to use for edxapp App servers.
# This is used to set appropriate firewall rules to prevent external access to
Expand Down Expand Up @@ -1024,3 +1024,7 @@
GROVE_DEFAULT_REPOSITORY_TRIGGER_TOKEN = env.str('GROVE_DEFAULT_REPOSITORY_TRIGGER_TOKEN', default=GITLAB_API_TOKEN)
DEFAULT_GITLAB_USER = env.str('DEFAULT_GITLAB_USER')
DEFAULT_GITLAB_PERSONAL_ACCESS_TOKEN = env.str('DEFAULT_GITLAB_PERSONAL_ACCESS_TOKEN', default=GITLAB_API_TOKEN)

# Studio Oauth SSO

STUDIO_SESSION_COOKIE_NAME = 'studio_sessionid'
2 changes: 1 addition & 1 deletion requirements/base.in
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ channels_redis==2.4.0
CherryPy<18.2
colour==0.1.5
cookies<2.3
cryptography==3.2.1
cryptography
dictdiffer<0.9
Django<2.3,>=2.2
django-compressor==2.4
Expand Down
Loading