From 43b7891249d548ecfd52c37fe770cf5497bc32d1 Mon Sep 17 00:00:00 2001 From: Kyle McCormick Date: Fri, 14 Apr 2023 11:25:53 -0400 Subject: [PATCH] feat: add `EDX_PLATFORM_NEW_ASSET_PROCESSING` flag When enabled, this flag tells Tutor's openedx-assets script (which processes assets using Python and edx-platform's Paver tools) to instead proxy through to edx-platform's new asset processing system (`scripts/build-assets.sh` and `./manage.py collectstatic`). Also, when enabled, openedx-assets will print out the shell command that could have been used to directly invoke the new asset processing system. In the future, openedx-assets will be deprecated, and the direct shell command will be the canonical way of compiling assets. Part of: https://github.com/openedx/edx-platform/issues/31798 ADR: https://github.com/openedx/edx-platform/blob/master/docs/decisions/0017-reimplement-asset-processing.rst --- changelog.d/20230414_161756_kdmc_assets_sh.md | 12 +++ tutor/templates/build/openedx/Dockerfile | 5 ++ .../build/openedx/bin/openedx-assets | 80 +++++++++++++++++-- tutor/templates/config/defaults.yml | 1 + 4 files changed, 92 insertions(+), 6 deletions(-) create mode 100644 changelog.d/20230414_161756_kdmc_assets_sh.md diff --git a/changelog.d/20230414_161756_kdmc_assets_sh.md b/changelog.d/20230414_161756_kdmc_assets_sh.md new file mode 100644 index 0000000000..16a35ea173 --- /dev/null +++ b/changelog.d/20230414_161756_kdmc_assets_sh.md @@ -0,0 +1,12 @@ + + + + +- [Feature] Add `EDX_PLATFORM_NEW_ASSET_SYSTEM` flag, defaulting off. When enabled, the experimental new edx-platform asset processing system will be used. The new system should produce identical output, but quicker and with better logging. In the future, this will become the default, supporting further image build optimizations. (@kdmccormick) diff --git a/tutor/templates/build/openedx/Dockerfile b/tutor/templates/build/openedx/Dockerfile index 69dfdd7ab9..fce8356acd 100644 --- a/tutor/templates/build/openedx/Dockerfile +++ b/tutor/templates/build/openedx/Dockerfile @@ -167,6 +167,11 @@ ENV PATH /openedx/bin:${PATH} {{ patch("openedx-dockerfile-pre-assets") }} +# For the experimental new asset processing: +# https://github.com/openedx/edx-platform/issues/31604 +ENV EDX_PLATFORM_NEW_ASSET_PROCESSING="{{ EDX_PLATFORM_NEW_ASSET_PROCESSING }}" +ENV EDX_PLATFORM_ASSET_OPTS="--static-root /openedx/staticfiles --theme-dirs /openedx/themes" + # Collect production assets. By default, only assets from the default theme # will be processed. This makes the docker image lighter and faster to build. # Only the custom themes added to /openedx/themes will be compiled. diff --git a/tutor/templates/build/openedx/bin/openedx-assets b/tutor/templates/build/openedx/bin/openedx-assets index 1b89434d67..aa9439c8cb 100755 --- a/tutor/templates/build/openedx/bin/openedx-assets +++ b/tutor/templates/build/openedx/bin/openedx-assets @@ -4,6 +4,7 @@ import argparse import os import subprocess import sys +import shlex import traceback from path import Path @@ -11,7 +12,10 @@ from path import Path from pavelib import assets +DEFAULT_ENV = "prod" +DEFAULT_SYSTEMS = ["lms", "cms"] DEFAULT_STATIC_ROOT = "/openedx/staticfiles" +DEFAULT_THEMES = ["all"] DEFAULT_THEMES_DIR = "/openedx/themes" @@ -25,11 +29,11 @@ def main(): npm.set_defaults(func=run_npm) build = subparsers.add_parser("build", help="Build all assets") - build.add_argument("-e", "--env", choices=["prod", "dev"], default="prod") + build.add_argument("-e", "--env", choices=["prod", "dev"], default=DEFAULT_ENV) build.add_argument("--theme-dirs", nargs="+", default=[DEFAULT_THEMES_DIR]) - build.add_argument("--themes", nargs="+", default=["all"]) + build.add_argument("--themes", nargs="+", default=DEFAULT_THEMES) build.add_argument("-r", "--static-root", default=DEFAULT_STATIC_ROOT) - build.add_argument("--systems", nargs="+", default=["lms", "cms"]) + build.add_argument("--systems", nargs="+", default=DEFAULT_SYSTEMS) build.set_defaults(func=run_build) xmodule = subparsers.add_parser("xmodule", help="Process assets from xmodule") @@ -37,7 +41,7 @@ def main(): webpack = subparsers.add_parser("webpack", help="Run webpack") webpack.add_argument("-r", "--static-root", default=DEFAULT_STATIC_ROOT) - webpack.add_argument("-e", "--env", choices=["prod", "dev"], default="prod") + webpack.add_argument("-e", "--env", choices=["prod", "dev"], default=DEFAULT_ENV) webpack.set_defaults(func=run_webpack) common = subparsers.add_parser( @@ -86,7 +90,72 @@ def main(): watch_themes.set_defaults(func=run_watch_themes) args = parser.parse_args() - args.func(args) + if should_use_new_implementation(): + new_command = get_new_command(args) + print("Running experimental new asset processing implementation!") + print("You can also run this directly by using the shell command:") + print(shlex.join(get_new_command(args))) + subprocess.check_call(new_command) + else: + args.func(args) + + +default_args = [ + "--themes-dirs", + DEFAULT_THEMES_DIR, + "--static-root", + DEFAULT_STATIC_ROOT, +] + + +def should_use_new_implementation(): + return os.environ.get("EDX_PLATFORM_NEW_ASSET_PROCESSING", "").lower() == "true" + + +def get_new_command(args): + if args.func is run_collect: + return [ + "bash", + "-c", + "./manage.py lms collecstatic --noinput && ./manage.py cms collecstatic --noinput", + ] + return [ + "scripts/build-assets.sh", + *{ + run_build: [], + run_xmodule: ["xmodule"], + run_npm: ["npm"], + run_webpack: ["webpack"], + run_common: ["css"], + run_themes: ["themes"], + run_watch_themes: ["--watch"] + }[args.func], + *( + ["--env", args.env] + if getattr(args, "env", DEFAULT_ENV) != DEFAULT_ENV + else [] + ), + *( + ["--static-root", args.static_root] + if getattr(args, "static_root", DEFAULT_STATIC_ROOT) != DEFAULT_STATIC_ROOT + else [] + ), + *( + ["--systems", *args.systems] + if set(getattr(args, "systems", DEFAULT_SYSTEMS)) != set(DEFAULT_SYSTEMS) + else [] + ), + *( + ["--themes", *args.themes] + if set(getattr(args, "themes", DEFAULT_THEMES)) != set(DEFAULT_THEMES) + else [] + ), + *( + ["--theme-dirs", *args.theme_dirs] + if getattr(args, "theme_dirs", [DEFAULT_THEMES_DIR]) != [DEFAULT_THEMES_DIR] + else [] + ), + ] def run_build(args): @@ -187,7 +256,6 @@ def list_subdirectories(path): if os.path.isdir(os.path.join(path, subpath)) ] - class ThemeWatcher(assets.SassWatcher): def __init__(self, theme_dir): super(ThemeWatcher, self).__init__() diff --git a/tutor/templates/config/defaults.yml b/tutor/templates/config/defaults.yml index 0bed4ca544..f0f38fa4ef 100644 --- a/tutor/templates/config/defaults.yml +++ b/tutor/templates/config/defaults.yml @@ -21,6 +21,7 @@ DOCKER_IMAGE_REDIS: "docker.io/redis:6.2.6" DOCKER_IMAGE_SMTP: "docker.io/devture/exim-relay:4.95-r0-2" EDX_PLATFORM_REPOSITORY: "https://github.com/openedx/edx-platform.git" EDX_PLATFORM_VERSION: "{{ OPENEDX_COMMON_VERSION }}" +EDX_PLATFORM_NEW_ASSET_PROCESSING: false ELASTICSEARCH_HOST: "elasticsearch" ELASTICSEARCH_PORT: 9200 ELASTICSEARCH_SCHEME: "http"