diff --git a/changelog.d/20230414_161756_kdmc_assets_sh.md b/changelog.d/20230414_161756_kdmc_assets_sh.md new file mode 100644 index 0000000000..16a35ea173 --- /dev/null +++ b/changelog.d/20230414_161756_kdmc_assets_sh.md @@ -0,0 +1,12 @@ + + + + +- [Feature] Add `EDX_PLATFORM_NEW_ASSET_SYSTEM` flag, defaulting off. When enabled, the experimental new edx-platform asset processing system will be used. The new system should produce identical output, but quicker and with better logging. In the future, this will become the default, supporting further image build optimizations. (@kdmccormick) diff --git a/tutor/templates/build/openedx/Dockerfile b/tutor/templates/build/openedx/Dockerfile index 69dfdd7ab9..fce8356acd 100644 --- a/tutor/templates/build/openedx/Dockerfile +++ b/tutor/templates/build/openedx/Dockerfile @@ -167,6 +167,11 @@ ENV PATH /openedx/bin:${PATH} {{ patch("openedx-dockerfile-pre-assets") }} +# For the experimental new asset processing: +# https://github.com/openedx/edx-platform/issues/31604 +ENV EDX_PLATFORM_NEW_ASSET_PROCESSING="{{ EDX_PLATFORM_NEW_ASSET_PROCESSING }}" +ENV EDX_PLATFORM_ASSET_OPTS="--static-root /openedx/staticfiles --theme-dirs /openedx/themes" + # Collect production assets. By default, only assets from the default theme # will be processed. This makes the docker image lighter and faster to build. # Only the custom themes added to /openedx/themes will be compiled. diff --git a/tutor/templates/build/openedx/bin/openedx-assets b/tutor/templates/build/openedx/bin/openedx-assets index 1b89434d67..aa9439c8cb 100755 --- a/tutor/templates/build/openedx/bin/openedx-assets +++ b/tutor/templates/build/openedx/bin/openedx-assets @@ -4,6 +4,7 @@ import argparse import os import subprocess import sys +import shlex import traceback from path import Path @@ -11,7 +12,10 @@ from path import Path from pavelib import assets +DEFAULT_ENV = "prod" +DEFAULT_SYSTEMS = ["lms", "cms"] DEFAULT_STATIC_ROOT = "/openedx/staticfiles" +DEFAULT_THEMES = ["all"] DEFAULT_THEMES_DIR = "/openedx/themes" @@ -25,11 +29,11 @@ def main(): npm.set_defaults(func=run_npm) build = subparsers.add_parser("build", help="Build all assets") - build.add_argument("-e", "--env", choices=["prod", "dev"], default="prod") + build.add_argument("-e", "--env", choices=["prod", "dev"], default=DEFAULT_ENV) build.add_argument("--theme-dirs", nargs="+", default=[DEFAULT_THEMES_DIR]) - build.add_argument("--themes", nargs="+", default=["all"]) + build.add_argument("--themes", nargs="+", default=DEFAULT_THEMES) build.add_argument("-r", "--static-root", default=DEFAULT_STATIC_ROOT) - build.add_argument("--systems", nargs="+", default=["lms", "cms"]) + build.add_argument("--systems", nargs="+", default=DEFAULT_SYSTEMS) build.set_defaults(func=run_build) xmodule = subparsers.add_parser("xmodule", help="Process assets from xmodule") @@ -37,7 +41,7 @@ def main(): webpack = subparsers.add_parser("webpack", help="Run webpack") webpack.add_argument("-r", "--static-root", default=DEFAULT_STATIC_ROOT) - webpack.add_argument("-e", "--env", choices=["prod", "dev"], default="prod") + webpack.add_argument("-e", "--env", choices=["prod", "dev"], default=DEFAULT_ENV) webpack.set_defaults(func=run_webpack) common = subparsers.add_parser( @@ -86,7 +90,72 @@ def main(): watch_themes.set_defaults(func=run_watch_themes) args = parser.parse_args() - args.func(args) + if should_use_new_implementation(): + new_command = get_new_command(args) + print("Running experimental new asset processing implementation!") + print("You can also run this directly by using the shell command:") + print(shlex.join(get_new_command(args))) + subprocess.check_call(new_command) + else: + args.func(args) + + +default_args = [ + "--themes-dirs", + DEFAULT_THEMES_DIR, + "--static-root", + DEFAULT_STATIC_ROOT, +] + + +def should_use_new_implementation(): + return os.environ.get("EDX_PLATFORM_NEW_ASSET_PROCESSING", "").lower() == "true" + + +def get_new_command(args): + if args.func is run_collect: + return [ + "bash", + "-c", + "./manage.py lms collecstatic --noinput && ./manage.py cms collecstatic --noinput", + ] + return [ + "scripts/build-assets.sh", + *{ + run_build: [], + run_xmodule: ["xmodule"], + run_npm: ["npm"], + run_webpack: ["webpack"], + run_common: ["css"], + run_themes: ["themes"], + run_watch_themes: ["--watch"] + }[args.func], + *( + ["--env", args.env] + if getattr(args, "env", DEFAULT_ENV) != DEFAULT_ENV + else [] + ), + *( + ["--static-root", args.static_root] + if getattr(args, "static_root", DEFAULT_STATIC_ROOT) != DEFAULT_STATIC_ROOT + else [] + ), + *( + ["--systems", *args.systems] + if set(getattr(args, "systems", DEFAULT_SYSTEMS)) != set(DEFAULT_SYSTEMS) + else [] + ), + *( + ["--themes", *args.themes] + if set(getattr(args, "themes", DEFAULT_THEMES)) != set(DEFAULT_THEMES) + else [] + ), + *( + ["--theme-dirs", *args.theme_dirs] + if getattr(args, "theme_dirs", [DEFAULT_THEMES_DIR]) != [DEFAULT_THEMES_DIR] + else [] + ), + ] def run_build(args): @@ -187,7 +256,6 @@ def list_subdirectories(path): if os.path.isdir(os.path.join(path, subpath)) ] - class ThemeWatcher(assets.SassWatcher): def __init__(self, theme_dir): super(ThemeWatcher, self).__init__() diff --git a/tutor/templates/config/defaults.yml b/tutor/templates/config/defaults.yml index 0bed4ca544..f0f38fa4ef 100644 --- a/tutor/templates/config/defaults.yml +++ b/tutor/templates/config/defaults.yml @@ -21,6 +21,7 @@ DOCKER_IMAGE_REDIS: "docker.io/redis:6.2.6" DOCKER_IMAGE_SMTP: "docker.io/devture/exim-relay:4.95-r0-2" EDX_PLATFORM_REPOSITORY: "https://github.com/openedx/edx-platform.git" EDX_PLATFORM_VERSION: "{{ OPENEDX_COMMON_VERSION }}" +EDX_PLATFORM_NEW_ASSET_PROCESSING: false ELASTICSEARCH_HOST: "elasticsearch" ELASTICSEARCH_PORT: 9200 ELASTICSEARCH_SCHEME: "http"