Use the new edx-platform assets build + other improvements

tutor/commands/compose.py

@@ -20,6 +20,7 @@
 from tutor.exceptions import TutorError
 from tutor.tasks import BaseComposeTaskRunner
 from tutor.types import Config
+from tutor.utils import execute as execute_shell
 
 
 class ComposeTaskRunner(BaseComposeTaskRunner):
@@ -114,6 +115,10 @@ def launch(
         click.echo(fmt.title("Docker image updates"))
         context.invoke(dc_command, command="pull")
 
+    if bindmount.get_mounts(config):
+        click.echo(fmt.title("Copying build artifacts into bind-mounted directories"))
+        context.invoke(copyartifacts)
+
     click.echo(fmt.title("Starting the platform in detached mode"))
     context.invoke(start, detach=True)
 
@@ -365,6 +370,63 @@ def copyfrom(
     )
 
 
+@click.command(
+    help="TODO describe"
+)
+@click.argument(
+    "mount_paths",
+    metavar="mount_path",
+    nargs=-1,
+    type=click.Path(dir_okay=True, file_okay=False, resolve_path=True),
+)
+@click.pass_obj
+def copyartifacts(context: BaseComposeContext, mount_paths: list[Path]) -> None:
+    """
+    TODO write docstring
+    TODO move this to `tutor images build <IMAGE> --populate` ??
+    """
+    config = tutor_config.load(context.root)
+    host_mount_paths: list[str] = [
+        os.path.abspath(os.path.expanduser(mount_path))
+        for mount_path
+        in mount_paths or bindmount.get_mounts(config)
+    ]
+
+    # Sort out the (source, target) pairs by image name so that we can work one-at-a-time later.
+    copies_by_image: dict[str, tuple[str, str]] = {}
+    for host_mount_path in host_mount_paths:
+        mount_name = os.path.basename(host_mount_path)
+        for image, container_mount_path in hooks.Filters.COMPOSE_MOUNTS.iterate(mount_name):
+            if image != "openedx-dev":
+                # TODO just do openedx-dev for now -- will generalize this soon.
+                continue
+            for path_in_mount in hooks.Filters.IMAGES_BUILD_MOUNT_ARTIFACTS.iterate(mount_name, image):
+                source = f"{container_mount_path}/{path_in_mount}"
+                target = f"{host_mount_path}/{path_in_mount}"
+                copies_by_image.setdefault(image, []).append((source, target))
+
+    container_name = "tutor_mounts_populate_temp" # TODO: improve name?
+    runner = context.job_runner(config)
+
+    # For each image: create a temporary container, do the copy operations, and then rm the container.
+    if not copies_by_image:
+        fmt.echo_alert("Nothing to copy.")
+        return
+    for image, copies in copies_by_image.items():
+        execute_shell("docker", "rm", "-f", container_name)
+        for name, _, tag, __ in hooks.Filters.IMAGES_BUILD.iterate(config):
+            if name == image:
+                execute_shell("docker", "create", "--name", container_name, tag)
+                break
+        else:
+            raise ValueError(f"unknown image name: {name}")
+        for source, target in copies:
+            execute_shell("rm", "-rf", target)                                   # Wipe any existing artifact.
+            execute_shell("sh", "-c", f'mkdir -p "$(dirname "{target}")"')       # Ensure parent dirs exist.
+            execute_shell("docker", "cp", f"{container_name}:{source}", target)  # Actually do the copy.
+    execute_shell("docker", "rm", "-f", container_name)
+
+
 @click.command(
     short_help="Run a command in a running container",
     help=(
@@ -426,6 +488,17 @@ def dc_command(
     context.job_runner(config).docker_compose(command, *args)
 
 
+@hooks.Filters.APP_PUBLIC_HOSTS.add()
+def _edx_platform_public_hosts(
+    hosts: list[str], context_name: t.Literal["local", "dev"]
+) -> list[str]:
+    if context_name == "dev":
+        hosts += ["{{ LMS_HOST }}:8000", "{{ CMS_HOST }}:8001"]
+    else:
+        hosts += ["{{ LMS_HOST }}", "{{ CMS_HOST }}"]
+    return hosts
+
+
 hooks.Filters.ENV_TEMPLATE_VARIABLES.add_item(("iter_mounts", bindmount.iter_mounts))
 
 
@@ -439,6 +512,7 @@ def add_commands(command_group: click.Group) -> None:
     command_group.add_command(dc_command)
     command_group.add_command(run)
     command_group.add_command(copyfrom)
+    command_group.add_command(copyartifacts)
     command_group.add_command(execute)
     command_group.add_command(logs)
     command_group.add_command(status)

tutor/commands/images.py

@@ -34,7 +34,7 @@ def _add_core_images_to_build(
                 image,
                 os.path.join("build", image),
                 tutor_config.get_typed(config, tag, str),
-                (),
+                ("--target=production",),
             )
         )
 

tutor/hooks/catalog.py

@@ -401,6 +401,34 @@ def your_filter_callback(some_data):
     #:   ``os.path.basename(path)``) to conditionally add mounts.
     IMAGES_BUILD_MOUNTS: Filter[list[tuple[str, str]], [str]] = Filter()
 
+    #: Relative paths of build artifacts, to be copied into a host-mounted folder from a service image.
+    #: (TODO update this description since Quince changes)
+    #:
+    #: Docker images contain many build artifacts, such as generated assets and packaging metadata, which
+    #: must exist at runtime in order for services to run properly. When a user bind-mounts a directory
+    #: from their host machine, there is no guarantee that the host directory will contain those essential
+    #: artifacts, and regenerating them from source may be time consuming. To remedy this, Tutor provides
+    #: the ``copyartifacts`` command, which efficiently copies the necessary artifacts from the original image
+    #: into the host directory. This command is also automatically run as part of ``launch``.
+    #:
+    #: The ``COMPOSE_MOUNT_ARTIFACTS`` filter tells Tutor which artifacts must be copied from which
+    #: service for any given host-mounted directory. By default, for edx-platform, this includes
+    #: several directories such as ``node_modules`` and ``lms/static/css``, to be copied from the lms
+    #: service's image.
+    #:
+    #: Note that any given artifact should only be specified once in this filter. If an artifact
+    #: exists on an image used by multiple services, choose one of those service for it to be
+    #: copied from.
+    #:
+    #: :parameter artifacts list[str]: files or directories considered build artifacts for the given
+    #:   host-mounted folder in the given service. Paths must be relative to the mount directory.
+    #: :parameter str mount_name: basename of a host-mounted folder.
+    #: :parameter image: internal name of a Docker image from which artifacts will be copied. The ``IMAGES_BUILD_MOUNTS``
+    #:   filter should map ``mount_name`` somewhere in this image; that mount location will be used
+    #:   as the source for ``artifacts``.
+    IMAGES_BUILD_MOUNT_ARTIFACTS: Filter[list[str], [str, str]] = Filter()
+
+
     #: List of images to be pulled when we run ``tutor images pull ...``.
     #:
     #: :parameter list[tuple[str, str]] tasks: list of ``(name, tag)`` tuples.

tutor/plugins/openedx.py

@@ -44,6 +44,27 @@ def _mount_edx_platform_build(
     return volumes
 
 
+@hooks.Filters.IMAGES_BUILD_MOUNT_ARTIFACTS.add()
+def _populate_edx_platform(
+    paths_to_copy: list[str], mount_name: str, image: str
+) -> list[tuple[str, str]]:
+    """
+    TODO describe
+    """
+    if mount_name == "edx-platform" and image in ["openedx", "openedx-dev"]:
+        paths_to_copy += [
+            "Open_edX.egg-info",
+            "node_modules",
+            "lms/static/css",
+            "lms/static/certificates/css",
+            "cms/static/css",
+            "common/static/bundles",
+            "common/static/common/js/vendor",
+            "common/static/common/css/vendor",
+        ]
+    return paths_to_copy
+
+
 @hooks.Filters.COMPOSE_MOUNTS.add()
 def _mount_edx_platform_compose(
     volumes: list[tuple[str, str]], name: str
@@ -55,20 +76,22 @@ def _mount_edx_platform_compose(
     if name == "edx-platform":
         path = "/openedx/edx-platform"
         volumes.append(("openedx", path))
+        volumes.append(("openedx-dev", path))
     return volumes
 
 
 # Auto-magically bind-mount xblock directories and some common dependencies.
-hooks.Filters.MOUNTED_DIRECTORIES.add_items(
-    [
-        ("openedx", r".*[xX][bB]lock.*"),
-        ("openedx", "edx-enterprise"),
-        ("openedx", "edx-ora2"),
-        ("openedx", "edx-search"),
-        ("openedx", "openedx-learning"),
-        ("openedx", r"platform-plugin-.*"),
-    ]
-)
+for openedx_image in ["openedx", "openedx-dev"]:
+    hooks.Filters.MOUNTED_DIRECTORIES.add_items(
+        [
+            (openedx_image, r".*[xX][bB]lock.*"),
+            (openedx_image, "edx-enterprise"),
+            (openedx_image, "edx-ora2"),
+            (openedx_image, "edx-search"),
+            (openedx_image, "openedx-learning"),
+            (openedx_image, r"platform-plugin-.*"),
+        ]
+    )
 
 
 @hooks.Filters.MOUNTED_DIRECTORIES.add(priority=hooks.priorities.LOW)