Remove the whole concept of development/production mounts

.github/actions/run-docker/action.yml

@@ -23,10 +23,6 @@ inputs:
     description: 'Docker target to run (development|production)'
     required: false
     default: 'production'
-  mount:
-    description: 'Mount host files at runtime? (development|production)'
-    required: false
-    default: 'production'
   deps:
     description: 'Which dependencies to install at runtime? (development|production)'
     required: false
@@ -44,7 +40,6 @@ runs:
           DOCKER_DIGEST="${{ inputs.digest }}" \
           DOCKER_TARGET="${{ inputs.target }}" \
           OLYMPIA_UID="$(id -u)" \
-          OLYMPIA_MOUNT="${{ inputs.mount }}" \
           OLYMPIA_DEPS="${{ inputs.deps }}" \
           DATA_BACKUP_SKIP="${{ inputs.data_backup_skip }}" \
           DOCKER_WAIT="true"

.github/workflows/_test_check.yml

@@ -46,7 +46,6 @@ jobs:
     name: |
       version: '${{ matrix.version }}' |
       target: '${{ matrix.target }}' |
-      mount: '${{ matrix.mount }}' |
       deps: '${{ matrix.deps }}'
     strategy:
       fail-fast: false
@@ -57,9 +56,6 @@ jobs:
         target:
           - development
           - production
-        mount:
-          - development
-          - production
         deps:
           - development
           - production
@@ -72,7 +68,6 @@ jobs:
             Values passed to the action:
             version: ${{ matrix.version }}
             target: ${{ matrix.target }}
-            mount: ${{ matrix.mount }}
             deps: ${{ matrix.deps }}
           EOF
       - name: ${{ matrix.version == 'local' && 'Uncached Build' || 'Pull' }} Check
@@ -84,7 +79,6 @@ jobs:
         with:
           version: ${{ matrix.version }}
           target: ${{ matrix.target }}
-          mount: ${{ matrix.mount }}
           deps: ${{ matrix.deps }}
           run: make check
       - name: Cached Build Check
@@ -93,7 +87,6 @@ jobs:
         with:
           version: ${{ matrix.version }}
           target: ${{ matrix.target }}
-          mount: ${{ matrix.mount }}
           deps: ${{ matrix.deps }}
           run: echo true
 

.github/workflows/_test_main.yml

@@ -88,7 +88,6 @@ jobs:
           services: ''
           digest: ${{ inputs.digest }}
           version: ${{ inputs.version }}
-          mount: development
           deps: development
           run: |
             split="--splits ${{ needs.test_config.outputs.splits }}"

.github/workflows/ci.yml

@@ -87,7 +87,6 @@ jobs:
         with:
           digest: ${{ needs.build.outputs.digest }}
           version: ${{ needs.build.outputs.version }}
-          mount: development
           deps: development
           target: development
           run: |
@@ -140,7 +139,6 @@ jobs:
         with:
           digest: ${{ needs.build.outputs.digest }}
           version: ${{ needs.build.outputs.version }}
-          mount: development
           deps: development
           run: make extract_locales
 

.gitignore

@@ -57,3 +57,5 @@ tmp/*
 !private/README.md
 !storage/.gitignore
 !deps/.gitkeep
+!site-static/.gitkeep
+!static-build/.gitkeep

Dockerfile

@@ -176,11 +176,10 @@ COPY --chown=olympia:olympia static/ ${HOME}/static/
 # This will shave nearly 1 minute off the best case build time
 RUN \
     --mount=type=bind,src=src,target=${HOME}/src \
-    --mount=type=bind,src=Makefile-docker,target=${HOME}/Makefile-docker \
+    --mount=type=bind,src=scripts/update_assets.py,target=${HOME}/scripts/update_assets.py \
     --mount=type=bind,src=manage.py,target=${HOME}/manage.py \
 <<EOF
-echo "from olympia.lib.settings_base import *" > settings_local.py
-DJANGO_SETTINGS_MODULE="settings_local" make -f Makefile-docker update_assets
+${HOME}/scripts/update_assets.py
 EOF
 
 FROM base AS production

Makefile-docker

@@ -45,14 +45,8 @@ data_load:
 	./manage.py data_load $(ARGS)
 
 .PHONY: update_assets
-update_assets:
-	# Copy files required in compress_assets to the static folder
-	# If changing this here, make sure to adapt tests in amo/test_commands.py
-	$(PYTHON_COMMAND) manage.py compress_assets
-	$(PYTHON_COMMAND) manage.py generate_jsi18n_files
-	# Collect static files: This MUST be run last or files will be missing
-	$(PYTHON_COMMAND) manage.py collectstatic --noinput
-
+update_assets: ## Update the static assets
+	$(HOME)/scripts/update_assets.py
 
 .PHONY: update_deps
 update_deps: ## Update the dependencies

Makefile-os

@@ -153,21 +153,22 @@ docker_clean_build_cache: ## Remove buildx build cache
 .PHONY: clean_docker
 clean_docker: docker_compose_down docker_mysqld_volume_remove docker_clean_images docker_clean_volumes docker_clean_build_cache ## Remove all docker resources taking space on the host machine
 
-.PHONY: docker_update_deps
-docker_update_deps: docker_mysqld_volume_create ## Update the dependencies in the container based on the docker tag and target
+.PHONY: docker_sync_host
+docker_sync_host: docker_mysqld_volume_create ## Update the dependencies in the container based on the docker tag and target
 	docker compose run \
 		--rm \
 		--no-deps \
 		$(DOCKER_RUN_ARGS) \
 		web \
-		make update_deps
+		make update_deps compile_locales update_assets
 
 .PHONY: up_pre
-up_pre: setup docker_pull_or_build docker_update_deps ## Pre-up the environment, setup files, volumes and host state
+up_pre: setup docker_pull_or_build docker_sync_host ## Pre-up the environment, setup files, volumes and host state
 
 .PHONY: up_start
 up_start: docker_mysqld_volume_create ## Start the docker containers
 	docker compose up $(DOCKER_COMPOSE_ARGS) $(ARGS)
+	docker compose rm olympia_volumes -f
 
 .PHONY: up_post
 up_post: docker_clean_images docker_clean_volumes ## Post-up the environment, setup files, volumes and host state

docker-compose.yml

@@ -45,7 +45,7 @@ services:
     command: ["sleep", "infinity"]
     volumes:
       # used by: web, worker, nginx
-      - ${HOST_MOUNT_SOURCE:?}:/data/olympia
+      - ./:/data/olympia
   worker:
     <<: *olympia
     command: [
@@ -60,7 +60,7 @@ services:
       "celery -A olympia.amo.celery:app worker -E -c 2 --loglevel=INFO",
     ]
     volumes:
-      - ${HOST_MOUNT_SOURCE:?}:/data/olympia
+      - ./:/data/olympia
     extra_hosts:
      - "olympia.test:127.0.0.1"
     restart: on-failure:5
@@ -93,7 +93,7 @@ services:
     image: nginx
     volumes:
       - data_nginx:/etc/nginx/conf.d
-      - ${HOST_MOUNT_SOURCE:?}:/srv
+      - ./:/srv
     ports:
       - "80:80"
     networks:
@@ -192,15 +192,6 @@ networks:
     enable_ipv6: false
 
 volumes:
-  # Volumes for the production olympia mounts
-  # allowing to conditionally mount directories
-  # from the host or from the image to <path>
-  # in the running docker container.
-  # If OLYMPIA_MOUNT_SOURCE matches (data_olympia_)
-  # then we use the production volume mounts. Otherwise
-  # it will map to the current directory ./<name>
-  # (data_olympia_)<name>:/<path>
-  data_olympia_:
   # Volume for rabbitmq/redis to avoid anonymous volumes
   data_rabbitmq:
   data_redis:

docker/entrypoint.sh

@@ -30,12 +30,6 @@ fi
 NEW_HOST_UID=$(get_olympia_uid)
 OLYMPIA_ID_STRING="${NEW_HOST_UID}:$(get_olympia_gid)"
 
-# If we are on production mode, update the ownership of /data/olympia to match the new id
-if [[ "${HOST_MOUNT}" == "production" ]]; then
-  echo "Updating ownership of ${HOME} to ${OLYMPIA_ID_STRING}"
-  chown -R ${OLYMPIA_ID_STRING} ${HOME}
-fi
-
 cat <<EOF | su -s /bin/bash $OLYMPIA_USER
   echo "Running command as ${OLYMPIA_USER} ${OLYMPIA_ID_STRING}"
   set -xue

docker/nginx/addons.conf

@@ -11,8 +11,8 @@ server {
     }
 
     location /static/ {
-        alias /srv/site-static/;
-        try_files $uri @olympia;
+        alias /srv
+        try_files site-static/$uri static/$uri @olympia;
         add_header X-Served-By "nginx" always;
     }
 

scripts/setup.py

@@ -75,27 +75,6 @@ def get_docker_image_meta():
     return tag, target, version, digest
 
 
-def get_olympia_mount(docker_target):
-    """
-    When running on production targets, the user can specify the olympia mount
-    to one of the valid values: development or production. In development, we
-    hard code the values to ensure we have necessary files and permissions.
-    """
-    dev_source = './'
-    prod_source = 'data_olympia_'
-    olympia_mount = docker_target
-    olympia_mount_source = dev_source if docker_target == 'development' else prod_source
-
-    if (
-        docker_target == 'production'
-        and os.environ.get('OLYMPIA_MOUNT') == 'development'
-    ):
-        olympia_mount = 'development'
-        olympia_mount_source = dev_source
-
-    return olympia_mount, olympia_mount_source
-
-
 # Env file should contain values that are referenced in docker-compose*.yml files
 # so running docker compose commands produce consistent results in terminal and make.
 # These values should not be referenced directly in the make file.
@@ -114,7 +93,6 @@ def main():
     docker_tag, docker_target, _, _ = get_docker_image_meta()
 
     olympia_uid = os.getuid()
-    olympia_mount, olympia_mount_source = get_olympia_mount(docker_target)
 
     # These variables are special, as we should allow the user to override them
     # but we should not set a default to the previously set value but instead
@@ -132,10 +110,6 @@ def main():
             # These values are mapped back to the olympia_* values in the environment
             # of the container so everywhere they can be referenced as the user expects.
             'HOST_UID': olympia_uid,
-            'HOST_MOUNT': olympia_mount,
-            # Save the docker compose volume name
-            # to use as the source of the /data/olympia volume
-            'HOST_MOUNT_SOURCE': olympia_mount_source,
             'DEBUG': debug,
             'OLYMPIA_DEPS': olympia_deps,
         }

scripts/update_assets.py

@@ -0,0 +1,59 @@
+#!/usr/bin/env python3
+
+import os
+import shutil
+import subprocess
+
+def clean_dir(dir_path):
+    if not os.path.exists(dir_path):
+        return
+
+    for item in os.listdir(dir_path):
+        item_path = os.path.join(dir_path, item)
+        if os.path.isdir(item_path):
+            shutil.rmtree(item_path)
+
+
+def main():
+    """
+    Update the static assets served by addons-server for production.
+    """
+    DOCKER_TARGET = os.environ.get('DOCKER_TARGET', '')
+    HOME = os.environ.get('HOME')
+
+    if DOCKER_TARGET != 'production':
+        print('Skipping update_assets as we are not in production')
+        return
+
+    for dir in ['static-build', 'site-static']:
+        path = os.path.join(HOME, dir)
+        clean_dir(path)
+        os.makedirs(path, exist_ok=True)
+
+    script_prefix = ['python3', 'manage.py']
+
+    environment = os.environ.copy()
+    environment['DJANGO_SETTINGS_MODULE'] = 'olympia.lib.settings_base'
+
+    subprocess.run(
+        script_prefix + ['compress_assets'],
+        check=True,
+        env=environment,
+        capture_output=False,
+    )
+    subprocess.run(
+        script_prefix + ['generate_jsi18n_files'],
+        check=True,
+        env=environment,
+        capture_output=False,
+    )
+    subprocess.run(
+        script_prefix + ['collectstatic', '--noinput'],
+        check=True,
+        env=environment,
+        capture_output=False,
+    )
+
+
+if __name__ == '__main__':
+    main()

settings.py

@@ -19,7 +19,6 @@
 DEV_MODE = DOCKER_TARGET != 'production'
 
 HOST_UID = os.environ.get('HOST_UID')
-HOST_MOUNT = os.environ.get('HOST_MOUNT')
 
 WSGI_APPLICATION = 'olympia.wsgi.application'
 

tests/make/make.spec.js

@@ -62,18 +62,13 @@ describe('docker-compose.yml', () => {
   describe.each(
     permutations({
       DOCKER_TARGET: ['development', 'production'],
-      OLYMPIA_MOUNT: ['development', 'production'],
       DOCKER_VERSION: ['local', 'latest'],
     }),
   )('\n%s\n', (config) => {
-    const { DOCKER_TARGET, OLYMPIA_MOUNT, DOCKER_VERSION } = config;
-    const isProdTarget = DOCKER_TARGET === 'production';
-    const isProdMount = OLYMPIA_MOUNT === 'production';
-    const isProdMountTarget = isProdMount && isProdTarget;
+    const { DOCKER_TARGET, DOCKER_VERSION } = config;
 
     const inputValues = {
       DOCKER_TARGET,
-      OLYMPIA_MOUNT,
       DOCKER_VERSION,
       DEBUG: 'debug',
       DATA_BACKUP_SKIP: 'skip',
@@ -118,13 +113,12 @@ describe('docker-compose.yml', () => {
         expect(service.volumes).toEqual(
           expect.arrayContaining([
             expect.objectContaining({
-              source: isProdMountTarget ? 'data_olympia_' : expect.any(String),
+              source: expect.any(String),
               target: '/data/olympia',
             }),
           ]),
         );
         const {
-          OLYMPIA_MOUNT,
           DOCKER_VERSION,
           DOCKER_TARGET,
           ...environmentOutput
@@ -163,7 +157,7 @@ describe('docker-compose.yml', () => {
           }),
           // mapping for /data/olympia/ directory to /srv
           expect.objectContaining({
-            source: isProdMountTarget ? 'data_olympia_' : expect.any(String),
+            source: expect.any(String),
             target: '/srv',
           }),
         ]),
@@ -263,7 +257,6 @@ describe('docker-compose.yml', () => {
   const ignoreKeys = [
     // Ignored because these values are explicitly mapped to the host_* values
     'OLYMPIA_UID',
-    'OLYMPIA_MOUNT',
     // Ignored because the HOST_UID is always set to the host user's UID
     'HOST_UID',
   ];