Remove the whole concept of development/production mounts

.github/actions/run-docker/action.yml

@@ -23,10 +23,6 @@ inputs:
     description: 'Docker target to run (development|production)'
     required: false
     default: 'production'
-  mount:
-    description: 'Mount host files at runtime? (development|production)'
-    required: false
-    default: 'production'
   deps:
     description: 'Which dependencies to install at runtime? (development|production)'
     required: false
@@ -44,7 +40,6 @@ runs:
           DOCKER_DIGEST="${{ inputs.digest }}" \
           DOCKER_TARGET="${{ inputs.target }}" \
           OLYMPIA_UID="$(id -u)" \
-          OLYMPIA_MOUNT="${{ inputs.mount }}" \
           OLYMPIA_DEPS="${{ inputs.deps }}" \
           DATA_BACKUP_SKIP="${{ inputs.data_backup_skip }}" \
           DOCKER_WAIT="true"

.github/workflows/_test_check.yml

@@ -46,7 +46,6 @@ jobs:
     name: |
       version: '${{ matrix.version }}' |
       target: '${{ matrix.target }}' |
-      mount: '${{ matrix.mount }}' |
       deps: '${{ matrix.deps }}'
     strategy:
       fail-fast: false
@@ -57,9 +56,6 @@ jobs:
         target:
           - development
           - production
-        mount:
-          - development
-          - production
         deps:
           - development
           - production
@@ -72,7 +68,6 @@ jobs:
             Values passed to the action:
             version: ${{ matrix.version }}
             target: ${{ matrix.target }}
-            mount: ${{ matrix.mount }}
             deps: ${{ matrix.deps }}
           EOF
       - name: ${{ matrix.version == 'local' && 'Uncached Build' || 'Pull' }} Check
@@ -84,7 +79,6 @@ jobs:
         with:
           version: ${{ matrix.version }}
           target: ${{ matrix.target }}
-          mount: ${{ matrix.mount }}
           deps: ${{ matrix.deps }}
           run: make check
       - name: Cached Build Check
@@ -93,7 +87,6 @@ jobs:
         with:
           version: ${{ matrix.version }}
           target: ${{ matrix.target }}
-          mount: ${{ matrix.mount }}
           deps: ${{ matrix.deps }}
           run: echo true
 

.github/workflows/_test_main.yml

@@ -88,7 +88,6 @@ jobs:
           services: ''
           digest: ${{ inputs.digest }}
           version: ${{ inputs.version }}
-          mount: development
           deps: development
           run: |
             split="--splits ${{ needs.test_config.outputs.splits }}"

.github/workflows/ci.yml

@@ -87,7 +87,6 @@ jobs:
         with:
           digest: ${{ needs.build.outputs.digest }}
           version: ${{ needs.build.outputs.version }}
-          mount: development
           deps: development
           target: development
           run: |
@@ -140,7 +139,6 @@ jobs:
         with:
           digest: ${{ needs.build.outputs.digest }}
           version: ${{ needs.build.outputs.version }}
-          mount: development
           deps: development
           run: make extract_locales
 

.gitignore

@@ -57,3 +57,5 @@ tmp/*
 !private/README.md
 !storage/.gitignore
 !deps/.gitkeep
+!site-static/.gitkeep
+!static-build/.gitkeep

Dockerfile

@@ -148,6 +148,9 @@ EOF
 
 FROM base AS development
 
+# Copy build info from info
+COPY --from=info ${BUILD_INFO} ${BUILD_INFO}
+
 FROM base AS locales
 ARG LOCALE_DIR=${HOME}/locale
 # Compile locales
@@ -157,7 +160,7 @@ COPY --chown=olympia:olympia locale ${LOCALE_DIR}
 RUN \
     --mount=type=bind,source=requirements/locale.txt,target=${HOME}/requirements/locale.txt \
     --mount=type=bind,source=Makefile-docker,target=${HOME}/Makefile-docker \
-    --mount=type=bind,source=locale/compile-mo.sh,target=${HOME}/compile-mo.sh \
+    --mount=type=bind,source=scripts/compile_locales.py,target=${HOME}/scripts/compile_locales.py \
     make -f Makefile-docker compile_locales
 
 # More efficient caching by mounting the exact files we need
@@ -176,11 +179,10 @@ COPY --chown=olympia:olympia static/ ${HOME}/static/
 # This will shave nearly 1 minute off the best case build time
 RUN \
     --mount=type=bind,src=src,target=${HOME}/src \
-    --mount=type=bind,src=Makefile-docker,target=${HOME}/Makefile-docker \
+    --mount=type=bind,src=scripts/update_assets.py,target=${HOME}/scripts/update_assets.py \
     --mount=type=bind,src=manage.py,target=${HOME}/manage.py \
 <<EOF
-echo "from olympia.lib.settings_base import *" > settings_local.py
-DJANGO_SETTINGS_MODULE="settings_local" make -f Makefile-docker update_assets
+${HOME}/scripts/update_assets.py
 EOF
 
 FROM base AS production

Makefile-docker

@@ -45,14 +45,8 @@ data_load:
 	./manage.py data_load $(ARGS)
 
 .PHONY: update_assets
-update_assets:
-	# Copy files required in compress_assets to the static folder
-	# If changing this here, make sure to adapt tests in amo/test_commands.py
-	$(PYTHON_COMMAND) manage.py compress_assets
-	$(PYTHON_COMMAND) manage.py generate_jsi18n_files
-	# Collect static files: This MUST be run last or files will be missing
-	$(PYTHON_COMMAND) manage.py collectstatic --noinput
-
+update_assets: ## Update the static assets
+	$(HOME)/scripts/update_assets.py
 
 .PHONY: update_deps
 update_deps: ## Update the dependencies
@@ -191,7 +185,7 @@ extract_locales: ## extracts and merges translation strings
 .PHONE: compile_locales
 compile_locales: ## compiles translation strings
 	$(PIP_COMMAND) install --progress-bar=off --no-deps -r requirements/locale.txt
-	./locale/compile-mo.sh ./locale/
+	$(HOME)/scripts/compile_locales.py
 
 .PHONY: help_submake
 help_submake:

Makefile-os

@@ -153,21 +153,22 @@ docker_clean_build_cache: ## Remove buildx build cache
 .PHONY: clean_docker
 clean_docker: docker_compose_down docker_mysqld_volume_remove docker_clean_images docker_clean_volumes docker_clean_build_cache ## Remove all docker resources taking space on the host machine
 
-.PHONY: docker_update_deps
-docker_update_deps: docker_mysqld_volume_create ## Update the dependencies in the container based on the docker tag and target
+.PHONY: docker_sync_host
+docker_sync_host: docker_mysqld_volume_create ## Update the dependencies in the container based on the docker tag and target
 	docker compose run \
 		--rm \
 		--no-deps \
 		$(DOCKER_RUN_ARGS) \
 		web \
-		make update_deps
+		./scripts/sync_host_files.py
 
 .PHONY: up_pre
-up_pre: setup docker_pull_or_build docker_update_deps ## Pre-up the environment, setup files, volumes and host state
+up_pre: setup docker_pull_or_build docker_sync_host ## Pre-up the environment, setup files, volumes and host state
 
 .PHONY: up_start
 up_start: docker_mysqld_volume_create ## Start the docker containers
 	docker compose up $(DOCKER_COMPOSE_ARGS) $(ARGS)
+	docker compose rm olympia_volumes -f
 
 .PHONY: up_post
 up_post: docker_clean_images docker_clean_volumes ## Post-up the environment, setup files, volumes and host state

docker-compose.yml

@@ -45,7 +45,7 @@ services:
     command: ["sleep", "infinity"]
     volumes:
       # used by: web, worker, nginx
-      - ${HOST_MOUNT_SOURCE:?}:/data/olympia
+      - ./:/data/olympia
   worker:
     <<: *olympia
     command: [
@@ -60,7 +60,7 @@ services:
       "celery -A olympia.amo.celery:app worker -E -c 2 --loglevel=INFO",
     ]
     volumes:
-      - ${HOST_MOUNT_SOURCE:?}:/data/olympia
+      - ./:/data/olympia
     extra_hosts:
      - "olympia.test:127.0.0.1"
     restart: on-failure:5
@@ -93,7 +93,7 @@ services:
     image: nginx
     volumes:
       - data_nginx:/etc/nginx/conf.d
-      - ${HOST_MOUNT_SOURCE:?}:/srv
+      - ./:/srv
     ports:
       - "80:80"
     networks:
@@ -192,15 +192,6 @@ networks:
     enable_ipv6: false
 
 volumes:
-  # Volumes for the production olympia mounts
-  # allowing to conditionally mount directories
-  # from the host or from the image to <path>
-  # in the running docker container.
-  # If OLYMPIA_MOUNT_SOURCE matches (data_olympia_)
-  # then we use the production volume mounts. Otherwise
-  # it will map to the current directory ./<name>
-  # (data_olympia_)<name>:/<path>
-  data_olympia_:
   # Volume for rabbitmq/redis to avoid anonymous volumes
   data_rabbitmq:
   data_redis:

docker/entrypoint.sh

@@ -30,12 +30,6 @@ fi
 NEW_HOST_UID=$(get_olympia_uid)
 OLYMPIA_ID_STRING="${NEW_HOST_UID}:$(get_olympia_gid)"
 
-# If we are on production mode, update the ownership of /data/olympia to match the new id
-if [[ "${HOST_MOUNT}" == "production" ]]; then
-  echo "Updating ownership of ${HOME} to ${OLYMPIA_ID_STRING}"
-  chown -R ${OLYMPIA_ID_STRING} ${HOME}
-fi
-
 cat <<EOF | su -s /bin/bash $OLYMPIA_USER
   echo "Running command as ${OLYMPIA_USER} ${OLYMPIA_ID_STRING}"
   set -xue

docker/nginx/addons.conf

@@ -10,10 +10,15 @@ server {
         alias /srv/storage/;
     }
 
-    location /static/ {
-        alias /srv/site-static/;
-        try_files $uri @olympia;
+    location ~ ^/static/(.*)$ {
         add_header X-Served-By "nginx" always;
+
+        # "root /srv" as we mount all files in this directory
+        root /srv;
+
+        # If it doesn't exist under /srv/site-static/$1,
+        # fall back to /srv/static/$1, else forward to @olympia
+        try_files /site-static/$1 /static/$1 @olympia;
     }
 
     location /user-media/ {

scripts/compile_locales.py

@@ -0,0 +1,71 @@
+#!/usr/bin/env python3
+
+import os
+import subprocess
+from concurrent.futures import ThreadPoolExecutor
+
+
+def process_po_file(pofile, attempt=0):
+    """Process a single .po file, creating corresponding .mo file."""
+    print('processing', pofile)
+    directory = os.path.dirname(pofile)
+    stem = os.path.splitext(os.path.basename(pofile))[0]
+    mo_path = os.path.join(directory, f'{stem}.mo')
+
+    # Touch the .mo file
+    open(mo_path, 'a').close()
+
+    try:
+        # Run dennis-cmd lint
+        subprocess.run(
+            ['dennis-cmd', 'lint', '--errorsonly', pofile],
+            capture_output=True,
+            check=False,
+        )
+        # If lint passes, run msgfmt
+        subprocess.run(['msgfmt', '-o', mo_path, pofile], check=True)
+        return
+    except subprocess.CalledProcessError as e:
+        if attempt < 3:
+            print(f'Failed attempt {attempt} for {pofile}, retrying...')
+            return process_po_file(pofile, attempt=attempt + 1)
+        raise e
+
+
+def main():
+    # Ensure 'dennis' is installed
+    try:
+        import dennis as _
+    except ImportError:
+        print(
+            'Error: dennis is not installed. Please install it with pip install dennis'
+        )
+        exit(1)
+
+    locale_dir = os.path.abspath(
+        os.path.join(
+            os.path.dirname(__file__),
+            '..',
+            'locale',
+        )
+    )
+
+    print(f'Compiling locales in {locale_dir}')
+
+    # Collect all files first
+    django_files = []
+    djangojs_files = []
+    for root, _, files in os.walk(locale_dir):
+        for file in files:
+            if file == 'django.po':
+                django_files.append(os.path.join(root, file))
+            elif file == 'djangojs.po':
+                djangojs_files.append(os.path.join(root, file))
+
+    # Process django.po files in parallel
+    with ThreadPoolExecutor() as executor:
+        executor.map(process_po_file, django_files + djangojs_files)
+
+
+if __name__ == '__main__':
+    main()