Now fix the issues

.github/actions/run-docker/action.yml

@@ -16,10 +16,6 @@ inputs:
   run:
     description: 'Run command in container'
     required: true
-  compose_file:
-    description: 'The docker-compose file to use'
-    required: false
-    default: 'docker-compose.yml:docker-compose.ci.yml'
   logs:
     description: 'Show logs'
     required: false
@@ -45,7 +41,6 @@ runs:
       env:
         DOCKER_VERSION: ${{ inputs.version }}
         DOCKER_DIGEST: ${{ inputs.digest }}
-        COMPOSE_FILE: ${{ inputs.compose_file }}
         HOST_UID: ${{ steps.id.outputs.id }}
         DATA_BACKUP_SKIP: ${{ inputs.data_backup_skip }}
         MOUNT_HOST: ${{ inputs.mount_host }}

.github/workflows/_test.yml

@@ -41,29 +41,24 @@ jobs:
           -
             name: Needs Locale Compilation
             services: ''
-            compose_file: docker-compose.yml:docker-compose.ci.yml
             run: |
               make compile_locales
               make test_needs_locales_compilation
           -
             name: Static Assets
             services: ''
-            compose_file: docker-compose.yml:docker-compose.ci.yml
             run: make test_static_assets
           -
             name: Internal Routes
             services: ''
-            compose_file: docker-compose.yml:docker-compose.ci.yml
             run: make test_internal_routes_allowed
           -
             name: Elastic Search
             services: ''
-            compose_file: docker-compose.yml:docker-compose.ci.yml
             run: make test_es_tests
           -
             name: Codestyle
             services: web
-            compose_file: docker-compose.yml:docker-compose.ci.yml
             run: make lint-codestyle
     steps:
       - uses: actions/checkout@v4
@@ -73,5 +68,4 @@ jobs:
           version: ${{ inputs.version }}
           digest: ${{ inputs.digest }}
           services: ${{ matrix.services }}
-          compose_file: ${{ matrix.compose_file }}
           run: ${{ matrix.run }}

.github/workflows/_test_check.yml

@@ -47,7 +47,6 @@ jobs:
       version: '${{ matrix.version }}' |
       target: '${{ matrix.target }}' |
       mount: '${{ matrix.mount }}' |
-      compose: '${{ matrix.compose_file }}'
     strategy:
       fail-fast: false
       matrix:
@@ -60,9 +59,6 @@ jobs:
         mount:
           - development
           - production
-        compose_file:
-          - 'docker-compose.yml'
-          - 'docker-compose.yml:docker-compose.ci.yml'
     steps:
       - uses: actions/checkout@v4
       - shell: bash
@@ -72,7 +68,6 @@ jobs:
             Values passed to the action:
             version: ${{ matrix.version }}
             target: ${{ matrix.target }}
-            compose_file: ${{ matrix.compose_file }}
             mount: ${{ matrix.mount }}
           EOF
       - uses: ./.github/actions/run-docker
@@ -83,7 +78,6 @@ jobs:
         with:
           version: ${{ matrix.version }}
           target: ${{ matrix.target }}
-          compose_file: ${{ matrix.compose_file }}
           mount: ${{ matrix.mount }}
           run: make check
 

.github/workflows/_test_main.yml

@@ -88,7 +88,7 @@ jobs:
           services: ''
           digest: ${{ inputs.digest }}
           version: ${{ inputs.version }}
-          compose_file: docker-compose.yml
+          mount: development
           run: |
             split="--splits ${{ needs.test_config.outputs.splits }}"
             group="--group ${{ matrix.group }}"

.github/workflows/ci.yml

@@ -88,7 +88,7 @@ jobs:
         with:
           digest: ${{ needs.build.outputs.digest }}
           version: ${{ needs.build.outputs.version }}
-          compose_file: docker-compose.yml
+          mount: development
           run: |
             make docs
 
@@ -139,7 +139,7 @@ jobs:
         with:
           digest: ${{ needs.build.outputs.digest }}
           version: ${{ needs.build.outputs.version }}
-          compose_file: docker-compose.yml
+          mount: development
           run: make extract_locales
 
       - name: Push Locales

.gitignore

@@ -53,6 +53,5 @@ tmp/*
 # End of .gitignore. Please keep this in sync with the top section of .dockerignore
 
 # do not ignore the following files
-!docker-compose.development.yml
 !docker-compose.private.yml
 !private/README.md

docker-compose.yml

@@ -21,10 +21,16 @@ x-env-mapping: &env
     - HISTCONTROL=erasedups
     - CIRCLECI
     - HOST_UID
-    - HOST_MOUNT=${HOST_MOUNT:-${DOCKER_TARGET}}
+    - HOST_MOUNT
     - DEBUG
     - DATA_BACKUP_SKIP
 
+# Define the volume to use for the olympia mount
+# production: mount the files from the container, effectively disabling the mount
+# development: mount the host files, mapping host files into the container
+x-olympia-mount: &olympia-mount
+  data_olympia_${HOST_MOUNT}:/data/olympia
+
 x-olympia: &olympia
   <<: *env
   image: ${DOCKER_TAG:-}
@@ -40,6 +46,17 @@ x-olympia: &olympia
   entrypoint: ["/data/olympia/docker/entrypoint.sh"]
 
 services:
+  olympia:
+    <<: *olympia
+    # We could let this container exit 0, but docker compose --wait
+    # interprets this as a failure, even with a passing healtcheck.
+    # so we just sleep indefinitely instead.
+    command: ["sleep", "infinity"]
+    volumes:
+      - *olympia-mount
+      - data_storage:/data/olympia/storage
+      - data_static_build:/data/olympia/static-build
+      - data_site_static:/data/olympia/site-static
   worker:
     <<: *olympia
     command: [
@@ -53,13 +70,11 @@ services:
       "celery -A olympia.amo.celery:app worker -E -c 2 --loglevel=INFO",
     ]
     volumes:
-      - .:/data/olympia
-      # Don't mount generated files. They only exist in the container
-      # and would otherwiser be deleted by mounting the cwd volume above
-      - /data/olympia/static-build
-      - /data/olympia/site-static
-      - ./package.json:/deps/package.json
-      - ./package-lock.json:/deps/package-lock.json
+      # Mount host files from the environment specific volume
+      - *olympia-mount
+      # Mount shared storage to ensure tasks have access to it
+      - data_storage:/data/olympia/storage
+
     extra_hosts:
      - "olympia.test:127.0.0.1"
     restart: on-failure:5
@@ -76,6 +91,7 @@ services:
       # The start period is 60s
       start_period: 120s
     depends_on:
+      - olympia
       - mysqld
       - elasticsearch
       - redis
@@ -94,6 +110,11 @@ services:
       start_period: 120s
     command:
       - uwsgi --ini /data/olympia/docker/uwsgi.ini
+    volumes:
+      # Additionally, mount static file volumes to prevent
+      # overwriting the original container files from the image
+      - data_site_static:/data/olympia/site-static
+      - data_static_build:/data/olympia/static-build
 
   nginx:
     image: nginx
@@ -184,6 +205,24 @@ networks:
   default:
 
 volumes:
+  # Volumes for static files that should not be
+  # mounted from the host.
+  data_static_build:
+  data_site_static:
+  # volumes for conditionally mounting host files
+  # we use two volumes to prvent corrupting the
+  # production volume when switching between the two
+  data_olympia_production:
+  # The development volume is mounted from the host files
+  data_olympia_development:
+    driver: local
+    driver_opts:
+      type: none
+      o: bind
+      device: ${PWD}
+  # Shared storage for web/worker containers
+  # this overrides the data_olympia_* volumes
+  data_storage:
   data_mysqld:
     # Keep this value in sync with Makefile-os
     # External volumes must be manually created/destroyed

docs/topics/development/building_and_running_services.md

@@ -16,9 +16,9 @@ The Dockerfile for the **addons-server** project uses a multi-stage build to opt
    - **Mounting Local Repository**: The volume `.:/data/olympia` mounts the local Git repository into the container, allowing real-time changes to files within the container.
    - **Mounting Dependencies**: The volume `./deps:/deps` mounts the dependencies directory, enabling better caching across builds and providing visibility for debugging directly on the host.
 
-4. **Environment Variables for OLYMPIA_USER**:
-   - **Development Setup**: The `HOST_UID` environment variable is set to the host user ID, ensuring that the container runs with the correct permissions.
-   - **CI Setup**: In CI environments, such as defined in `docker-compose.ci.yml`, the user ID is reset to the default 9500, and the Olympia mount is removed. This makes the container a closed system, mimicking production behavior closely.
+4. **Environment Variables**:
+   - **Development Setup**: The `HOST_UID` and `HOST_MOUNT` environment variables are set to the host user ID and mount host, ensuring that the container runs with the correct permissions and mounts the correct host directory.
+   - **CI Setup**: In CI environments, the Olympia mount is removed by default. This makes the container a closed system, mimicking production behavior closely.
 
 ### Best Practices for the Dockerfile
 
@@ -145,9 +145,6 @@ This section provides a thorough understanding of the Dockerfile stages, build p
 We use docker compose under the hood to orchestrate container both locally and in CI.
 The `docker-compose.yml` file defines the services, volumes, and networks required for the project.
 
-Our docker compose project is split into a root [docker-compose.yml](../../../docker-compose.yml) file and additional files for specific environments,
-such as [docker-compose.ci.yml](../../../docker-compose.ci.yml) for CI environments.
-
 ### Healthchecks
 
 We define healthchecks for the web and worker services to ensure that the containers are healthy and ready to accept traffic.
@@ -156,14 +153,13 @@ The health checks ensure the django wsgi server and celery worker node are runni
 ### Environment specific compose files
 
 - **Local Development**: The `docker-compose.yml` file is used for local development. It defines services like `web`, `db`, `redis`, and `elasticsearch`.
-- **CI Environment**: The `docker-compose.ci.yml` file is used for CI environments. It overrides the HOST_UID as well as removing volumes to make the container more production like.
 - **Private**: This file includes the customs service that is not open source and should therefore not be included by default.
 - **Override**: This file allows modifying the default configuration without changing the main `docker-compose.yml` file. This file is larglely obsolete and should not be used.
 
 To mount with a specific  set of docker compose files you can add the COMPOSE_FILE argument to make up. This will persist your setting to .env.
 
 ```sh
-make up COMPOSE_FILE=docker-compose.yml:docker-compose.ci.yml
+make up COMPOSE_FILE=docker-compose.yml:docker-compose.override.yml
 ```
 
 Files should be separated with a colon.

docs/topics/development/setup_and_configuration.md

@@ -187,7 +187,6 @@ Though it is **highly recommended to use the make commands** instead of directly
 ### Docker Compose Files
 
 - **[docker-compose.yml][docker-compose]**: The primary Docker Compose file defining services, networks, and volumes for local and CI environments.
-- **[docker-compose.ci.yml][docker-compose-ci]**: Overrides certain configurations for CI-specific needs, ensuring the environment is optimized for automated testing and builds.
 - **[docker-compose.private.yml][docker-compose-private]**: Runs addons-server with the _customs_ service that is only available to Mozilla employees
 
 Our docker compose files rely on substituted values, all of which are included in our .env file for direct CLI compatibility.
@@ -317,7 +316,6 @@ and docker-comose.yml file locally.
 To fix this error `rm -f .env` to remove your .env and `make up` to restart the containers.
 
 [docker-compose]: ../../../docker-compose.yml
-[docker-compose-ci]: ../../../docker-compose.ci.yml
 [docker-compose-private]: ../../../docker-compose.private.yml
 [docker-image-digest]: https://github.com/opencontainers/.github/blob/main/docs/docs/introduction/digests.md
 [addons-server-tags]: https://hub.docker.com/r/mozilla/addons-server/tags

scripts/setup.py

@@ -94,18 +94,6 @@ def main():
 
     is_production = docker_target == 'production'
 
-    # The default value for which compose files to use is based on the target
-    # but can be freely overridden by the user.
-    # E.g running a production image in development mode with source code changes
-    compose_file = get_value(
-        'COMPOSE_FILE',
-        (
-            'docker-compose.yml:docker-compose.ci.yml'
-            if is_production
-            else 'docker-compose.yml'
-        ),
-    )
-
     # DEBUG and HOST_MOUNT are special, as we should allow the user to override it
     # but we should not set a default to the previously set value but instead
     # to the most sensible default.
@@ -114,7 +102,6 @@ def main():
 
     set_env_file(
         {
-            'COMPOSE_FILE': compose_file,
             'DOCKER_TAG': docker_tag,
             'DOCKER_TARGET': docker_target,
             'HOST_UID': get_value('HOST_UID', os.getuid()),