Add the CI tests and watch it burn

.github/actions/build-docker/action.yml

@@ -10,9 +10,6 @@ inputs:
   version:
     required: true
     description: The image version to tag with
-  target:
-    required: true
-    description: The stage to target in the build
   push:
     required: false
     description: Push the image?
@@ -57,7 +54,8 @@ runs:
     - name: Create .env and version.json files
       shell: bash
       run: |
-        echo "DOCKER_TARGET=${{ inputs.target }}" >> $GITHUB_ENV
+        # We can only build the production image in CI
+        echo "DOCKER_TARGET=production" >> $GITHUB_ENV
         echo "DOCKER_VERSION=${{ steps.meta.outputs.version }}" >> $GITHUB_ENV
         echo "DOCKER_COMMIT=${{ steps.context.outputs.git_sha }}" >> $GITHUB_ENV
         echo "DOCKER_BUILD=${{ steps.context.outputs.git_build_url }}" >> $GITHUB_ENV

.github/actions/run-docker/action.yml

@@ -9,14 +9,18 @@ inputs:
     description: 'The build digest of the image to run. Overrides version.'
     required: true
     default: ''
+  target:
+    description: 'Docker target to run (development|production)'
+    required: false
+    default: 'development'
   run:
     description: 'Run command in container'
     required: true
   logs:
     description: 'Show logs'
     required: false
-  data_backup_skip:
-    description: 'Skip data backup'
+  initialize:
+    description: 'Run the initialize script'
     required: false
     default: 'true'
   mount:
@@ -32,12 +36,15 @@ runs:
   using: 'composite'
   steps:
     - name: Run Docker Container
+      id: run
+      continue-on-error: true
       shell: bash
       run: |
         # Start the specified services
         make up \
           DOCKER_VERSION="${{ inputs.version }}" \
           DOCKER_DIGEST="${{ inputs.digest }}" \
+          DOCKER_TARGET="${{ inputs.target }}" \
           OLYMPIA_UID="$(id -u)" \
           OLYMPIA_MOUNT="${{ inputs.mount }}" \
           OLYMPIA_DEPS="${{ inputs.deps }}" \
@@ -47,11 +54,13 @@ runs:
 
         # Exec the run command in the container
         # quoted 'EOF' to prevent variable expansion
-        cat <<'EOF' | docker compose exec --user olympia web sh
+        cat <<'EOF' | docker compose exec --user olympia web bash
           ${{ inputs.run }}
         EOF
 
     - name: Logs
       shell: bash
-      if: ${{ inputs.logs }}
-      run: docker compose logs
+      if: ${{ steps.run.outcome == 'failure' }}
+      run: |
+        docker compose logs
+        exit 1

.github/workflows/_test_check.yml

@@ -45,6 +45,7 @@ jobs:
     runs-on: ubuntu-latest
     name: |
       version: '${{ matrix.version }}' |
+      target: '${{ matrix.target }}' |
       mount: '${{ matrix.mount }}'
       deps: '${{ matrix.deps }}'
     strategy:
@@ -53,6 +54,9 @@ jobs:
         version:
           - local
           - ${{ inputs.version }}
+        target:
+          - development
+          - production
         mount:
           - development
           - production
@@ -67,6 +71,7 @@ jobs:
           cat <<EOF
             Values passed to the action:
             version: ${{ matrix.version }}
+            target: ${{ matrix.target }}
             mount: ${{ matrix.mount }}
             deps: ${{ matrix.deps }}
           EOF
@@ -78,6 +83,7 @@ jobs:
           DOCKER_VERSION: 'not-expected'
         with:
           version: ${{ matrix.version }}
+          target: ${{ matrix.target }}
           mount: ${{ matrix.mount }}
           deps: ${{ matrix.deps }}
           run: make check

.github/workflows/ci.yml

@@ -72,7 +72,6 @@ jobs:
           registry: ${{ steps.docker_hub.outputs.registry }}
           image: ${{ steps.docker_hub.outputs.image }}
           version: ci-${{ needs.context.outputs.docker_version }}
-          target: development
           push: true
 
   docs_build:
@@ -88,6 +87,7 @@ jobs:
         with:
           digest: ${{ needs.build.outputs.digest }}
           version: ${{ needs.build.outputs.version }}
+          target: development
           mount: development
           deps: development
           run: |
@@ -215,7 +215,6 @@ jobs:
           registry: ${{ steps.docker_hub.outputs.registry }}
           image: ${{ steps.docker_hub.outputs.image }}
           version: ${{ needs.context.outputs.docker_version }}
-          target: production
           push: true
 
   push_gar:
@@ -247,5 +246,4 @@ jobs:
           registry: ${{ steps.docker_gar.outputs.registry }}
           image: ${{ steps.docker_gar.outputs.image }}
           version: ${{ needs.context.outputs.docker_version }}
-          target: production
           push: true

.gitignore

@@ -53,7 +53,6 @@ tmp/*
 # End of .gitignore. Please keep this in sync with the top section of .dockerignore
 
 # do not ignore the following files
-!docker-compose.ci.yml
 !docker-compose.private.yml
 !private/README.md
 !storage/.gitignore

Dockerfile

@@ -122,6 +122,9 @@ EOF
 ARG DOCKER_TARGET
 ENV DOCKER_TARGET=${DOCKER_TARGET}
 
+# Add our custom mime types (required for for ts/json/md files)
+COPY docker/etc/mime.types /etc/mime.types
+
 # Define production dependencies as a single layer
 # let's the rest of the stages inherit prod dependencies
 # and makes copying the /deps dir to the final layer easy.
@@ -141,6 +144,8 @@ RUN \
 ${HOME}/scripts/install_deps.py prod
 EOF
 
+FROM base AS development
+
 FROM base AS locales
 ARG LOCALE_DIR=${HOME}/locale
 # Compile locales
@@ -176,29 +181,20 @@ echo "from olympia.lib.settings_base import *" > settings_local.py
 DJANGO_SETTINGS_MODULE="settings_local" make -f Makefile-docker update_assets
 EOF
 
-FROM base AS sources
+FROM base AS production
 
 
 
-# Add our custom mime types (required for for ts/json/md files)
-COPY docker/etc/mime.types /etc/mime.types
 # Copy the rest of the source files from the host
 COPY --chown=olympia:olympia . ${HOME}
+# Copy compiled locales from builder
+COPY --from=locales --chown=olympia:olympia ${HOME}/locale ${HOME}/locale
 # Copy assets from assets
 COPY --from=assets --chown=olympia:olympia ${HOME}/site-static ${HOME}/site-static
 COPY --from=assets --chown=olympia:olympia ${HOME}/static-build ${HOME}/static-build
+# Copy build info from info
 COPY --from=info ${BUILD_INFO} ${BUILD_INFO}
-
-# Set shell back to sh until we can prove we can use bash at runtime
-SHELL ["/bin/sh", "-c"]
-
-FROM sources AS development
-
-FROM sources AS production
-
 # Copy compiled locales from builder
 COPY --from=locales --chown=olympia:olympia ${HOME}/locale ${HOME}/locale
 # Copy dependencies from `pip_production`
 COPY --from=pip_production --chown=olympia:olympia /deps /deps
-
-

Makefile-docker

@@ -71,7 +71,7 @@ data_dump:
 
 .PHONY: data_load
 data_load:
-	./manage.py data_load $(ARGS)
+	./manage.py data_load --name $(LOAD)
 
 .PHONY: update_assets
 update_assets:

Makefile-os

@@ -18,32 +18,38 @@ export DATA_BACKUP_SKIP ?=
 override DOCKER_MYSQLD_VOLUME = addons-server_data_mysqld
 
 INITIALIZE_ARGS ?=
-INIT_CLEAN ?=
-INIT_LOAD ?=
-
-ifneq ($(INIT_CLEAN),)
-	INITIALIZE_ARGS += --clean
-endif
-
-ifneq ($(INIT_LOAD),)
-	INITIALIZE_ARGS += --load $(INIT_LOAD)
-endif
+export CLEAN ?=
+export LOAD ?=
 
 DOCKER_BAKE_ARGS := \
 	--file docker-bake.hcl \
 	--file .env \
 	--progress $(DOCKER_PROGRESS) \
 	--metadata-file $(DOCKER_METADATA_FILE) \
 
-ifeq ($(DOCKER_PUSH), true)
-	DOCKER_BAKE_ARGS += --push
-endif
-
 DOCKER_COMPOSE_ARGS := \
 	-d \
 	--remove-orphans \
 	--no-build \
 	--quiet-pull \
+	--timestamps
+
+ifneq ($(CLEAN),)
+	# Build without cache
+	DOCKER_COMPOSE_ARGS += --no-cache
+	# Force recreate all containers
+	DOCKER_COMPOSE_ARGS += --force-recreate
+	# Clean the database and all other data when Initializing
+	INITIALIZE_ARGS += --clean
+endif
+
+ifneq ($(LOAD),)
+	INITIALIZE_ARGS += --load $(LOAD)
+endif
+
+ifeq ($(DOCKER_PUSH), true)
+	DOCKER_BAKE_ARGS += --push
+endif
 
 ifneq ($(DOCKER_WAIT),)
 	DOCKER_COMPOSE_ARGS += --wait

docker-compose.yml

@@ -186,10 +186,14 @@ services:
 
   redis:
     image: redis:6.2
+    volumes:
+      - data_redis:/data
 
   rabbitmq:
     image: rabbitmq:3.12
     hostname: olympia
+    volumes:
+      - data_rabbitmq:/var/lib/rabbitmq
     expose:
       - "5672"
     environment:

docs/topics/development/data_management.md

@@ -27,18 +27,18 @@ But there are a few additional edge cases that it supports.
 ### Clean the database
 
   ```sh
-  make initialize INIT_CLEAN=true
+  make initialize CLEAN=true
   ```
 
   This will force the database to be recreated, and re-initialized.
 
 ### Load a data backup
 
   ```sh
-  make initialize [INIT_LOAD=<backup_name>]
+  make initialize [LOAD=<backup_name>]
   ```
 
-  This command will load a data backup from a specified path. The optional `INIT_LOAD` argument allows you to
+  This command will load a data backup from a specified path. The optional `LOAD` argument allows you to
   specify the path to the data backup file. If not specified, the initialize command will determine if
   data should be loaded based on the current state of the databse, and will load the `_init` data backup.
 

scripts/setup.py

@@ -11,27 +11,38 @@ def set_env_file(values):
             print(f'{key}={value}')
 
 
-def get_env_file():
+def get_env_file(path='.env'):
     env = {}
 
-    if os.path.exists('.env'):
-        with open('.env', 'r') as f:
+    if os.path.exists(path):
+        with open(path, 'r') as f:
             for line in f:
                 key, value = line.strip().split('=', 1)
-                env[key] = value.strip('"')
+                if value.startswith('"') and value.endswith('"'):
+                    value = value[1:-1]
+                env[key] = value
     return env
 
 
-def get_value(key, default_value):
-    if key in os.environ:
-        return os.environ[key]
+def get_value(key, default_value, from_file=True):
+    value = default_value
+    # Try to read the value from the .env file
+    # if we are allowed to do so
+    if from_file:
+        env = get_env_file()
+        if key in env:
+            value = env[key]
 
-    from_file = get_env_file()
+    # Always allow environment to override
+    # the default value
+    if key in os.environ:
+        value = os.environ[key]
 
-    if key in from_file:
-        return from_file[key]
+    # Do not allow empty or None values
+    if value is not None and value != '':
+        return value
 
-    return default_value
+    raise ValueError(f'{key} is not set')
 
 
 def get_docker_tag():

src/olympia/core/apps.py

@@ -44,19 +44,26 @@ def host_check(app_configs, **kwargs):
     """Check that the host settings are valid."""
     errors = []
 
-    # In production, we expect settings.HOST_UID to be None and so
-    # set the expected uid to 9500, otherwise we expect the uid
-    # passed to the environment to be the expected uid.
-    expected_uid = 9500 if settings.HOST_UID is None else int(settings.HOST_UID)
-    actual_uid = os.getuid()
-
-    if actual_uid != expected_uid:
-        return [
+    if (
+        settings.HOST_MOUNT is None or settings.HOST_MOUNT == 'production'
+    ) and os.path.exists('/data/olympia/Makefile-os'):
+        errors.append(
             Error(
-                f'Expected user uid to be {expected_uid}, received {actual_uid}',
-                id='setup.E002',
+                'Makefile-os should be excluded by dockerignore',
+                id='setup.E003',
             )
-        ]
+        )
+
+    # If we are on a production image, or the HOST_UID is not defined,
+    # then we expect to retain the original uid of 9500.
+    if settings.HOST_UID is None:
+        if os.getuid() != 9500:
+            return [
+                Error(
+                    'Expected user uid to be 9500',
+                    id='setup.E002',
+                )
+            ]
 
     return errors