/deps

mozilla · Nov 29, 2024 · 6e70dbb · 6e70dbb
1 parent 9389e83
commit 6e70dbb
Show file tree

Hide file tree

Showing 20 changed files with 538 additions and 148 deletions.
diff --git a/.github/actions/run-docker/action.yml b/.github/actions/run-docker/action.yml
@@ -16,17 +16,21 @@ inputs:
     description: 'The docker-compose file to use'
     required: false
     default: 'docker-compose.yml'
-  logs:
-    description: 'Show logs'
-    required: false
-  data_backup_skip:
+  initialize:
     description: 'Skip data backup'
     required: false
-    default: 'true'
+    default: ''
   install_ci_deps:
     description: 'Install CI dependencies'
     required: false
     default: 'true'
+  mount_olympia:
+    description: 'The volume to mount the olympia directory to (host, container)'
+    required: false
+  docker_target:
+    description: 'The docker target to run'
+    required: false
+    default: 'production'
 
 runs:
   using: 'composite'
@@ -37,25 +41,31 @@ runs:
         echo "id=$(id -u)" >> $GITHUB_OUTPUT
 
     - name: Run Docker Container
+      id: run
+      continue-on-error: true
       shell: bash
       env:
         DOCKER_VERSION: ${{ inputs.version }}
         DOCKER_DIGEST: ${{ inputs.digest }}
         COMPOSE_FILE: ${{ inputs.compose_file }}
         HOST_UID: ${{ steps.id.outputs.id }}
-        DATA_BACKUP_SKIP: ${{ inputs.data_backup_skip }}
+        DATA_BACKUP_SKIP: ${{ inputs.initialize == 'true' && '' || 'true' }}
         INSTALL_CI_DEPS: ${{ inputs.install_ci_deps }}
+        DOCKER_TARGET: ${{ inputs.docker_target }}
+        MOUNT_OLYMPIA: ${{ inputs.mount_olympia }}
       run: |
         # Start the specified services
         make up
 
         # Exec the run command in the container
         # quoted 'EOF' to prevent variable expansion
-        cat <<'EOF' | docker compose exec --user olympia web sh
+        cat <<'EOF' | docker compose exec --user olympia web bash
           ${{ inputs.run }}
         EOF
 
     - name: Logs
       shell: bash
-      if: ${{ inputs.logs }}
-      run: docker compose logs
+      if: ${{ steps.run.outcome == 'failure' }}
+      run: |
+        docker compose logs
+        exit 1
diff --git a/.github/workflows/_test.yml b/.github/workflows/_test.yml
@@ -64,7 +64,6 @@ jobs:
             name: Manage Check
             services: web nginx
             run: make check
-            data_backup_skip: true
     steps:
       - uses: actions/checkout@v4
       - name: Test (${{ matrix.name }})
@@ -74,4 +73,3 @@ jobs:
           digest: ${{ inputs.digest }}
           services: ${{ matrix.services }}
           run: ${{ matrix.run }}
-          data_backup_skip: ${{ matrix.data_backup_skip || 'true' }}
diff --git a/.github/workflows/_test_check.yml b/.github/workflows/_test_check.yml
@@ -28,7 +28,7 @@ on:
         required: false
 
 concurrency:
-  group: test-${{ github.workflow }}-${{ github.event_name}}-${{ github.ref}}-${{ toJson(inputs) }}
+  group: test_check-${{ github.workflow }}-${{ github.event_name}}-${{ github.ref}}-${{ toJson(inputs) }}
   cancel-in-progress: true
 
 jobs:
@@ -43,27 +43,155 @@ jobs:
 
   test_make_up:
     runs-on: ubuntu-latest
+    name: |
+      lcl: ${{ matrix.docker_version == 'local' }},
+      tgt: '${{ matrix.docker_target == 'production' && 'prod'|| matrix.docker_target == 'development' && 'dev' || 'null' }}',
+      mnt: '${{ matrix.mount_olympia == 'production' && 'prod' || 'dev' }}',
+      ci: '${{ matrix.install_ci_deps }}'
     strategy:
       fail-fast: false
       matrix:
-        docker_target:
-          - production
-          - development
         docker_version:
           - local
           - ${{ inputs.version }}
+        docker_target:
+          - development
+          - production
+        mount_olympia:
+          - ''
+          - 'development'
+          - 'production'
         install_ci_deps:
-          - true
           - false
+          - true
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/run-docker
         with:
           version: ${{ matrix.docker_version }}
-          data_backup_skip: false
+          docker_target: ${{ matrix.docker_target }}
+          mount_olympia: ${{ matrix.mount_olympia }}
+          install_ci_deps: ${{ matrix.install_ci_deps }}
+          initialize: true
           run: |
+            is_production="${{ matrix.docker_target == 'production' }}"
+            is_development="${{ matrix.docker_target == 'development' }}"
+            is_local="${{ matrix.docker_version == 'local' }}"
+            # This is the value we pass to make up. This is our desired input.
+            mount_olympia_input="${{ matrix.mount_olympia }}"
+            # This value is set during make up
+            # If the docker target is development, then the value is
+            # forced to be 'development'. This is because in the development
+            # stage, the source files are not copied yet, so unless we mount
+            # the host files, the container will not be able to run.
+            mount_olympia_actual="${DATA_OLYMPIA_MOUNT}"
+
+            # Expect bash shell
+            shell=$(echo $0)
+            if [[ "$shell" != *"bash"* ]]; then
+              echo "Expected bash shell"
+              exit 1
+            fi
+
+            # If docker target is development, and mount_olympia_input is production
+            # then expect the actual value to be forced to development
+            if [[ "$is_development" == "true" && "$mount_olympia_input" != "development" ]]; then
+              if [[ "$mount_olympia_actual" != "development" ]]; then
+                echo "Expected mount olympia to be forced to development"
+                exit 1
+              fi
+            fi
+
+            # If the actual mount olympia value is development,
+            # then the olympia user should be set to the host uid
+            if [[ "$mount_olympia_actual" == "development" ]]; then
+              if [[ "$(id -u olympia)" != "$HOST_UID" ]]; then
+                echo "olympia user should be set to the host uid"
+                exit 1
+              fi
+            fi
+
+            # Expect container to pass checks in the current environment
             make check
 
+            function check_env_var() {
+              local name="$1"
+              local expected="$2"
+              local actual="${!name}"
+              if [ "$actual" != "$expected" ]; then
+                echo "$name: '$actual' is not equal to '$expected'"
+                echo "debug env:"
+                printenv
+                exit 1
+              fi
+            }
+
+            # We expect commit/version to be set during the build for non local images
+            # and to be unset for local image.
+            if [[ "$is_local" == "false" ]]; then
+              check_env_var DOCKER_COMMIT "${{ github.sha }}"
+              check_env_var DOCKER_VERSION "${{ matrix.docker_version }}"
+            fi
+
+            # Expect docker target to be set at runtime
+            check_env_var DOCKER_TARGET "${{ matrix.docker_target }}"
+
+
+            # In production, we expect the site-static directory to exist
+            # and to be sourced from the built image
+            fallback_text='<unknown>'
+            actual_text="$fallback_text"
+            # If the marker file exists, read the contents
+            if [[ -f /data/olympia/site-static/.docker ]]; then
+              actual_text="$(cat /data/olympia/site-static/.docker)"
+            fi
+
+            function check_static_marker() {
+              local expected="$1"
+              local message="$2"
+
+              if [[ "$actual_text" != "$expected" ]]; then
+                echo "Error checking static assets marker: '$message'"
+                echo "actual: '$actual_text'"
+                echo "expected: '$expected'"
+                exit 1
+              fi
+            }
+
+            # If docker target is production and if mount olympia is production
+            # then we expect the site-static directory to be sourced from the built image
+            if [[ "$is_production" == "true" && "$mount_olympia_input" == "production" ]]; then
+              check_static_marker "build" "Assets were overwritten by the host"
+            fi
+
+            # In development, we expect the site-static directory to be empty
+            # regardless of the mount olympia input
+            if [[ "$is_development" == "true" ]]; then
+              check_static_marker "$fallback_text" "Compiled assets should not exist in development"
+            fi
+
+            # If docker target is production and if mount olympia is production
+            # then we should not mount host files and dockerignore should be used
+            make_os_file_exists=false
+            if [[ -f /data/olympia/Makefile-os ]]; then
+              make_os_file_exists=true
+            fi
+
+            if [[ "$is_production" == "true" && "$mount_olympia_input" == "production" ]]; then
+              # Makefile-os is on the .dockerignore file it should not be present
+              if [[ "$make_os_file_exists" == "true" ]]; then
+                echo "Makefile-os should not be present"
+                exit 1
+              fi
+            # Otherwise, we expect Makefile-os to be present
+            else
+              if [[ "$make_os_file_exists" == "false" ]]; then
+                echo "Makefile-os should be present"
+                exit 1
+              fi
+            fi
+
+
   test_make_docker_configuration:
     runs-on: ubuntu-latest
 
@@ -78,6 +206,13 @@ jobs:
         run: |
           docker compose version
           make test_setup
+      - name: Test setup
+        uses: ./.github/actions/run-docker
+        with:
+          digest: ${{ inputs.digest }}
+          version: ${{ inputs.version }}
+          run: |
+            pytest tests/make/
 
   test_run_docker_action:
     runs-on: ubuntu-latest
@@ -113,44 +248,3 @@ jobs:
             echo 'this is a question?'
             echo 'a * is born'
             echo 'wow an array []'
-
-      - name: Verify Build Metadata
-        uses: ./.github/actions/run-docker
-        if: needs.context.outputs.is_fork == 'false'
-        with:
-          digest: ${{ inputs.digest }}
-          version: ${{ inputs.version }}
-          run: |
-            expected_version="${{ inputs.version }}"
-            expected_commit="${{ github.sha }}"
-
-            if [ "$DOCKER_COMMIT" != "$expected_commit" ]; then
-              echo "DOCKER_COMMIT: '$DOCKER_COMMIT' is not equal to '$expected_commit'"
-              exit 1
-            fi
-
-            if [ "$DOCKER_VERSION" != "$expected_version" ]; then
-              echo "DOCKER_VERSION: '$DOCKER_VERSION' is not equal to '$expected_version'"
-              exit 1
-            fi
-
-      - name: Check ignored files
-        uses: ./.github/actions/run-docker
-        with:
-          digest: ${{ inputs.digest }}
-          version: ${{ inputs.version }}
-          run: |
-            # Verify that .dockerignore is working and the
-            # Makefile-os is not in the production container
-            if [ -f Makefile-os ]; then
-              echo "Makefile-os exists"
-              exit 1
-            fi
-
-      - name: Test setup
-        uses: ./.github/actions/run-docker
-        with:
-          digest: ${{ inputs.digest }}
-          version: ${{ inputs.version }}
-          run: |
-            pytest tests/make/
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -182,7 +182,7 @@ jobs:
 
   test_check:
     needs: [context, build]
-    uses: ./.github/workflows/_test_main.yml
+    uses: ./.github/workflows/_test_check.yml
     with:
       version: ${{ needs.build.outputs.version }}
       digest: ${{ needs.build.outputs.digest }}

diff --git a/.gitignore b/.gitignore
@@ -56,3 +56,4 @@ tmp/*
 !docker-compose.development.yml
 !docker-compose.private.yml
 !private/README.md
+!site-static/.docker
diff --git a/.prettierignore b/.prettierignore
@@ -5,6 +5,7 @@
 Dockerfile*
 LICENSE
 Makefile*
+Procfile
 
 # exclude these directories
 .pytest_cache/

diff --git a/Dockerfile b/Dockerfile
@@ -158,6 +158,10 @@ echo "from olympia.lib.settings_base import *" > settings_local.py
 DJANGO_SETTINGS_MODULE="settings_local" make -f Makefile-docker update_assets
 EOF
 
+# Create a marker file to indicate that the assets folder
+# has not been overwritten by the host
+RUN touch /data/olympia/site-static/.docker
+
 FROM base AS production
 
 ARG DOCKER_BUILD DOCKER_COMMIT DOCKER_VERSION
@@ -175,6 +179,3 @@ COPY --from=assets --chown=olympia:olympia ${HOME}/site-static ${HOME}/site-stat
 COPY --from=assets --chown=olympia:olympia ${HOME}/static-build ${HOME}/static-build
 # Copy dependencies from `pip_production`
 COPY --from=pip_production --chown=olympia:olympia /deps /deps
-
-# Set shell back to sh until we can prove we can use bash at runtime
-SHELL ["/bin/sh", "-c"]
diff --git a/Makefile-docker b/Makefile-docker
@@ -8,9 +8,6 @@ export PYTHON_COMMAND=python3
 export PIP_COMMAND=$(PYTHON_COMMAND) -m pip
 APP=src/olympia/
 
-# Get the docker version from the tag e.g repository/name:version@digest -> version
-override DOCKER_VERSION = $(shell echo $(DOCKER_TAG) | sed -n 's/.*:\([0-9]\.[0-9]\.[0-9]\).*/\1/p')
-
 NODE_MODULES := $(NPM_CONFIG_PREFIX)node_modules/
 
 REQUIRED_FILES := \
@@ -31,6 +28,7 @@ check_debian_packages: ## check the existence of multiple debian packages
 
 .PHONY: check_pip_packages
 check_pip_packages: ## check the existence of multiple python packages
+	./scripts/check_pip_packages.sh pip.txt
 	./scripts/check_pip_packages.sh prod.txt
 # We expect dev dependencies only if DOCKER_TARGET is not production
 ifeq ($(DOCKER_TARGET), development)
@@ -53,9 +51,7 @@ check_olympia_user: ## check if the olympia user exists and is current user
 
 .PHONY: check_django
 check_django: ## check if the django app is configured properly
-	echo 'from olympia.lib.settings_base import *' > settings_local.py
-	DJANGO_SETTINGS_MODULE='settings_local' python3 ./manage.py check
-	rm settings_local.py
+	./manage.py check
 
 .PHONY: check_nginx
 check_nginx:  ## check if the nginx config for local development is configured properly
@@ -73,7 +69,7 @@ data_dump:
 
 .PHONY: data_load
 data_load:
-	./manage.py data_load $(ARGS)
+	./manage.py data_load --name $(LOAD)
 
 .PHONY: update_assets
 update_assets:
@@ -100,8 +96,6 @@ endif
 update_deps: ## Update the dependencies
 # If we're running a non local or production image, remove the existing deps and install clean
 ifeq ($(or $(findstring local, $(DOCKER_TAG)), $(findstring development, $(DOCKER_TARGET))),)
-	id
-	ls -lan /deps
 	rm -rf /deps/**
 endif
 # Install all dependencies in one call