Optimise CI runs (#342)

* use docker layer caching locally and on CI for faster (re)builds * make e2e tests only run on master/main for efficiency * swap out e2e test harness - selenium ⬇️ - playwright ⬆️ * reimplement e2e tests in playwright * implement previously-manual tests in playwright --------- Co-authored-by: Cameron Lamb <[email protected]>
uktrade · Mar 27, 2023 · 084c878 · 084c878
1 parent 05a4d93
commit 084c878
Show file tree

Hide file tree

Showing 49 changed files with 1,629 additions and 1,212 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -1,13 +1,11 @@
 version: 2
 jobs:
-  build:
+  build-and-test:
     machine:
       image: ubuntu-2004:202101-01
+      docker_layer_caching: true
     steps:
       - checkout
-      - run:
-          name: Check for fixme comments
-          command: make check-fixme
       - run:
           name: Copy env file
           command: cp .env.ci .env
@@ -34,12 +32,54 @@ jobs:
           command: make flake8
       - run:
           name: run containers
-          command: docker-compose --profile selenium up -d
+          command: docker-compose up -d
+      - run:
+          name: run unit tests
+          command: make test
+      - run:
+          name: Publish unit test coverage
+          command: |
+            wget -O codecov.sh https://codecov.io/bash
+            bash ./codecov.sh -t ${CODECOV_TOKEN} -s test-reports -f "*.xml"
+      - run:
+          name: Check for fixme comments
+          command: make check-fixme
+  e2e-tests:
+    machine:
+      image: ubuntu-2004:202101-01
+      docker_layer_caching: true
+    steps:
+      - checkout
+      - run:
+          name: Copy env file
+          command: cp .env.ci .env
+      - run:
+          name: install npm packages
+          command: npm ci
+      - run:
+          name: build webpack
+          command: npm run build
+      - run:
+          name: build containers
+          command: make build-all
       - run:
-          name: run all tests
-          command: make test-all
+          name: run e2e tests (playwright)
+          command: make test-e2e
       - run:
           name: Publish unit test coverage
           command: |
             wget -O codecov.sh https://codecov.io/bash
             bash ./codecov.sh -t ${CODECOV_TOKEN} -s test-reports -f "*.xml"
+
+workflows:
+  version: 2
+  build_and_test:
+    jobs:
+      - build-and-test
+      - e2e-tests:
+          filters:
+            branches:
+              only:
+                - main
+                - master
+                - /.*e2e.*/ # run on any branch containing "e2e" in the name
diff --git a/.env.ci b/.env.ci
@@ -3,7 +3,7 @@ DJANGO_SETTINGS_MODULE=config.settings.test
 DJANGO_SECRET_KEY=test
 SECRET_KEY=ci-secret
 ALLOWED_HOSTS=*
-AUTHBROKER_URL=test
+AUTHBROKER_URL=https://test.gov.uk
 AUTHBROKER_CLIENT_ID=test
 AUTHBROKER_CLIENT_SECRET=test
 WAGTAIL_BASE_URL=http://localhost:8000
@@ -53,3 +53,5 @@ PERSON_UPDATE_WEBHOOK_URL=
 
 # Home page
 HIDE_NEWS=False
+
+# TESTS_KEEP_DB=1
diff --git a/.gitignore b/.gitignore
@@ -46,6 +46,7 @@ backup.sql
 backup.dump
 backup_s3_asset_only.backup
 backup_file_post_live.backup
+.env.orig
 
 # Add core app static folder
 !core/static/core
@@ -61,3 +62,7 @@ ner_output_file.txt
 # Data tasks
 .db
 *.sql
+
+# Playwright
+/playwright-report
+/playwright/.auth
diff --git a/Dockerfile b/Dockerfile
@@ -1,10 +1,38 @@
-FROM python:3.9-buster
+FROM ubuntu:focal
 
+ARG DEBIAN_FRONTEND=noninteractive
+
+ENV PYTHON_VERSION=3.9
+ENV NODE_VERSION=18
 ENV PYTHONUNBUFFERED=1
 ENV PYTHONDONTWRITEBYTECODE=1
 ENV PYTHONSTARTUP=.pythonrc.py
 ENV POETRY_VIRTUALENVS_CREATE=false
 
+# Install Python & Node
+RUN curl -sL https://deb.nodesource.com/setup_${NODE_VERSION}.x | bash - && \
+    apt-get update && apt-get install -y --no-install-recommends \
+    # Install general packages
+    git nano \
+    curl wget gpg \
+    openssh-client \
+    tzdata \
+    build-essential \
+    libpq-dev \
+    # Install python and supporting packages
+    python${PYTHON_VERSION} \
+    python${PYTHON_VERSION}-dev \
+    python3-pip \
+    # Install Node
+    nodejs \
+    npm && \
+    # Setup python symlinks
+    rm -rf /usr/bin/python3 && \
+    ln -s /usr/bin/python${PYTHON_VERSION} /usr/bin/python3 && \
+    ln -s /usr/bin/python3 /usr/bin/python && \
+    # clean apt cache
+    rm -rf /var/lib/apt/lists/*
+
 WORKDIR /app
 
 COPY pyproject.toml poetry.lock ./

diff --git a/Makefile b/Makefile
@@ -1,4 +1,4 @@
-SHELL := /bin/bash
+SHELL := /bin/sh
 
 clean:
 	npm run clean
@@ -29,10 +29,11 @@ compilescss:
 	$(wagtail) python manage.py compilescss
 
 test:
-	docker-compose run --rm --name testrunner wagtail pytest -m "not selenium" --reuse-db $(tests)
+	docker-compose run --rm --name testrunner wagtail pytest -m "not e2e" --reuse-db $(tests)
 
-test-selenium:
-	docker-compose run --rm --name testrunner wagtail pytest -m "selenium"
+test-e2e: up-all
+	docker-compose exec playwright poetry run pytest -m "e2e"
+	docker-compose stop playwright
 
 test-all:
 	docker-compose run --rm --name testrunner wagtail pytest
@@ -64,11 +65,20 @@ check-fixme:
 up:
 	docker-compose up
 
+up-all:
+	docker-compose --profile playwright up -d
+
 down:
 	docker-compose down
 
+down-all:
+	docker-compose --profile playwright down
+
 build:
-	docker-compose build
+	DOCKER_BUILDKIT=1 COMPOSE_DOCKER_CLI_BUILD=1 BUILDKIT_INLINE_CACHE=1 docker-compose build
+
+build-all:
+	DOCKER_BUILDKIT=1 COMPOSE_DOCKER_CLI_BUILD=1 BUILDKIT_INLINE_CACHE=1 docker-compose --profile playwright build
 
 webpack:
 	npm run dev
@@ -85,6 +95,9 @@ findstatic:
 bash:
 	$(wagtail) bash
 
+psql:
+	PGPASSWORD='postgres' psql -h localhost -U postgres
+
 requirements:
 	$(wagtail) poetry export --without-hashes --output requirements.txt
 
@@ -113,7 +126,7 @@ create_section_homepages:
 	$(wagtail) python manage.py create_section_homepages
 
 first-use:
-	docker-compose down
+	docker-compose --profile playwright down
 	make migrate
 	make data-countries
 	make menus
@@ -133,3 +146,13 @@ data-countries:
 local-setup:
 	poetry install
 	npm install
+
+e2e-codegen:
+	cp .env .env.orig
+	cp .env.ci .env
+	docker-compose stop wagtail
+	docker-compose run --rm -d -p 8000:8000 --env DJANGO_SETTINGS_MODULE=config.settings.test --name wagtail-test-server wagtail
+	sleep 5
+	poetry run playwright codegen http://localhost:8000
+	mv .env.orig .env
+	docker stop wagtail-test-server
diff --git a/README.md b/README.md
@@ -101,32 +101,36 @@ pytest --create-db
 
 The pytest settings can be found in the `pyproject.toml` file.
 
-## Selenium tests
+## End to end tests
 
-You can write selenium tests using [selenium](https://selenium-python.readthedocs.io/)
-and [pytest-selenium](https://pytest-selenium.readthedocs.io/en/latest/).
+End-to-end (e2e) tests only run in CI against the master/main branch, and any branch with
+`e2e` in the branch name.
 
-The tests are ran against the latest version of Chrome using the Remote WebDriver.
+[Playwright](https://playwright.dev/python/) is used as the e2e test runner, using the python variant,
+and executed with `pytest-playwright`.
 
-To run the tests make sure you have started the `selenium` docker-compose service. This
-can be done using the `selenium` docker-compose
-[profile](https://docs.docker.com/compose/profiles/), e.g. `docker-compose --profile selenium up`.
-Then you can use the make command `make test-selenium` to run the tests.
+To run the tests make sure you have started the `playwright` docker-compose service. This
+can be done using the `playwright` docker-compose
+[profile](https://docs.docker.com/compose/profiles/), e.g. `docker-compose --profile playwright up`.
+Then you can use the make command `make test-e2e` to run the tests.
 
-All selenium based tests must live in `selenium_tests/`. This is necessary because in
-order to get multiple selenium tests to work, the default pytest-django database
-handling has to be overridden. See the module docstring for `selenium_tests/conftest.py`
-for complete details of selenium based tests.
+Playwright tests live in the `e2e_tests` folder for convenience.
 
-> Note: It is important that any selenium tests are marked appropriately with
-> `@pytest.mark.selenium`.
+> Note: It is important that any e2e tests are marked appropriately with
+> `@pytest.mark.e2e` so they don't run in unit-test runs.
 
 Please make use of html [data](https://developer.mozilla.org/en-US/docs/Learn/HTML/Howto/Use_data_attributes)
 attributes to make your tests more resistant to changes.
 
-This project uses the `data-test-XXX` naming convention for html data attributes which
+This project uses the `data-testid="XXX"` naming convention for html data attributes which
 are there to support tests.
 
+Playwright also has a [test generator](https://playwright.dev/python/docs/codegen-intro) -
+install the dependencies on your host machine and run `make e2e-codegen` to generate test cases
+as you browse (using the CI settings).
+
+> Note: if you're running e2e tests many times in a session and don't want to destroy and recreate the DB each time (to make the run faster), set the `TESTS_KEEP_DB` environment variable to a truthy value (most easily by modifying .env.ci)
+
 ## Coverage
 
 We use [coverage.py](https://coverage.readthedocs.io) to track our test coverage.

diff --git a/config/settings/base.py b/config/settings/base.py
@@ -434,6 +434,13 @@
             "level": os.getenv("DJANGO_LOG_LEVEL", "INFO"),
             "propagate": True,
         },
+        "django.request": {
+            "handlers": [
+                "stdout",
+            ],
+            "level": os.getenv("DJANGO_LOG_LEVEL", "INFO"),
+            "propagate": False,
+        },
         "django.server": {
             "handlers": [
                 "stdout",

diff --git a/config/settings/test.py b/config/settings/test.py
@@ -1,7 +1,11 @@
+import logging
+
 from .base import *  # noqa
 
 
 APP_ENV = "test"
+DEBUG = True
+TEMPLATE_DEBUG = True
 
 # Required for tests to bypass SSO.
 MIDDLEWARE.remove("authbroker_client.middleware.ProtectAllViewsMiddleware")  # noqa
@@ -13,3 +17,28 @@
     "INDEX": "test_wagtail",
     "AUTO_UPDATE": False,
 }
+
+INSTALLED_APPS += [  # noqa F405
+    "django_extensions",
+]
+
+LOGGING["handlers"] |= {  # noqa F405
+    "file": {
+        "level": "DEBUG",
+        "class": "logging.FileHandler",
+        "filename": "/tmp/wagtail-debug.log",  # noqa S108
+    }
+}
+LOGGING["loggers"] = {  # noqa F405
+    "django.db.backends.schema": {
+        "handlers": ["file"],
+        "propagate": True,
+        "level": "INFO",
+    },
+    # "": {
+    #     "handlers": ["file"],
+    #     "level": "DEBUG",
+    # },
+}
+
+logging.disable(logging.WARN)
diff --git a/content/management/commands/create_menus.py b/content/management/commands/create_menus.py
@@ -23,12 +23,12 @@ def handle(self, *args, **options):
                 site=site,
             )
 
+        # if main_menu.get_base_page_queryset().count() < 5:
         news_home = NewsHome.objects.all()
         transition_home = TransitionHome.objects.all()
         working_at_dit_home = WorkingAtDITHome.objects.all()
         about_us_home = AboutUsHome.objects.all()
         tools_home = ToolsHome.objects.all()
-        # TODO - figure out a way of dealing with Data Hub link
 
         main_menu.add_menu_items_for_pages(news_home)
         main_menu.add_menu_items_for_pages(transition_home)