From 5a085012c5f52bc542a11290255799c0a7ab2ec0 Mon Sep 17 00:00:00 2001 From: Thu Trang Pham Date: Tue, 22 Jun 2021 11:17:21 -0700 Subject: [PATCH] Adding localstack to docker and S3 file storage testing (#26) * Adding localstack to docker * Unit tests for localstack S3 working * Github flow * Github flow fix s3 tests * Playing with github actions, tests working on dev via localhost and docker * Try using IP * Try using github services in CI * Try without volumes on services * Try something else..' * Use seperate docker compose yaml for CI * Specify docker compose ym;l file to use for test * Fix -f option * volume mounting * try mount again * try use permissions * Update dir for permissioning * Update create bucket script to output commands * Try to create bucket * Try using awscli not awslocal * Add region * Add connection timeout * Add overwrite * Add debug * More debug * Use conftest to create s3 bucket instead * Adding health check for localstack service * Try netwrok mode bridge for tests * Try some other stuff * Ignore when failing create bucket if exists * Make sure github actions is using the localhost as the ip for selenium * Try setting values from the docker compose for diff envs * Try using network mode host * Remove ports * Use container name instead of docker-compose run * Clean up host variables * Clean up code Co-authored-by: Thu Trang Pham --- .github/workflows/test.yml | 52 ++++- Dockerfile | 2 + admin_confirm/tests/helpers.py | 4 +- .../integration/test_with_form_input_types.py | 1 - .../tests/integration/test_with_s3_storage.py | 200 ++++++++++++++++++ docker-compose.yml => docker-compose.ci.yml | 14 +- docker-compose.dev.yml | 51 +++++ docker-entrypoint-initaws.d/create_bucket.sh | 4 + docs/testing_notes.md | 2 +- requirements.txt | 8 + setup.cfg | 2 +- tests/storage_backends.py | 12 ++ tests/test_project/settings/base.py | 38 +++- tests/test_project/settings/local.py | 2 + tests/test_project/settings/test.py | 2 + 15 files changed, 372 insertions(+), 22 deletions(-) create mode 100644 admin_confirm/tests/integration/test_with_s3_storage.py rename docker-compose.yml => docker-compose.ci.yml (54%) create mode 100644 docker-compose.dev.yml create mode 100644 docker-entrypoint-initaws.d/create_bucket.sh create mode 100644 tests/storage_backends.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index cf7ea1e..582713d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -16,6 +16,34 @@ on: jobs: test: runs-on: ubuntu-latest + services: + selenium: + image: selenium/standalone-firefox:latest + ports: + - "4444:4444" # Selenium + - "5900:5900" # VNC + localstack: + image: localstack/localstack:latest + env: + SERVICES: s3 + DEFAULT_REGION: us-west-1 + AWS_ACCESS_KEY_ID: test + AWS_SECRET_ACCESS_KEY: test + # enable persistance + DATA_DIR: /tmp/localstack/data + LAMBDA_EXECUTOR: local + DOCKER_HOST: unix:///var/run/docker.sock + DEBUG: true + volumes: + # It doesn't seem like the scripts in entrypoint are being ran... or they are not copied over since + # the checkout action happens after init services on Github Actions + # - "${{ github.workspace }}/docker-entrypoint-initaws.d:/docker-entrypoint-initaws.d" + - "${{ github.workspace }}/tmp/localstack:/tmp/localstack" + - "/var/run/docker.sock:/var/run/docker.sock" + ports: + - 4566:4566 + - 4571:4571 + options: --health-cmd="curl http://localhost:4566/health?reload" --health-interval=10s --health-timeout=5s --health-retries=3 strategy: matrix: python-version: [3.6, 3.7, 3.8, 3.9] @@ -24,22 +52,36 @@ jobs: DJANGO_VERSION: ${{ matrix.django-version }} PYTHON_VERSION: ${{ matrix.python-version }} COMPOSE_INTERACTIVE_NO_CLI: 1 + AWS_ACCESS_KEY_ID: test + AWS_SECRET_ACCESS_KEY: test + AWS_DEFAULT_REGION: us-west-1 steps: + - name: Update Permissions + run: | + sudo chown -R $USER:$USER ${{ github.workspace }} + # required because actions/checkout@2 wants to delete the /tmp/localstack folder - uses: actions/checkout@v2 - name: Build Docker for Python 3.6 if: ${{ matrix.python-version == 3.6 }} run: | export SELENIUM_VERSION=3.141.0 - docker-compose build + docker-compose -f docker-compose.ci.yml up -d --build - name: Build Docker for other Python versions if: ${{ matrix.python-version != 3.6 }} run: | export SELENIUM_VERSION=4.0.0a7 - docker-compose build - - name: Start Docker - run: docker-compose up -d + docker-compose -f docker-compose.ci.yml up -d --build + - name: Attempt to connect to localstack and create bucket + run: | + curl -X GET http://localhost:4566/health + aws --endpoint-url http://localhost:4566 s3 mb s3://mybucket 2> /dev/null || true + # Since docker-entrypoint-initaws.d can't be used to create the s3 bucket on CI - name: Integration Test - run: docker-compose run web make test-all + run: | + docker exec -t web_main make test-all + # Known Issue: docker-compose cannot run/exec in container via service name when in host network_mode. + # See: https://github.com/docker/compose/issues/4548 + # IE: this doesn't work: docker-compose -f docker-compose.ci.yml run web make test-all - name: Coveralls uses: AndreMiras/coveralls-python-action@develop with: diff --git a/Dockerfile b/Dockerfile index f06a924..714a130 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,8 +5,10 @@ ENV USE_DOCKER=true WORKDIR /code COPY . /code/ ARG DJANGO_VERSION="3.1.7" +RUN echo "Installing Django Version: ${DJANGO_VERSION}" RUN pip install django==${DJANGO_VERSION} RUN pip install -r requirements.txt RUN pip install -e . ARG SELENIUM_VERSION="4.0.0a7" +RUN echo "Installing Selenium Version: ${SELENIUM_VERSION}" RUN pip install selenium~=${SELENIUM_VERSION} diff --git a/admin_confirm/tests/helpers.py b/admin_confirm/tests/helpers.py index 23f3925..489fa77 100644 --- a/admin_confirm/tests/helpers.py +++ b/admin_confirm/tests/helpers.py @@ -4,6 +4,8 @@ from django.test import TestCase, RequestFactory from django.contrib.auth.models import User from django.test import LiveServerTestCase +from tests.test_project.settings import SELENIUM_HOST + from selenium import webdriver from selenium.webdriver.common.desired_capabilities import DesiredCapabilities from selenium.webdriver.support.ui import Select @@ -76,7 +78,7 @@ class AdminConfirmIntegrationTestCase(LiveServerTestCase): def setUpClass(cls): cls.host = socket.gethostbyname(socket.gethostname()) cls.selenium = webdriver.Remote( - command_executor="http://selenium:4444/wd/hub", + command_executor=f"http://{SELENIUM_HOST}:4444/wd/hub", desired_capabilities=DesiredCapabilities.FIREFOX, ) super().setUpClass() diff --git a/admin_confirm/tests/integration/test_with_form_input_types.py b/admin_confirm/tests/integration/test_with_form_input_types.py index 1157e77..0374a6b 100644 --- a/admin_confirm/tests/integration/test_with_form_input_types.py +++ b/admin_confirm/tests/integration/test_with_form_input_types.py @@ -12,7 +12,6 @@ from django.contrib.admin import VERTICAL from admin_confirm.constants import CONFIRM_ADD, CONFIRM_CHANGE -from selenium.webdriver.support.ui import Select from selenium.webdriver.common.by import By diff --git a/admin_confirm/tests/integration/test_with_s3_storage.py b/admin_confirm/tests/integration/test_with_s3_storage.py new file mode 100644 index 0000000..8c95a09 --- /dev/null +++ b/admin_confirm/tests/integration/test_with_s3_storage.py @@ -0,0 +1,200 @@ +""" +Tests confirmation of add/change +on ModelAdmin that utilize caches +and S3 as a storage backend +""" +import os + +import pytest +import pkg_resources +import localstack_client.session + +from importlib import reload +from selenium.webdriver.remote.file_detector import LocalFileDetector +from selenium.webdriver.common.by import By +from django.core.files.uploadedfile import SimpleUploadedFile +from django.conf import settings + +from tests.market.models import Item + +from admin_confirm.tests.helpers import AdminConfirmIntegrationTestCase +from tests.market.admin import shoppingmall_admin + +from admin_confirm.constants import CONFIRM_CHANGE + + +class ConfirmWithS3StorageTests(AdminConfirmIntegrationTestCase): + def setUp(self): + self.selenium.file_detector = LocalFileDetector() + session = localstack_client.session.Session(region_name="us-west-1") + self.s3 = session.resource("s3") + self.bucket = self.s3.Bucket(settings.AWS_STORAGE_BUCKET_NAME) + # Delete all current files + for obj in self.bucket.objects.all(): + obj.delete() + super().setUp() + + def tearDown(self): + reload(shoppingmall_admin) + # Delete all current files + for obj in self.bucket.objects.all(): + obj.delete() + super().tearDown() + + def test_s3_is_being_used(self): + self.assertTrue(settings.USE_S3) + self.assertIsNotNone(settings.AWS_ACCESS_KEY_ID) + self.assertEqual( + settings.DEFAULT_FILE_STORAGE, + "tests.storage_backends.PublicMediaStorage", + ) + + def test_should_save_file_additions(self): + selenium_version = pkg_resources.get_distribution("selenium").parsed_version + if selenium_version.major < 4: + pytest.skip( + "Known issue `https://github.com/SeleniumHQ/selenium/issues/8762` with this selenium version." + ) + + item = Item.objects.create( + name="item", price=1, currency=Item.VALID_CURRENCIES[0][0] + ) + + self.selenium.get( + self.live_server_url + f"/admin/market/item/{item.id}/change/" + ) + self.assertIn(CONFIRM_CHANGE, self.selenium.page_source) + + # Make a change to trigger confirmation page + price = self.selenium.find_element(By.NAME, "price") + price.send_keys(2) + + # Upload a new file + self.selenium.find_element(By.ID, "id_file").send_keys( + os.getcwd() + "/screenshot.png" + ) + + self.selenium.find_element(By.NAME, "_continue").click() + + # Should have hidden form containing the updated price + self.assertIn("Confirm", self.selenium.page_source) + hidden_form = self.selenium.find_element(By.ID, "hidden-form") + price = hidden_form.find_element(By.NAME, "price") + self.assertEqual("21.00", price.get_attribute("value")) + + self.selenium.find_element(By.NAME, "_confirmation_received") + self.selenium.find_element(By.NAME, "_continue").click() + + item.refresh_from_db() + self.assertRegex(item.file.name, r"screenshot.*\.png$") + self.assertEqual(21, int(item.price)) + + # Check S3 for the file + objects = [obj for obj in self.bucket.objects.all()] + self.assertEqual(len(objects), 1) + self.assertRegex(objects[0].key, r"screenshot.*\.png$") + + def test_should_save_file_changes(self): + selenium_version = pkg_resources.get_distribution("selenium").parsed_version + if selenium_version.major < 4: + pytest.skip( + "Known issue `https://github.com/SeleniumHQ/selenium/issues/8762` with this selenium version." + ) + + file = SimpleUploadedFile( + name="old_file.jpg", + content=open("screenshot.png", "rb").read(), + content_type="image/jpeg", + ) + item = Item.objects.create( + name="item", price=1, currency=Item.VALID_CURRENCIES[0][0], file=file + ) + + self.selenium.get( + self.live_server_url + f"/admin/market/item/{item.id}/change/" + ) + self.assertIn(CONFIRM_CHANGE, self.selenium.page_source) + + # Make a change to trigger confirmation page + price = self.selenium.find_element(By.NAME, "price") + price.send_keys(2) + + # Upload a new file + self.selenium.find_element(By.ID, "id_file").send_keys( + os.getcwd() + "/screenshot.png" + ) + + self.selenium.find_element(By.NAME, "_continue").click() + + # Should have hidden form containing the updated price + self.assertIn("Confirm", self.selenium.page_source) + hidden_form = self.selenium.find_element(By.ID, "hidden-form") + price = hidden_form.find_element(By.NAME, "price") + self.assertEqual("21.00", price.get_attribute("value")) + + self.selenium.find_element(By.NAME, "_confirmation_received") + self.selenium.find_element(By.NAME, "_continue").click() + + item.refresh_from_db() + self.assertEqual(21, int(item.price)) + self.assertRegex(item.file.name, r"screenshot.*\.png$") + + # Check S3 for the file + objects = [obj for obj in self.bucket.objects.all()] + self.assertEqual(len(objects), 2) + get_last_modified = lambda obj: int(obj.last_modified.strftime("%s")) + objects_by_last_modified = [ + obj for obj in sorted(objects, key=get_last_modified) + ] + self.assertRegex(objects_by_last_modified[-1].key, r"screenshot.*\.png$") + self.assertRegex(objects_by_last_modified[0].key, r"old_file.*\.jpg$") + + def test_should_remove_file_if_clear_selected(self): + file = SimpleUploadedFile( + name="old_file.jpg", + content=open("screenshot.png", "rb").read(), + content_type="image/jpeg", + ) + item = Item.objects.create( + name="item", price=1, currency=Item.VALID_CURRENCIES[0][0], file=file + ) + + self.selenium.get( + self.live_server_url + f"/admin/market/item/{item.id}/change/" + ) + self.assertIn(CONFIRM_CHANGE, self.selenium.page_source) + + # Make a change to trigger confirmation page + price = self.selenium.find_element(By.NAME, "price") + price.send_keys(2) + + # Choose to clear the existing file + self.selenium.find_element(By.ID, "file-clear_id").click() + self.assertTrue( + self.selenium.find_element( + By.XPATH, ".//*[@id='file-clear_id']" + ).get_attribute("checked") + ) + + self.selenium.find_element(By.NAME, "_continue").click() + + # Should have hidden form containing the updated price + self.assertIn("Confirm", self.selenium.page_source) + hidden_form = self.selenium.find_element(By.ID, "hidden-form") + price = hidden_form.find_element(By.NAME, "price") + self.assertEqual("21.00", price.get_attribute("value")) + + self.selenium.find_element(By.NAME, "_confirmation_received") + self.selenium.find_element(By.NAME, "_continue").click() + + item.refresh_from_db() + self.assertEqual(21, int(item.price)) + # Should have cleared `file` since clear was selected + self.assertFalse(item.file) + + # Check S3 for the file + # Since deleting from model instance in Django does not automatically + # delete from storage, the old file should still be in S3 + objects = [obj for obj in self.bucket.objects.all()] + self.assertEqual(len(objects), 1) + self.assertRegex(objects[0].key, r"old_file.*\.jpg$") diff --git a/docker-compose.yml b/docker-compose.ci.yml similarity index 54% rename from docker-compose.yml rename to docker-compose.ci.yml index f28f953..f4e07a2 100644 --- a/docker-compose.yml +++ b/docker-compose.ci.yml @@ -12,15 +12,5 @@ services: command: python tests/manage.py runserver 0.0.0.0:8000 volumes: - .:/code - ports: - - "8000:8000" - depends_on: - - selenium - selenium: - # image: selenium/standalone-firefox - image: selenium/standalone-firefox-debug:latest - ports: - - "4444:4444" # Selenium - - "5900:5900" # VNC - volumes: - - .:/code + network_mode: host + container_name: web_main diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml new file mode 100644 index 0000000..3641735 --- /dev/null +++ b/docker-compose.dev.yml @@ -0,0 +1,51 @@ +version: "3.9" + +services: + web: + build: + context: . + dockerfile: Dockerfile + args: + PYTHON_VERSION: "$PYTHON_VERSION" + DJANGO_VERSION: "$DJANGO_VERSION" + SELENIUM_VERSION: "$SELENIUM_VERSION" + command: python tests/manage.py runserver 0.0.0.0:8000 + volumes: + - .:/code + ports: + - "8000:8000" + depends_on: + - selenium + - localstack + environment: + - SELENIUM_HOST=selenium + # Used for localstack_client as well as our project + - LOCALSTACK_HOST=host.docker.internal + + selenium: + image: selenium/standalone-firefox + ports: + - "4444:4444" # Selenium + - "5900:5900" # VNC + volumes: + - .:/code + + localstack: + image: localstack/localstack + container_name: localstack_main + network_mode: bridge + ports: + - "4566:4566" + - "4571:4571" + environment: + - SERVICES=s3 + - DEBUG=true + # enable persistance + - DATA_DIR=/tmp/localstack/data + - LAMBDA_EXECUTOR=docker + - DOCKER_HOST=unix:///var/run/docker.sock + - HOSTNAME_EXTERNAL=localstack + volumes: + - "./docker-entrypoint-initaws.d:/docker-entrypoint-initaws.d" + - "./tmp/localstack:/tmp/localstack" + - "/var/run/docker.sock:/var/run/docker.sock" diff --git a/docker-entrypoint-initaws.d/create_bucket.sh b/docker-entrypoint-initaws.d/create_bucket.sh new file mode 100644 index 0000000..47fd507 --- /dev/null +++ b/docker-entrypoint-initaws.d/create_bucket.sh @@ -0,0 +1,4 @@ +#!/bin/bash +set -x +awslocal s3 mb s3://mybucket +set +x diff --git a/docs/testing_notes.md b/docs/testing_notes.md index f1a0b6b..884b1d1 100644 --- a/docs/testing_notes.md +++ b/docs/testing_notes.md @@ -17,7 +17,7 @@ These are some areas which might/probably have issues that are not currently tes ## Save Options - [x] Save -- [x] Conitnue +- [x] Continue - [x] Save As New - [x] Add another diff --git a/requirements.txt b/requirements.txt index 9cc4d8f..12069d0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,9 +8,17 @@ twine~=3.3.0 coveralls~=3.0.0 Pillow~=8.1.0 # For ImageField +### SELENIUM ### # Known issue: https://github.com/SeleniumHQ/selenium/issues/8762 # Python 3.6 should use because selenium 4 doesn't work with py3.6 # selenium~=3.141.0 # Others should use selenium~=4.0.0.a5 +### END SELENIUM ### + +### S3 ### +localstack~=0.12.9.1 # For testing with S3 +django-storages~=1.11.1 +boto3~=1.17.47 +### END S3 ### diff --git a/setup.cfg b/setup.cfg index e07435d..1664c2a 100644 --- a/setup.cfg +++ b/setup.cfg @@ -25,4 +25,4 @@ branch = True [tool:pytest] DJANGO_SETTINGS_MODULE=tests.test_project.settings.test addopts = --doctest-modules -ra -l --tb=short --show-capture=all --color=yes -testpaths = admin_confirm +testpaths = admin_confirm/tests diff --git a/tests/storage_backends.py b/tests/storage_backends.py new file mode 100644 index 0000000..ce92d2e --- /dev/null +++ b/tests/storage_backends.py @@ -0,0 +1,12 @@ +from storages.backends.s3boto3 import S3Boto3Storage + + +class StaticStorage(S3Boto3Storage): + location = "static" + default_acl = "public-read" + + +class PublicMediaStorage(S3Boto3Storage): + location = "media" + default_acl = "public-read" + file_overwrite = False diff --git a/tests/test_project/settings/base.py b/tests/test_project/settings/base.py index c046859..092a27f 100644 --- a/tests/test_project/settings/base.py +++ b/tests/test_project/settings/base.py @@ -43,6 +43,7 @@ "django.contrib.sessions", "django.contrib.messages", "django.contrib.staticfiles", + "storages", ] MIDDLEWARE = [ @@ -114,8 +115,43 @@ USE_TZ = True +# Setting the hostnames of the services which we are running +# On github actions, services can be configured on the jobs themselves +# and can be accessed at localhost. See: https://docs.github.com/en/actions/guides/about-service-containers#communicating-with-service-containers + +LOCALSTACK_HOST = os.getenv("LOCALSTACK_HOST", "localhost") +SELENIUM_HOST = os.getenv("SELENIUM_HOST", "localhost") # Static files (CSS, JavaScript, Images) # https://docs.djangoproject.com/en/3.0/howto/static-files/ -STATIC_URL = "/static/" +# STATIC_URL = "/static/" + +# S3 Storage settings + +USE_S3 = os.getenv("USE_S3", "true").lower() == "true" + +if USE_S3: + # aws settings + AWS_S3_ENDPOINT_URL = f"http://{LOCALSTACK_HOST}:4566" + AWS_ACCESS_KEY_ID = os.getenv("AWS_ACCESS_KEY_ID", "test") + AWS_SECRET_ACCESS_KEY = os.getenv("AWS_SECRET_ACCESS_KEY", "test") + AWS_STORAGE_BUCKET_NAME = os.getenv("AWS_STORAGE_BUCKET_NAME", "mybucket") + AWS_DEFAULT_ACL = None + AWS_S3_CUSTOM_DOMAIN = f"{AWS_STORAGE_BUCKET_NAME}.s3.amazonaws.com" + AWS_S3_OBJECT_PARAMETERS = {"CacheControl": "max-age=86400"} + # s3 static settings + STATIC_LOCATION = "static" + STATIC_URL = f"https://{AWS_S3_CUSTOM_DOMAIN}/{STATIC_LOCATION}/" + STATICFILES_STORAGE = "tests.storage_backends.StaticStorage" + # s3 public media settings + PUBLIC_MEDIA_LOCATION = "media" + MEDIA_URL = f"https://{AWS_S3_CUSTOM_DOMAIN}/{PUBLIC_MEDIA_LOCATION}/" + DEFAULT_FILE_STORAGE = "tests.storage_backends.PublicMediaStorage" +else: + STATIC_URL = "/staticfiles/" + STATIC_ROOT = os.path.join(BASE_DIR, "staticfiles") + MEDIA_URL = "/mediafiles/" + MEDIA_ROOT = os.path.join(BASE_DIR, "mediafiles") + +STATICFILES_DIRS = (os.path.join(BASE_DIR, "static"),) diff --git a/tests/test_project/settings/local.py b/tests/test_project/settings/local.py index 178e709..865e254 100644 --- a/tests/test_project/settings/local.py +++ b/tests/test_project/settings/local.py @@ -3,3 +3,5 @@ INSTALLED_APPS = INSTALLED_APPS + ["market"] WSGI_APPLICATION = "test_project.wsgi.application" ROOT_URLCONF = "test_project.urls" + +USE_S3 = "True" diff --git a/tests/test_project/settings/test.py b/tests/test_project/settings/test.py index 8518949..0bfa80b 100644 --- a/tests/test_project/settings/test.py +++ b/tests/test_project/settings/test.py @@ -3,3 +3,5 @@ INSTALLED_APPS = INSTALLED_APPS + ["tests.market"] WSGI_APPLICATION = "tests.test_project.wsgi.application" ROOT_URLCONF = "tests.test_project.urls" + +USE_S3 = "True"