From b2444ea0d943527206991e93e6d15fccc0188806 Mon Sep 17 00:00:00 2001 From: Andy Driver Date: Fri, 1 Feb 2019 11:53:57 +0000 Subject: [PATCH] Tidy up, fix warnings and use a Makefile (#508) * Tidy up, fix warnings and use a Makefile * Update README.md * Remove make dependency in docker image * Remove unused USE_VENV variable * Can't run specific tests with docker-compose * Fix pagination with no items --- Dockerfile | 71 +++++++++++------------------- Makefile | 64 +++++++++++++++++++++++++++ README.md | 41 +++++++++-------- control_panel_api/k8s_patch.py | 15 +++---- control_panel_api/pagination.py | 2 +- control_panel_api/serializers.py | 2 +- control_panel_api/settings/base.py | 6 +-- control_panel_api/settings/test.py | 4 ++ control_panel_api/views.py | 4 +- docker-compose.test.yml | 4 +- docker-compose.yml | 6 +-- requirements.txt | 2 +- run_api | 6 --- run_tests | 7 --- 14 files changed, 134 insertions(+), 100 deletions(-) create mode 100644 Makefile delete mode 100755 run_api delete mode 100755 run_tests diff --git a/Dockerfile b/Dockerfile index d60677034..7fb3b9cb2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,63 +2,42 @@ FROM alpine:3.7 MAINTAINER Andy Driver -# install build dependencies (they'll be uninstalled after pip install) -RUN apk add --no-cache \ - --virtual build-deps \ - gcc \ - musl-dev +ENV HELM_VERSION 2.9.1 +ENV HELM_HOME /tmp/helm +ENV DJANGO_SETTINGS_MODULE "control_panel_api.settings" -# install python3 and 'ca-certificates' so that HTTPS works consistently +WORKDIR /home/control-panel + +# install build dependencies (they'll be uninstalled after pip install) RUN apk add --no-cache \ + build-base \ openssl \ ca-certificates \ libffi-dev \ - python3-dev - -# Temporary bugfix for libressl -# Postgres needs libressl-dev, but cryptography only works with openssl-dev -RUN apk add --no-cache --virtual temp-ssl-fix \ - openssl-dev \ - && pip3 install cryptography==2.2.2 \ - && apk del temp-ssl-fix \ - && apk add --no-cache \ + python3-dev \ libressl-dev \ postgresql-dev -# Install helm -ENV HELM_VERSION 2.9.1 -RUN wget https://storage.googleapis.com/kubernetes-helm/helm-v$HELM_VERSION-linux-amd64.tar.gz \ - && tar xzf helm-v$HELM_VERSION-linux-amd64.tar.gz \ - && mv linux-amd64/helm /usr/local/bin \ - && rm -rf helm-v$HELM_VERSION-linux-amd64.tar.gz linux-amd64 - -# Configure helm -ENV HELM_HOME /tmp/helm -RUN helm init --client-only +# Install and configure helm COPY helm-repositories.yaml /tmp/helm/repository/repositories.yaml -RUN helm repo update - -WORKDIR /home/control-panel - -# install python dependencies -ADD requirements.txt requirements.txt -RUN pip3 install -r requirements.txt - -# uninstall build dependencies -RUN apk del build-deps - -ENV DJANGO_SETTINGS_MODULE "control_panel_api.settings" - -ADD manage.py manage.py -ADD run_api run_api -ADD run_tests run_tests -ADD wait_for_db wait_for_db -ADD control_panel_api control_panel_api -ADD moj_analytics moj_analytics +RUN wget https://storage.googleapis.com/kubernetes-helm/helm-v${HELM_VERSION}-linux-amd64.tar.gz -O helm.tgz \ + && tar fxz helm.tgz \ + && mv linux-amd64/helm /usr/local/bin \ + && rm -rf helm.tgz linux-amd64 \ + && helm init --client-only \ + && helm repo update + +# install python dependencies (and then remove build dependencies) +COPY requirements.txt ./ +RUN pip3 install -r requirements.txt \ + && apk del build-base + +COPY control_panel_api control_panel_api +COPY moj_analytics moj_analytics +COPY manage.py wait_for_db ./ # collect static files for deployment RUN python3 manage.py collectstatic EXPOSE 8000 - -CMD ["./run_api"] +CMD ["gunicorn", "-b", "0.0.0.0:8000", "control_panel_api.wsgi:application"] diff --git a/Makefile b/Makefile new file mode 100644 index 000000000..79b62b592 --- /dev/null +++ b/Makefile @@ -0,0 +1,64 @@ +HOST=0.0.0.0 +PORT=8000 +PROJECT=control-panel +MODULE=control_panel_api +VENV=venv +BIN=${VENV}/bin + +-include .env +export + +.PHONY: collectstatic dependencies help run test wait_for_db + +venv/bin: + @if ${USE_VENV} && [ ! -d "${VENV}" ] ; then python3 -m venv ${VENV} ; fi + +## dependencies: Install dependencies +dependencies: ${BIN} requirements.txt + @echo + @echo "> Fetching dependencies..." + @${BIN}/pip3 install -r requirements.txt + +## collectstatic: Collect assets into static folder +collectstatic: dependencies + @echo + @echo "> Collecting static assets..." + @${BIN}/python3 manage.py collectstatic --noinput + +## run: Run webapp +run: collectstatic + @echo + @echo "> Running webapp..." + @${BIN}/gunicorn -b ${HOST}:${PORT} ${MODULE}.wsgi:application + +wait_for_db: + @echo + @echo "> Waiting for database..." + @${BIN}/python3 wait_for_db + +## test: Run tests +test: export DJANGO_SETTINGS_MODULE=${MODULE}.settings.test +test: wait_for_db + @echo + @echo "> Running tests..." + @NAMED_TESTS="$(shell if [ -n "${TEST_NAME}" ]; then echo "-k ${TEST_NAME}" ; fi)" && \ + ${BIN}/pytest --color=yes ${MODULE} $$NAMED_TESTS + +## docker-image: Build docker image +docker-image: + @echo + @echo "> Building docker image..." + @docker build -t ${PROJECT} . + +## docker-test: Run tests in Docker container +docker-test: + @echo + @echo "> Running tests in Docker..." + @docker-compose -f docker-compose.test.yml up --abort-on-container-exit + +help: Makefile + @echo + @echo " Commands in "$(PROJECT)":" + @echo + @sed -n 's/^##//p' $< | column -t -s ':' | sed -e 's/^/ /' + @echo diff --git a/README.md b/README.md index f911602bd..7b4220054 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ https://github.com/ministryofjustice/analytics-platform-control-panel-frontend ## Running with Docker ```sh -docker-compose build +docker-compose build # OR make docker-image docker-compose up ``` and then in a separate terminal window, @@ -21,12 +21,7 @@ Then browse to http://localhost:8000/ ### Running tests with docker ```sh -docker-compose -f docker-compose.test.yml up --build --abort-on-container-exit -``` - -You can run a particular test using the pytest '-k' parameter: -```sh -docker-compose -f docker-compose.test.yml build && docker-compose -f docker-compose.test.yml run app ./run_tests -k TEST-NAME +make docker-test ``` ## Running directly on your machine @@ -35,11 +30,11 @@ docker-compose -f docker-compose.test.yml build && docker-compose -f docker-comp The Control Panel app requires Python 3.6+ -It is best to use a virtual environment to install python dependencies, eg: +Install dependencies with the following command: ```sh -python -m venv venv -. venv/bin/activate -pip install -r requirements.txt +python3 -m venv venv +source venv/bin/activate +pip3 install -r requirements.txt ``` ### Kubernetes setup @@ -90,18 +85,19 @@ export DJANGO_SETTINGS_MODULE=control_panel_api.settings The Control Panel app connects to a PostgreSQL database, which should have a database with the expected name: ```sh -createdb $DB_NAME +createuser -d controlpanel +createdb -U controlpanel controlpanel ``` Then you can run migrations: ```sh -python manage.py migrate +python3 manage.py migrate ``` ### Create superuser (on first run only) ```sh -python manage.py createsuperuser +python3 manage.py createsuperuser ``` NB `Username` needs to be your GitHub username @@ -109,21 +105,30 @@ NB `Username` needs to be your GitHub username Before the first run (or after changes to static assets), you need to run ```sh -python manage.py collectstatic +python3 manage.py collectstatic ``` ### Run the app -You can run the app with +You can run the app with the Django development server with +```sh +python3 manage.py runserver +``` +Or with Gunicorn WSGI server: ```sh -./run_api +gunicorn -b 0.0.0.0:8000 control_panel_api.wsgi:application ``` Go to http://localhost:8000/ ### How to run the tests ```sh -./run_tests +make test +``` + +You can run a specific test class or function by passing the `TEST_NAME` parameter, eg: +```sh +make test TEST_NAME=test_something ``` # Deployment diff --git a/control_panel_api/k8s_patch.py b/control_panel_api/k8s_patch.py index 03ed53e1b..299c56167 100644 --- a/control_panel_api/k8s_patch.py +++ b/control_panel_api/k8s_patch.py @@ -7,15 +7,8 @@ from kubernetes.config.kube_config import _is_expired -def load_token(self): - if 'auth-provider' not in self._user: - return - - provider = self._user['auth-provider'] - - if ('name' not in provider - or 'config' not in provider - or provider['name'] != 'oidc'): +def load_token(self, provider): + if 'config' not in provider: return parts = provider['config']['id-token'].split('.') @@ -23,8 +16,10 @@ def load_token(self): if len(parts) != 3: # Not a valid JWT return None + padding = (4 - len(parts[1]) % 4) * '=' + jwt_attributes = json.loads( - base64.b64decode(parts[1] + '==').decode('utf-8') + base64.b64decode(parts[1] + padding).decode('utf-8') ) expire = jwt_attributes.get('exp') diff --git a/control_panel_api/pagination.py b/control_panel_api/pagination.py index cd546ef83..34e37abf6 100644 --- a/control_panel_api/pagination.py +++ b/control_panel_api/pagination.py @@ -17,7 +17,7 @@ def paginate_queryset(self, queryset, request, view=None): if not self._page_size_is_all(page_size): return super().paginate_queryset(queryset, request, view) - paginator = self.django_paginator_class(queryset, queryset.count()) + paginator = self.django_paginator_class(queryset, queryset.count() or 1) self.page = paginator.page(1) return list(self.page) diff --git a/control_panel_api/serializers.py b/control_panel_api/serializers.py index de0342661..de21d2351 100644 --- a/control_panel_api/serializers.py +++ b/control_panel_api/serializers.py @@ -295,7 +295,7 @@ def to_representation(self, bucket_hits): return sorted(results, key=itemgetter('count'), reverse=True) def _get_accessed_by(self, key): - match = re.search(f"{settings.ENV}_(app|user)_([\w-]+)/", key) + match = re.search(rf"{settings.ENV}_(app|user)_([\w-]+)/", key) if match: return match.group(1), match.group(2) diff --git a/control_panel_api/settings/base.py b/control_panel_api/settings/base.py index 6400d1b8d..46a7294b1 100644 --- a/control_panel_api/settings/base.py +++ b/control_panel_api/settings/base.py @@ -116,9 +116,9 @@ def is_enabled(value): # Static files (CSS, JavaScript, Images) # https://docs.djangoproject.com/en/1.11/howto/static-files/ -STATIC_ROOT = os.path.join(BASE_DIR, 'static/') - -STATIC_URL = '/static/' +STATIC_ROOT = os.path.join(os.path.dirname(BASE_DIR), 'static') +STATIC_HOST = os.environ.get('DJANGO_STATIC_HOST', '') +STATIC_URL = STATIC_HOST + '/static/' REST_FRAMEWORK = { 'DEFAULT_AUTHENTICATION_CLASSES': [ diff --git a/control_panel_api/settings/test.py b/control_panel_api/settings/test.py index 7d59ca7af..6cf9f33c3 100644 --- a/control_panel_api/settings/test.py +++ b/control_panel_api/settings/test.py @@ -21,3 +21,7 @@ } LOGGING['handlers']['console']['level'] = 'CRITICAL' +ENABLED = { + 'k8s_rbac': False, + 'write_to_cluster': True, +} diff --git a/control_panel_api/views.py b/control_panel_api/views.py index 0a8547cfc..18be5c61e 100644 --- a/control_panel_api/views.py +++ b/control_panel_api/views.py @@ -123,7 +123,7 @@ class AppViewSet(viewsets.ModelViewSet): filter_backends = (DjangoFilterBackend,) permission_classes = (AppPermissions,) - filter_fields = ('name', 'repo_url', 'slug') + filterset_fields = ('name', 'repo_url', 'slug') @handle_external_exceptions @transaction.atomic @@ -238,7 +238,7 @@ class S3BucketViewSet(viewsets.ModelViewSet): serializer_class = S3BucketSerializer filter_backends = (S3BucketFilter,) permission_classes = (S3BucketPermissions,) - filter_fields = ('is_data_warehouse',) + filterset_fields = ('is_data_warehouse',) @handle_external_exceptions @transaction.atomic diff --git a/docker-compose.test.yml b/docker-compose.test.yml index 04e68d597..3dfd6b020 100644 --- a/docker-compose.test.yml +++ b/docker-compose.test.yml @@ -2,7 +2,7 @@ version: '3' services: app: build: . - image: "app" + image: "control-panel" ports: - "8000:8000" depends_on: @@ -15,7 +15,7 @@ services: DB_USER: "postgres" DEBUG: "True" DJANGO_SETTINGS_MODULE: "control_panel_api.settings.test" - command: ["./run_tests"] + command: sh -c "python3 wait_for_db && pytest --color=yes control_panel_api" db: image: "postgres:9.6.2" logging: diff --git a/docker-compose.yml b/docker-compose.yml index 8fa98d785..b25025d58 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,7 +2,7 @@ version: '3' services: app: build: . - image: app + image: control-panel ports: - "8000:8000" depends_on: @@ -22,8 +22,8 @@ services: POSTGRES_USER: "controlpanel" POSTGRES_DB: "controlpanel" migration: - image: app:latest - command: ["sh", "-c", "python3 wait_for_db && python3 manage.py migrate"] + image: control-panel + command: sh -c "python3 wait_for_db && python3 manage.py migrate" environment: DB_HOST: "db" DB_NAME: "controlpanel" diff --git a/requirements.txt b/requirements.txt index 49fd1e4a1..8c7a3b6c7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -30,7 +30,7 @@ kubernetes==8.0.1 MarkupSafe==1.1.0 model-mommy==1.6.0 openapi-codec==1.3.2 -psycopg2==2.7.7 +psycopg2-binary==2.7.7 py==1.7.0 pyasn1==0.4.4 pyasn1-modules==0.2.2 diff --git a/run_api b/run_api deleted file mode 100755 index 16b898269..000000000 --- a/run_api +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/sh - -set -e -o pipefail - -python3 wait_for_db && \ - gunicorn -b 0.0.0.0:8000 control_panel_api.wsgi:application diff --git a/run_tests b/run_tests deleted file mode 100755 index 991749d54..000000000 --- a/run_tests +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/sh - -set -e -o pipefail - -export DJANGO_SETTINGS_MODULE="control_panel_api.settings.test" - -python3 wait_for_db && pytest --color=yes --spec control_panel_api $@