Skip to content

Commit

Permalink
Tidy up, fix warnings and use a Makefile (#508)
Browse files Browse the repository at this point in the history
* Tidy up, fix warnings and use a Makefile

* Update README.md

* Remove make dependency in docker image

* Remove unused USE_VENV variable

* Can't run specific tests with docker-compose

* Fix pagination with no items
  • Loading branch information
andyhd authored Feb 1, 2019
1 parent 936fcc5 commit b2444ea
Show file tree
Hide file tree
Showing 14 changed files with 134 additions and 100 deletions.
71 changes: 25 additions & 46 deletions Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -2,63 +2,42 @@ FROM alpine:3.7

MAINTAINER Andy Driver <[email protected]>

# install build dependencies (they'll be uninstalled after pip install)
RUN apk add --no-cache \
--virtual build-deps \
gcc \
musl-dev
ENV HELM_VERSION 2.9.1
ENV HELM_HOME /tmp/helm
ENV DJANGO_SETTINGS_MODULE "control_panel_api.settings"

# install python3 and 'ca-certificates' so that HTTPS works consistently
WORKDIR /home/control-panel

# install build dependencies (they'll be uninstalled after pip install)
RUN apk add --no-cache \
build-base \
openssl \
ca-certificates \
libffi-dev \
python3-dev

# Temporary bugfix for libressl
# Postgres needs libressl-dev, but cryptography only works with openssl-dev
RUN apk add --no-cache --virtual temp-ssl-fix \
openssl-dev \
&& pip3 install cryptography==2.2.2 \
&& apk del temp-ssl-fix \
&& apk add --no-cache \
python3-dev \
libressl-dev \
postgresql-dev

# Install helm
ENV HELM_VERSION 2.9.1
RUN wget https://storage.googleapis.com/kubernetes-helm/helm-v$HELM_VERSION-linux-amd64.tar.gz \
&& tar xzf helm-v$HELM_VERSION-linux-amd64.tar.gz \
&& mv linux-amd64/helm /usr/local/bin \
&& rm -rf helm-v$HELM_VERSION-linux-amd64.tar.gz linux-amd64

# Configure helm
ENV HELM_HOME /tmp/helm
RUN helm init --client-only
# Install and configure helm
COPY helm-repositories.yaml /tmp/helm/repository/repositories.yaml
RUN helm repo update

WORKDIR /home/control-panel

# install python dependencies
ADD requirements.txt requirements.txt
RUN pip3 install -r requirements.txt

# uninstall build dependencies
RUN apk del build-deps

ENV DJANGO_SETTINGS_MODULE "control_panel_api.settings"

ADD manage.py manage.py
ADD run_api run_api
ADD run_tests run_tests
ADD wait_for_db wait_for_db
ADD control_panel_api control_panel_api
ADD moj_analytics moj_analytics
RUN wget https://storage.googleapis.com/kubernetes-helm/helm-v${HELM_VERSION}-linux-amd64.tar.gz -O helm.tgz \
&& tar fxz helm.tgz \
&& mv linux-amd64/helm /usr/local/bin \
&& rm -rf helm.tgz linux-amd64 \
&& helm init --client-only \
&& helm repo update

# install python dependencies (and then remove build dependencies)
COPY requirements.txt ./
RUN pip3 install -r requirements.txt \
&& apk del build-base

COPY control_panel_api control_panel_api
COPY moj_analytics moj_analytics
COPY manage.py wait_for_db ./

# collect static files for deployment
RUN python3 manage.py collectstatic

EXPOSE 8000

CMD ["./run_api"]
CMD ["gunicorn", "-b", "0.0.0.0:8000", "control_panel_api.wsgi:application"]
64 changes: 64 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
HOST=0.0.0.0
PORT=8000
PROJECT=control-panel
MODULE=control_panel_api
VENV=venv
BIN=${VENV}/bin

-include .env
export

.PHONY: collectstatic dependencies help run test wait_for_db

venv/bin:
@if ${USE_VENV} && [ ! -d "${VENV}" ] ; then python3 -m venv ${VENV} ; fi

## dependencies: Install dependencies
dependencies: ${BIN} requirements.txt
@echo
@echo "> Fetching dependencies..."
@${BIN}/pip3 install -r requirements.txt

## collectstatic: Collect assets into static folder
collectstatic: dependencies
@echo
@echo "> Collecting static assets..."
@${BIN}/python3 manage.py collectstatic --noinput

## run: Run webapp
run: collectstatic
@echo
@echo "> Running webapp..."
@${BIN}/gunicorn -b ${HOST}:${PORT} ${MODULE}.wsgi:application

wait_for_db:
@echo
@echo "> Waiting for database..."
@${BIN}/python3 wait_for_db

## test: Run tests
test: export DJANGO_SETTINGS_MODULE=${MODULE}.settings.test
test: wait_for_db
@echo
@echo "> Running tests..."
@NAMED_TESTS="$(shell if [ -n "${TEST_NAME}" ]; then echo "-k ${TEST_NAME}" ; fi)" && \
${BIN}/pytest --color=yes ${MODULE} $$NAMED_TESTS

## docker-image: Build docker image
docker-image:
@echo
@echo "> Building docker image..."
@docker build -t ${PROJECT} .

## docker-test: Run tests in Docker container
docker-test:
@echo
@echo "> Running tests in Docker..."
@docker-compose -f docker-compose.test.yml up --abort-on-container-exit

help: Makefile
@echo
@echo " Commands in "$(PROJECT)":"
@echo
@sed -n 's/^##//p' $< | column -t -s ':' | sed -e 's/^/ /'
@echo
41 changes: 23 additions & 18 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ https://github.com/ministryofjustice/analytics-platform-control-panel-frontend
## Running with Docker

```sh
docker-compose build
docker-compose build # OR make docker-image
docker-compose up
```
and then in a separate terminal window,
Expand All @@ -21,12 +21,7 @@ Then browse to http://localhost:8000/
### Running tests with docker

```sh
docker-compose -f docker-compose.test.yml up --build --abort-on-container-exit
```

You can run a particular test using the pytest '-k' parameter:
```sh
docker-compose -f docker-compose.test.yml build && docker-compose -f docker-compose.test.yml run app ./run_tests -k TEST-NAME
make docker-test
```

## Running directly on your machine
Expand All @@ -35,11 +30,11 @@ docker-compose -f docker-compose.test.yml build && docker-compose -f docker-comp

The Control Panel app requires Python 3.6+

It is best to use a virtual environment to install python dependencies, eg:
Install dependencies with the following command:
```sh
python -m venv venv
. venv/bin/activate
pip install -r requirements.txt
python3 -m venv venv
source venv/bin/activate
pip3 install -r requirements.txt
```

### Kubernetes setup
Expand Down Expand Up @@ -90,40 +85,50 @@ export DJANGO_SETTINGS_MODULE=control_panel_api.settings

The Control Panel app connects to a PostgreSQL database, which should have a database with the expected name:
```sh
createdb $DB_NAME
createuser -d controlpanel
createdb -U controlpanel controlpanel
```

Then you can run migrations:
```sh
python manage.py migrate
python3 manage.py migrate
```

### Create superuser (on first run only)

```sh
python manage.py createsuperuser
python3 manage.py createsuperuser
```
NB `Username` needs to be your GitHub username

### Compile Sass and Javascript

Before the first run (or after changes to static assets), you need to run
```sh
python manage.py collectstatic
python3 manage.py collectstatic
```

### Run the app

You can run the app with
You can run the app with the Django development server with
```sh
python3 manage.py runserver
```
Or with Gunicorn WSGI server:
```sh
./run_api
gunicorn -b 0.0.0.0:8000 control_panel_api.wsgi:application
```
Go to http://localhost:8000/

### How to run the tests

```sh
./run_tests
make test
```

You can run a specific test class or function by passing the `TEST_NAME` parameter, eg:
```sh
make test TEST_NAME=test_something
```

# Deployment
Expand Down
15 changes: 5 additions & 10 deletions control_panel_api/k8s_patch.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,24 +7,19 @@
from kubernetes.config.kube_config import _is_expired


def load_token(self):
if 'auth-provider' not in self._user:
return

provider = self._user['auth-provider']

if ('name' not in provider
or 'config' not in provider
or provider['name'] != 'oidc'):
def load_token(self, provider):
if 'config' not in provider:
return

parts = provider['config']['id-token'].split('.')

if len(parts) != 3: # Not a valid JWT
return None

padding = (4 - len(parts[1]) % 4) * '='

jwt_attributes = json.loads(
base64.b64decode(parts[1] + '==').decode('utf-8')
base64.b64decode(parts[1] + padding).decode('utf-8')
)

expire = jwt_attributes.get('exp')
Expand Down
2 changes: 1 addition & 1 deletion control_panel_api/pagination.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ def paginate_queryset(self, queryset, request, view=None):
if not self._page_size_is_all(page_size):
return super().paginate_queryset(queryset, request, view)

paginator = self.django_paginator_class(queryset, queryset.count())
paginator = self.django_paginator_class(queryset, queryset.count() or 1)
self.page = paginator.page(1)

return list(self.page)
Expand Down
2 changes: 1 addition & 1 deletion control_panel_api/serializers.py
Original file line number Diff line number Diff line change
Expand Up @@ -295,7 +295,7 @@ def to_representation(self, bucket_hits):
return sorted(results, key=itemgetter('count'), reverse=True)

def _get_accessed_by(self, key):
match = re.search(f"{settings.ENV}_(app|user)_([\w-]+)/", key)
match = re.search(rf"{settings.ENV}_(app|user)_([\w-]+)/", key)

if match:
return match.group(1), match.group(2)
Expand Down
6 changes: 3 additions & 3 deletions control_panel_api/settings/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -116,9 +116,9 @@ def is_enabled(value):
# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/1.11/howto/static-files/

STATIC_ROOT = os.path.join(BASE_DIR, 'static/')

STATIC_URL = '/static/'
STATIC_ROOT = os.path.join(os.path.dirname(BASE_DIR), 'static')
STATIC_HOST = os.environ.get('DJANGO_STATIC_HOST', '')
STATIC_URL = STATIC_HOST + '/static/'

REST_FRAMEWORK = {
'DEFAULT_AUTHENTICATION_CLASSES': [
Expand Down
4 changes: 4 additions & 0 deletions control_panel_api/settings/test.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,3 +21,7 @@
}

LOGGING['handlers']['console']['level'] = 'CRITICAL'
ENABLED = {
'k8s_rbac': False,
'write_to_cluster': True,
}
4 changes: 2 additions & 2 deletions control_panel_api/views.py
Original file line number Diff line number Diff line change
Expand Up @@ -123,7 +123,7 @@ class AppViewSet(viewsets.ModelViewSet):
filter_backends = (DjangoFilterBackend,)
permission_classes = (AppPermissions,)

filter_fields = ('name', 'repo_url', 'slug')
filterset_fields = ('name', 'repo_url', 'slug')

@handle_external_exceptions
@transaction.atomic
Expand Down Expand Up @@ -238,7 +238,7 @@ class S3BucketViewSet(viewsets.ModelViewSet):
serializer_class = S3BucketSerializer
filter_backends = (S3BucketFilter,)
permission_classes = (S3BucketPermissions,)
filter_fields = ('is_data_warehouse',)
filterset_fields = ('is_data_warehouse',)

@handle_external_exceptions
@transaction.atomic
Expand Down
4 changes: 2 additions & 2 deletions docker-compose.test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ version: '3'
services:
app:
build: .
image: "app"
image: "control-panel"
ports:
- "8000:8000"
depends_on:
Expand All @@ -15,7 +15,7 @@ services:
DB_USER: "postgres"
DEBUG: "True"
DJANGO_SETTINGS_MODULE: "control_panel_api.settings.test"
command: ["./run_tests"]
command: sh -c "python3 wait_for_db && pytest --color=yes control_panel_api"
db:
image: "postgres:9.6.2"
logging:
Expand Down
6 changes: 3 additions & 3 deletions docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ version: '3'
services:
app:
build: .
image: app
image: control-panel
ports:
- "8000:8000"
depends_on:
Expand All @@ -22,8 +22,8 @@ services:
POSTGRES_USER: "controlpanel"
POSTGRES_DB: "controlpanel"
migration:
image: app:latest
command: ["sh", "-c", "python3 wait_for_db && python3 manage.py migrate"]
image: control-panel
command: sh -c "python3 wait_for_db && python3 manage.py migrate"
environment:
DB_HOST: "db"
DB_NAME: "controlpanel"
Expand Down
2 changes: 1 addition & 1 deletion requirements.txt
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ kubernetes==8.0.1
MarkupSafe==1.1.0
model-mommy==1.6.0
openapi-codec==1.3.2
psycopg2==2.7.7
psycopg2-binary==2.7.7
py==1.7.0
pyasn1==0.4.4
pyasn1-modules==0.2.2
Expand Down
6 changes: 0 additions & 6 deletions run_api

This file was deleted.

7 changes: 0 additions & 7 deletions run_tests

This file was deleted.

0 comments on commit b2444ea

Please sign in to comment.