From 5fbafad2d64d46d89c261c1b74fff190cec9f3ea Mon Sep 17 00:00:00 2001 From: Ori Hoch Date: Mon, 24 Apr 2023 11:37:51 +0300 Subject: [PATCH] dockerize, add ci/cd --- .dockerignore | 4 +++ .github/workflows/ci.yml | 51 +++++++++++++++++++++++++++ Dockerfile | 8 +++++ README.md | 45 ++++++++++++++++++++++++ compose.yaml | 71 ++++++++++++++++++++++++++++++++++++++ djang/djang/settings.py | 45 ++++++++++++++++++++---- djang/entrypoint.sh | 7 ++++ djang/gunicorn_conf.py | 68 ++++++++++++++++++++++++++++++++++++ ingress-nginx-default.conf | 12 +++++++ nginx.Dockerfile | 6 ++++ requirements.txt | 3 ++ 11 files changed, 313 insertions(+), 7 deletions(-) create mode 100644 .dockerignore create mode 100644 .github/workflows/ci.yml create mode 100644 Dockerfile create mode 100644 README.md create mode 100644 compose.yaml create mode 100755 djang/entrypoint.sh create mode 100644 djang/gunicorn_conf.py create mode 100644 ingress-nginx-default.conf create mode 100644 nginx.Dockerfile diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..0d2024c --- /dev/null +++ b/.dockerignore @@ -0,0 +1,4 @@ +**/*.sqlite3 +venv +.git +**/.gitignore diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..e683aab --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,51 @@ +name: CI +on: + push: +jobs: + ci: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v2 + - env: + DOCKER_APP_IMAGE_NAME: "ghcr.io/hasadna/open-pension-ng/open-pension-ng-app" + DOCKER_NGINX_IMAGE_NAME: "ghcr.io/hasadna/open-pension-ng/open-pension-ng-nginx" + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + HASADNA_K8S_DEPLOY_KEY: ${{ secrets.HASADNA_K8S_DEPLOY_KEY }} + MAIN_BRANCH: main + run: | + echo "${GITHUB_TOKEN}" | docker login ghcr.io -u hasadna --password-stdin &&\ + if docker pull "${DOCKER_APP_IMAGE_NAME}:latest"; then + CACHE_FROM_ARG="--cache-from ${DOCKER_APP_IMAGE_NAME}:latest" + else + CACHE_FROM_ARG="" + fi &&\ + docker build $CACHE_FROM_ARG --build-arg VERSION=${GITHUB_SHA} -t open-pension-ng . &&\ + docker tag open-pension-ng "${DOCKER_APP_IMAGE_NAME}:${GITHUB_SHA}" &&\ + docker push "${DOCKER_APP_IMAGE_NAME}:${GITHUB_SHA}" &&\ + if docker pull "${DOCKER_NGINX_IMAGE_NAME}:latest"; then + CACHE_FROM_ARG="--cache-from ${DOCKER_NGINX_IMAGE_NAME}:latest" + else + CACHE_FROM_ARG="" + fi &&\ + docker build $CACHE_FROM_ARG --build-arg VERSION=${GITHUB_SHA} -t nginx -f nginx.Dockerfile . &&\ + docker tag nginx "${DOCKER_NGINX_IMAGE_NAME}:${GITHUB_SHA}" &&\ + docker push "${DOCKER_NGINX_IMAGE_NAME}:${GITHUB_SHA}" &&\ + if [ "${GITHUB_REF}" == "refs/heads/${MAIN_BRANCH}" ]; then + docker tag open-pension-ng "${DOCKER_APP_IMAGE_NAME}:latest" &&\ + docker push "${DOCKER_APP_IMAGE_NAME}:latest" &&\ + docker tag nginx "${DOCKER_NGINX_IMAGE_NAME}:latest" &&\ + docker push "${DOCKER_NGINX_IMAGE_NAME}:latest" &&\ + if ! git log -1 --pretty=format:"%s" | grep -- --no-deploy; then + cd `mktemp -d` &&\ + echo "${HASADNA_K8S_DEPLOY_KEY}" > hasadna_k8s_deploy_key &&\ + chmod 400 hasadna_k8s_deploy_key &&\ + export GIT_SSH_COMMAND="ssh -i `pwd`/hasadna_k8s_deploy_key -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no" &&\ + git clone git@github.com:hasadna/hasadna-k8s.git &&\ + cd hasadna-k8s &&\ + python update_yaml.py '{"ngAppImage":"'"${DOCKER_APP_IMAGE_NAME}:${GITHUB_SHA}"'","ngNginxImage":"'"${DOCKER_NGINX_IMAGE_NAME}:${GITHUB_SHA}"'"}' apps/openpension/values-hasadna-auto-updated.yaml &&\ + git config --global user.name "open-pension-ng CI" &&\ + git config --global user.email "open-pension-ng-ci@localhost" &&\ + git add apps/openpension/values-hasadna-auto-updated.yaml && git commit -m "automatic update of open-pension-ng app" &&\ + git push origin master + fi + fi diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..cd51b91 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,8 @@ +# Pulled Apr 24, 2023 +FROM python:3.8@sha256:6aea47c16a4fe8a30f184060a2347caa24ea156b224041543fdc6b901dbec699 +WORKDIR /srv +COPY requirements.txt ./ +RUN pip install --upgrade pip && pip install -r requirements.txt && rm requirements.txt +COPY djang ./djang +WORKDIR /srv/djang +ENTRYPOINT ["/srv/djang/entrypoint.sh"] diff --git a/README.md b/README.md new file mode 100644 index 0000000..d9253c6 --- /dev/null +++ b/README.md @@ -0,0 +1,45 @@ +# Open Pension Next Generation + +## Setup + +``` +make init +``` + +## Running + +``` +make serve +``` + + +## Docker Compose development + +This environment resembles the production environment as closely as possible. + +Run migrations: + +```bash +docker-compose run --build --rm migrate +``` + +Start the web app: + +```bash +docker-compose up -d --build ingress +``` + +Access at http://localhost:8000 + +Start a shell to run management commands: + +```bash +docker-compose exec web bash +pytyhon manage.py +``` + +Start the Q Cluster: + +``` +docker-compose up -d --build qcluster +``` diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 0000000..c25b4ab --- /dev/null +++ b/compose.yaml @@ -0,0 +1,71 @@ +services: + db: + # Pulled Apr 24, 2023 + image: postgres:15@sha256:6cc97262444f1c45171081bc5a1d4c28b883ea46a6e0d1a45a8eac4a7f4767ab + environment: + POSTGRES_PASSWORD: "123456" + volumes: + - pgdata:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready --username=postgres"] + interval: 5s + timeout: 10s + retries: 5 + start_period: 80s + + migrate: + image: open-pension-ng + build: . + restart: on-failure + environment: &commonenv + DJANGO_SECRET_KEY: "#-@^btdzoe9n-swq=fx5(db5^ibkqsytrt0ie2n55efz7wtpk#" + DEBUG: "True" + DJANGO_DATABASE_ENGINE: "postgres" + DJANGO_DATABASE_NAME: "postgres" + DJANGO_DATABASE_USER: "postgres" + DJANGO_DATABASE_PASSWORD: "123456" + DJANGO_DATABASE_HOST: "db" + DJANGO_DATABASE_PORT: "5432" + DJANGO_ALLOWED_HOSTS: "*" + DJANGO_CSRF_TRUSTED_ORIGINS: "http://localhost:8000" + depends_on: &dependsondb + db: + condition: service_healthy + command: ["migrate"] + + nginx: + image: open-pension-ng-nginx + build: + context: . + dockerfile: nginx.Dockerfile + + web: + image: open-pension-ng + build: . + environment: + <<: *commonenv + depends_on: *dependsondb + command: ["web"] + + ingress: + # Pulled Apr 24, 2023 + image: nginx@sha256:63b44e8ddb83d5dd8020327c1f40436e37a6fffd3ef2498a6204df23be6e7e94 + volumes: + - ./ingress-nginx-default.conf:/etc/nginx/conf.d/default.conf + ports: + - "8000:80" + depends_on: + - nginx + - web + +# qcluster: +# image: open-pension-ng +# build: . +# environment: +# <<: *commonenv +# DJANGO_Q_CLUSTER_SETTINGS_JSON: '{"workers": 2}' +# depends_on: *dependsondb +# command: ["qcluster"] + +volumes: + pgdata: diff --git a/djang/djang/settings.py b/djang/djang/settings.py index 02bac63..b809e43 100644 --- a/djang/djang/settings.py +++ b/djang/djang/settings.py @@ -9,8 +9,10 @@ For the full list of settings and their values, see https://docs.djangoproject.com/en/4.2/ref/settings/ """ - +import os +import json from pathlib import Path +from django.core.management.utils import get_random_secret_key # Build paths inside the project like this: BASE_DIR / 'subdir'. BASE_DIR = Path(__file__).resolve().parent.parent @@ -20,12 +22,27 @@ # See https://docs.djangoproject.com/en/4.2/howto/deployment/checklist/ # SECURITY WARNING: keep the secret key used in production secret! -SECRET_KEY = "django-insecure-$q(pwx#ok@ih#z(met-+wur@_c1nu1&)r6aiojgm3tr5w(6%ak" +SECRET_KEY = os.environ.get("DJANGO_SECRET_KEY") +if not SECRET_KEY: + # for developement it's OK, for production we should set the DJANGO_SECRET_KEY env variable + SECRET_KEY = get_random_secret_key() # SECURITY WARNING: don't run with debug turned on in production! -DEBUG = True +DEBUG = os.environ.get("DJANGO_DEBUG") != "False" + +ALLOWED_HOSTS = [ + h.strip() + for h in os.environ.get("DJANGO_ALLOWED_HOSTS", "").split(",") + if h.strip() +] +if not ALLOWED_HOSTS: + ALLOWED_HOSTS = ["*"] -ALLOWED_HOSTS = [] +CSRF_TRUSTED_ORIGINS = [ + h.strip() + for h in os.environ.get("DJANGO_CSRF_TRUSTED_ORIGINS", "").split(",") + if h.strip() +] # Application definition @@ -73,12 +90,25 @@ # Database # https://docs.djangoproject.com/en/4.2/ref/settings/#databases -DATABASES = { - "default": { +database_engine = os.environ.get("DJANGO_DATABASE_ENGINE") +if database_engine == "postgres": + default_database = { + "ENGINE": "django.db.backends.postgresql", + "NAME": os.environ["DJANGO_DATABASE_NAME"], + "USER": os.environ["DJANGO_DATABASE_USER"], + "PASSWORD": os.environ["DJANGO_DATABASE_PASSWORD"], + "HOST": os.environ["DJANGO_DATABASE_HOST"], + "PORT": os.environ["DJANGO_DATABASE_PORT"], + } +elif not database_engine: + default_database = { "ENGINE": "django.db.backends.sqlite3", "NAME": BASE_DIR / "db.sqlite3", } -} +else: + raise Exception(f"Unsupported database engine: {database_engine}") + +DATABASES = {"default": default_database} # Password validation @@ -116,6 +146,7 @@ # https://docs.djangoproject.com/en/4.2/howto/static-files/ STATIC_URL = "static/" +STATIC_ROOT = os.environ.get("DJANGO_STATIC_ROOT") # Default primary key field type # https://docs.djangoproject.com/en/4.2/ref/settings/#default-auto-field diff --git a/djang/entrypoint.sh b/djang/entrypoint.sh new file mode 100755 index 0000000..92716ea --- /dev/null +++ b/djang/entrypoint.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +if [ "${1}" == "web" ]; then + exec gunicorn -k uvicorn.workers.UvicornWorker -c gunicorn_conf.py djang.asgi:application +else + exec python manage.py "$@" +fi diff --git a/djang/gunicorn_conf.py b/djang/gunicorn_conf.py new file mode 100644 index 0000000..bb26e43 --- /dev/null +++ b/djang/gunicorn_conf.py @@ -0,0 +1,68 @@ +# copied from https://github.com/tiangolo/uvicorn-gunicorn-docker/blob/master/docker-images/gunicorn_conf.py August 24, 2021 +import json +import multiprocessing +import os + +workers_per_core_str = os.getenv("WORKERS_PER_CORE", "1") +max_workers_str = os.getenv("MAX_WORKERS") +use_max_workers = None +if max_workers_str: + use_max_workers = int(max_workers_str) +web_concurrency_str = os.getenv("WEB_CONCURRENCY", None) + +host = os.getenv("HOST", "0.0.0.0") +port = os.getenv("PORT", "80") +bind_env = os.getenv("BIND", None) +use_loglevel = os.getenv("LOG_LEVEL", "info") +if bind_env: + use_bind = bind_env +else: + use_bind = f"{host}:{port}" + +cores = multiprocessing.cpu_count() +workers_per_core = float(workers_per_core_str) +default_web_concurrency = workers_per_core * cores +if web_concurrency_str: + web_concurrency = int(web_concurrency_str) + assert web_concurrency > 0 +else: + web_concurrency = max(int(default_web_concurrency), 2) + if use_max_workers: + web_concurrency = min(web_concurrency, use_max_workers) +accesslog_var = os.getenv("ACCESS_LOG", "-") +use_accesslog = accesslog_var or None +errorlog_var = os.getenv("ERROR_LOG", "-") +use_errorlog = errorlog_var or None +graceful_timeout_str = os.getenv("GRACEFUL_TIMEOUT", "120") +timeout_str = os.getenv("TIMEOUT", "120") +keepalive_str = os.getenv("KEEP_ALIVE", "5") + +# Gunicorn config variables +loglevel = use_loglevel +workers = web_concurrency +bind = use_bind +errorlog = use_errorlog +worker_tmp_dir = "/dev/shm" +accesslog = use_accesslog +graceful_timeout = int(graceful_timeout_str) +timeout = int(timeout_str) +keepalive = int(keepalive_str) + + +# For debugging and testing +log_data = { + "loglevel": loglevel, + "workers": workers, + "bind": bind, + "graceful_timeout": graceful_timeout, + "timeout": timeout, + "keepalive": keepalive, + "errorlog": errorlog, + "accesslog": accesslog, + # Additional, non-gunicorn variables + "workers_per_core": workers_per_core, + "use_max_workers": use_max_workers, + "host": host, + "port": port, +} +print(json.dumps(log_data)) diff --git a/ingress-nginx-default.conf b/ingress-nginx-default.conf new file mode 100644 index 0000000..ae21cf5 --- /dev/null +++ b/ingress-nginx-default.conf @@ -0,0 +1,12 @@ +server { + listen 80; + server_name localhost; + + location /static { + proxy_pass http://nginx/static; + } + + location / { + proxy_pass http://web; + } +} diff --git a/nginx.Dockerfile b/nginx.Dockerfile new file mode 100644 index 0000000..461d684 --- /dev/null +++ b/nginx.Dockerfile @@ -0,0 +1,6 @@ +FROM open-pension-ng +RUN DJANGO_STATIC_ROOT=/srv/static python manage.py collectstatic --noinput -c +# Pulled Apr 24, 2023 +FROM nginx@sha256:63b44e8ddb83d5dd8020327c1f40436e37a6fffd3ef2498a6204df23be6e7e94 +COPY --from=0 /srv/static /usr/share/nginx/html/static +RUN rm /usr/share/nginx/html/*.html diff --git a/requirements.txt b/requirements.txt index 6f47e34..8293bb5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,6 @@ asgiref==3.6.0 Django==4.2 sqlparse==0.4.4 +gunicorn==20.1.0 +uvicorn==0.20.0 +psycopg2-binary==2.9.5