Add Waitress support for production

Add resource metrics
Add psutil memory info metrics

.pre-commit-config.yaml

@@ -22,9 +22,6 @@ repos:
     rev: 1.1.2
     hooks:
       - id: copyright
-      - id: poetry-check
-        additional_dependencies:
-          - poetry==1.8.4 # pypi
       - id: poetry-lock
         additional_dependencies:
           - poetry==1.8.4 # pypi

Dockerfile

@@ -13,7 +13,7 @@ ENV PATH=/venv/bin:$PATH
 
 # Used to convert the locked packages by poetry to pip requirements format
 # We don't directly use `poetry install` because it force to use a virtual environment.
-FROM base-all-0 as poetry
+FROM base-all-0 AS poetry
 
 # Install Poetry
 WORKDIR /tmp
@@ -28,7 +28,7 @@ RUN poetry export --extras=all --output=requirements.txt \
   && poetry export --extras=all --with=dev --output=requirements-dev.txt
 
 # Base, the biggest thing is to install the Python packages
-FROM base-all-0 as base-all
+FROM base-all-0 AS base-all
 
 # The /poetry/requirements.txt file is build with the command
 # poetry export --extras=all --output=requirements.txt, see above
@@ -102,7 +102,7 @@ CMD ["/venv/bin/gunicorn"]
 
 COPY production.ini /app/
 
-FROM base-lint as tests
+FROM base-lint AS tests
 
 WORKDIR /opt/c2cwsgiutils
 COPY c2cwsgiutils ./c2cwsgiutils
@@ -113,4 +113,4 @@ RUN --mount=type=cache,target=/root/.cache \
   POETRY_DYNAMIC_VERSIONING_BYPASS=${VERSION} python3 -m pip install --disable-pip-version-check --no-deps --editable=. \
   && python3 -m pip freeze > /requirements.txt
 
-FROM base as standard
+FROM base AS standard

Makefile

@@ -33,7 +33,7 @@ tests: build_docker_test ## Run the unit tests
 acceptance: acceptance-in acceptance-out  ## Run the acceptance tests
 
 .PHONY: acceptance-run
-acceptance-run: tests  ## Start the application used to run the acceptance tests
+acceptance-run: tests build_test_app  ## Start the application used to run the acceptance tests
 	cd acceptance_tests/tests/; docker compose up --detach db db_slave
 	cd acceptance_tests/tests/; docker compose run -T --no-deps app /app/scripts/wait-db
 	cd acceptance_tests/tests/; docker compose up --detach
@@ -60,8 +60,15 @@ build_docker_test:
 	docker build --tag=$(DOCKER_BASE):tests --target=tests .
 
 .PHONY: build_test_app
-build_test_app: build_docker
-	docker build --tag=$(DOCKER_BASE)_test_app --build-arg="GIT_HASH=$(GIT_HASH)" acceptance_tests/app
+build_test_app: build_test_app_gunicorn build_test_app_waitress
+
+.PHONY: build_test_app_gunicorn
+build_test_app_gunicorn: build_docker
+	docker build --tag=$(DOCKER_BASE)_test_app --build-arg="GIT_HASH=$(GIT_HASH)" acceptance_tests/gunicorn_app
+
+.PHONY: build_test_app_waitress
+build_test_app_waitress: build_docker
+	docker build --tag=$(DOCKER_BASE)_test_app_waitress --build-arg="GIT_HASH=$(GIT_HASH)" acceptance_tests/waitress_app
 
 .PHONY: checks
 checks: prospector ## Run the checks

README.md

@@ -608,6 +608,64 @@ def hello_get(request):
     return {'hello': True}
 ```
 
+## Waitress
+
+In production mode we usually use Gunicorn but we can also use Waitress.
+
+The advantage to use Waitress it that he creates only one process, that makes it easier to manage especially on Kubernetes:
+
+- The memory is more stable.
+- The OOM killer will restart the container.
+- Prometheus didn't request trick to aggregate the metrics.
+
+Then to migrate from Gunicorn to Waitress you should do:
+
+Add call to `c2cwsgiutils.prometheus.start_single_process()` on your application main function.
+
+Changes to do in your docker file:
+
+```diff
+
+ ENV \
+-   GUNICORN_LOG_LEVEL=WARNING \
++   WAITRESS_LOG_LEVEL=WARNING \
++   WAITRESS_THREADS=10 \
+
+-RUN mkdir -p /prometheus-metrics \
+-  && chmod a+rwx /prometheus-metrics
+-ENV PROMETHEUS_MULTIPROC_DIR=/prometheus-metrics
+
+
+-CMD ["/venv/bin/gunicorn", "--paste=/app/production.ini"]
++CMD ["/venv/bin/pserve", "c2c:///app/production.ini"]
+```
+
+Remove the no more needed file `gunicorn.conf.py`.
+
+Update the `production.ini` file:
+
+```diff
+
+-# this file should be used by gunicorn.
+
+ [server:main]
++threads = %(WAITRESS_THREADS)s
++trusted_proxy = True
++clear_untrusted_proxy_headers = False
+
+ [loggers]
+-keys = root, gunicorn, sqlalchemy, c2cwsgiutils, c2cwsgiutils_app
++keys = root, waitress, sqlalchemy, c2cwsgiutils, c2cwsgiutils_app
+
+-[logger_gunicorn]
+-level = %(GUNICORN_LOG_LEVEL)s
++[logger_waitress]
++level = %(WAITRESS_LOG_LEVEL)s
+ handlers =
+-qualname = gunicorn.error
++qualname = waitress
+```
+
 # Exception handling
 
 c2cwsgiutils can install exception handling views that will catch any exception raised by the

acceptance_tests/gunicorn_app/Dockerfile

@@ -0,0 +1,70 @@
+FROM camptocamp/c2cwsgiutils AS base-all
+LABEL maintainer Camptocamp "[email protected]"
+SHELL ["/bin/bash", "-o", "pipefail", "-cux"]
+
+# Used to convert the locked packages by poetry to pip requirements format
+# We don't directly use `poetry install` because it force to use a virtual environment.
+FROM base-all AS poetry
+
+RUN --mount=type=cache,target=/var/lib/apt/lists \
+  --mount=type=cache,target=/var/cache,sharing=locked \
+  apt-get update \
+  && apt-get install --assume-yes --no-install-recommends python-is-python3
+
+# Install Poetry
+WORKDIR /tmp
+COPY requirements.txt ./
+RUN python3 -m pip install --disable-pip-version-check --requirement=requirements.txt
+
+# Do the conversion
+COPY poetry.lock pyproject.toml ./
+RUN poetry export --output=requirements.txt \
+  && poetry export --with=dev --output=requirements-dev.txt
+
+# Base, the biggest thing is to install the Python packages
+FROM base-all AS base
+
+WORKDIR /app
+
+EXPOSE 8080
+RUN --mount=type=cache,target=/root/.cache \
+  --mount=type=bind,from=poetry,source=/tmp,target=/poetry \
+  python3 -m pip install --disable-pip-version-check --no-deps --requirement=/poetry/requirements.txt
+
+COPY . /app
+
+ARG GIT_HASH
+
+RUN --mount=type=cache,target=/root/.cache \
+  python3 -m pip install --disable-pip-version-check --no-deps --editable=. \
+  && python3 -m pip freeze > /requirements.txt
+RUN ./models_graph.py > models.dot \
+  && ./models_graph.py Hello > models-hello.dot \
+  && c2cwsgiutils-genversion $GIT_HASH \
+  && python3 -m compileall -q .
+
+ENV \
+  DOCKER_RUN=1 \
+  DEVELOPMENT=0 \
+  SQLALCHEMY_POOL_RECYCLE=30 \
+  SQLALCHEMY_POOL_SIZE=5 \
+  SQLALCHEMY_MAX_OVERFLOW=25 \
+  SQLALCHEMY_SLAVE_POOL_RECYCLE=30 \
+  SQLALCHEMY_SLAVE_POOL_SIZE=5 \
+  SQLALCHEMY_SLAVE_MAX_OVERFLOW=25 \
+  LOG_TYPE=console \
+  OTHER_LOG_LEVEL=WARNING \
+  GUNICORN_LOG_LEVEL=WARNING \
+  SQL_LOG_LEVEL=WARNING \
+  C2CWSGIUTILS_LOG_LEVEL=WARNING \
+  LOG_LEVEL=INFO \
+  VISIBLE_ENTRY_POINT=/
+
+RUN mkdir -p /prometheus-metrics \
+  && chmod a+rwx /prometheus-metrics
+ENV PROMETHEUS_MULTIPROC_DIR=/prometheus-metrics
+
+# www-data
+USER 33
+
+CMD ["/venv/bin/gunicorn", "--paste=/app/production.ini"]

acceptance_tests/app/c2cwsgiutils_app/__init__.py → acceptance_tests/gunicorn_app/c2cwsgiutils_app/__init__.py

@@ -1,11 +1,12 @@
-from c2cwsgiutils_app import models
 from pyramid.config import Configurator
 from pyramid.httpexceptions import HTTPInternalServerError
 
 import c2cwsgiutils.pyramid
 from c2cwsgiutils import broadcast, db
 from c2cwsgiutils.health_check import HealthCheck, JsonCheckException
 
+from c2cwsgiutils_app import models
+
 
 def _failure(_request):
     raise HTTPInternalServerError("failing check")

acceptance_tests/app/c2cwsgiutils_app/get_hello.py → acceptance_tests/gunicorn_app/c2cwsgiutils_app/get_hello.py

@@ -2,11 +2,12 @@
 
 import psycopg2
 import transaction
-from c2cwsgiutils_app import models
 
 import c2cwsgiutils.db
 import c2cwsgiutils.setup_process
 
+from c2cwsgiutils_app import models
+
 
 def _fill_db():
     for db, value in (("db", "master"), ("db_slave", "slave")):

acceptance_tests/app/c2cwsgiutils_app/services.py → acceptance_tests/gunicorn_app/c2cwsgiutils_app/services.py

@@ -2,7 +2,6 @@
 
 import prometheus_client
 import requests
-from c2cwsgiutils_app import models
 from pyramid.httpexceptions import (
     HTTPBadRequest,
     HTTPForbidden,
@@ -13,6 +12,8 @@
 
 from c2cwsgiutils import sentry, services
 
+from c2cwsgiutils_app import models
+
 _PROMETHEUS_TEST_COUNTER = prometheus_client.Counter("test_counter", "Test counter")
 _PROMETHEUS_TEST_GAUGE = prometheus_client.Gauge("test_gauge", "Test gauge", ["value", "toto"])
 _PROMETHEUS_TEST_SUMMARY = prometheus_client.Summary("test_summary", "Test summary")

acceptance_tests/app/models_graph.py → acceptance_tests/gunicorn_app/models_graph.py

@@ -1,8 +1,8 @@
 #!/usr/bin/env python3
-from c2cwsgiutils_app import models
-
 from c2cwsgiutils.models_graph import generate_model_graph
 
+from c2cwsgiutils_app import models
+
 
 def main():
     generate_model_graph(models)

acceptance_tests/app/production.ini → acceptance_tests/gunicorn_app/production.ini

@@ -1,11 +1,12 @@
 ###
-# app configuration
+# Application configuration
 # http://docs.pylonsproject.org/projects/pyramid/en/1.6-branch/narr/environment.html
 # this file should be used by gunicorn.
 ###
 
 [app:app]
 use = egg:c2cwsgiutils_app
+filter-with = proxy-prefix
 
 pyramid.reload_templates = %(DEVELOPMENT)s
 pyramid.debug_authorization = %(DEVELOPMENT)s
@@ -27,15 +28,23 @@ sqlalchemy_slave.max_overflow = %(SQLALCHEMY_SLAVE_MAX_OVERFLOW)s
 c2c.sql_request_id = True
 c2c.requests_default_timeout = 2
 
+[filter:proxy-prefix]
+use = egg:PasteDeploy#prefix
+prefix = %(VISIBLE_ENTRY_POINT)s
+
+[filter:translogger]
+use = egg:Paste#translogger
+setup_console_handler = False
+
 [pipeline:main]
-pipeline = egg:c2cwsgiutils#client_info egg:c2cwsgiutils#sentry app
+pipeline = egg:c2cwsgiutils#client_info egg:c2cwsgiutils#sentry translogger app
 
 [server:main]
 use = egg:waitress#main
 listen = *:8080
 
 ###
-# logging configuration
+# Logging configuration
 # http://docs.pylonsproject.org/projects/pyramid/en/latest/narr/logging.html
 ###
 

acceptance_tests/gunicorn_app/run-alembic

@@ -0,0 +1,24 @@
+#!/bin/bash
+# Upgrade the DB
+set -eu
+
+# wait for the DB to be UP
+while ! echo "import sqlalchemy; sqlalchemy.create_engine('${SQLALCHEMY_URL}').connect()" | python3 2> /dev/null; do
+  echo "Waiting for the DB to be reachable"
+  sleep 1
+done
+
+for ini in *alembic*.ini; do
+  if [[ -f "${ini}" ]]; then
+    echo "${ini} ==========================="
+    echo History
+    alembic -c "${ini}" history
+    echo Head
+    alembic -c "${ini}" heads
+    echo Upgrade
+    alembic -c "${ini}" upgrade head
+    echo Current
+    alembic -c "${ini}" current
+    echo "==========================="
+  fi
+done

acceptance_tests/app/scripts/wait-db → acceptance_tests/gunicorn_app/scripts/wait-db

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 
-# Copyright (c) 2023, Camptocamp SA
+# Copyright (c) 2023-2024, Camptocamp SA
 # All rights reserved.
 
 # Redistribution and use in source and binary forms, with or without

acceptance_tests/tests/docker-compose.override.sample.yaml

@@ -4,20 +4,28 @@ services:
   app:
     # Uncomment to use pserve
     # command:
-    #   - pserve
+    #   - /venv/bin/pserve
     #   - --reload
     #   - c2c:///app/production.ini
     volumes:
       # This mounts the local filesystem inside the container so that
       # the views are automatically reloaded when a file change
       - ../app/c2cwsgiutils_app/:/app/c2cwsgiutils_app/:ro
       - ../../c2cwsgiutils/:/opt/c2cwsgiutils/c2cwsgiutils/:ro
+    environment:
+      - DEVELOPMENT=TRUE
     ports:
       - 9090:9090
 
   app2:
+    # command:
+    #   - /venv/bin/pserve
+    #   - --reload
+    #   - c2c:///app/production.ini
     ports:
       - 9092:9090
+    environment:
+      - DEVELOPMENT=TRUE
 
   db:
     ports: