Feat(): Enable async workers for flask and django (#11)

* Feat(): Enable async workers for flask and django

* Chore(): Log error when worker_class setting is wrong.

* chore(Lint): Run linter

* chore(): Formatted strings

* fix(websockets): Pinned websockets version to <14.0

* Chore(): Minor refactors. Apply comments

* Chore(async): Use Enum

* Chore(Integration test): Write integration tests for async workers.

* Chore(): Make linter happy

* Chore(): Renamed new tests.

* Chore(tests): Add async config tests for Django framework

* Chore(lint): Format the code

* Chore(docs): Update async doc link

* Chore(): Change config tests to uni, add integration test. Other comments

* Chore(lint): Format code

* Chore(): Add Django test, simplify Django async test app

* Chore(test): Improve tests and gevent module check

* Chore(): Changed implementation

* Update examples/flask/flask-async.rst

Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>

* Chore(): Remove wrongly placed doc

* chore(test): Use branch to source compile rockcraft & charmcraft for integration tests

* chore(test): Fix repo links

* chore(test): Increase timeout in integration tests

* chore(): Fix compatibility issue

* chore(test): Fix unit tests.

* chore(test): Use the latest/edge rockcraft instead of fork

* chore(trivy): Add 2 ignore lines for go libraries

---------

Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>

.github/workflows/integration_test.yaml

@@ -12,9 +12,12 @@ jobs:
     secrets: inherit
     with:
       extra-arguments: -x --localstack-address 172.17.0.1
+      charmcraft-repository: alithethird/charmcraft
+      charmcraft-ref: flask-async-worker
       pre-run-script: localstack-installation.sh
-      charmcraft-channel: latest/edge
+      # charmcraft-channel: latest/edge
       modules: '["test_charm.py", "test_cos.py", "test_database.py", "test_db_migration.py", "test_django.py", "test_django_integrations.py", "test_fastapi.py", "test_go.py", "test_integrations.py", "test_proxy.py", "test_workers.py"]'
       rockcraft-channel: latest/edge
       juju-channel: ${{ matrix.juju-version }}
       channel: 1.29-strict/stable
+      test-timeout: 30

.trivyignore

@@ -4,3 +4,7 @@ CVE-2022-40897
 CVE-2024-6345
 # pebble: Calling Decoder.Decode on a message which contains deeply nested structures can cause a panic due to stack exhaustion
 CVE-2024-34156
+# pebble: Go stdlib
+CVE-2024-45338
+# go-app: Go crypto lib
+CVE-2024-45337 

examples/django/charm/charmcraft.yaml

@@ -50,9 +50,9 @@ config:
       type: string
     django-secret-key-id:
       description: >-
-        This configuration is similar to `django-secret-key`, but instead accepts a Juju user secret ID. 
-        The secret should contain a single key, "value", which maps to the actual Django secret key. 
-        To create the secret, run the following command: 
+        This configuration is similar to `django-secret-key`, but instead accepts a Juju user secret ID.
+        The secret should contain a single key, "value", which maps to the actual Django secret key.
+        To create the secret, run the following command:
         `juju add-secret my-django-secret-key value=<secret-string> && juju grant-secret my-django-secret-key django-k8s`,
         and use the outputted secret ID to configure this option.
       type: secret
@@ -69,6 +69,9 @@ config:
     webserver-workers:
       description: The number of webserver worker processes for handling requests.
       type: int
+    webserver-worker-class:
+      description: The method of webserver worker processes for handling requests. Can be either 'gevent' or 'sync'.
+      type: string
 containers:
   django-app:
     resource: django-app-image

examples/django/django_async_app/django_async_app/django_async_app/__init__.py

@@ -0,0 +1,2 @@
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.

examples/django/django_async_app/django_async_app/django_async_app/asgi.py

@@ -0,0 +1,19 @@
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+"""
+ASGI config for django_async_app project.
+
+It exposes the ASGI callable as a module-level variable named ``application``.
+
+For more information on this file, see
+https://docs.djangoproject.com/en/5.0/howto/deployment/asgi/
+"""
+
+import os
+
+from django.core.asgi import get_asgi_application
+
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "django_async_app.settings")
+
+application = get_asgi_application()

examples/django/django_async_app/django_async_app/django_async_app/settings.py

@@ -0,0 +1,130 @@
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+"""
+Django settings for django_async_app project.
+
+Generated by 'django-admin startproject' using Django 5.0.2.
+
+For more information on this file, see
+https://docs.djangoproject.com/en/5.0/topics/settings/
+
+For the full list of settings and their values, see
+https://docs.djangoproject.com/en/5.0/ref/settings/
+"""
+
+import json
+import os
+import urllib.parse
+from pathlib import Path
+
+# Build paths inside the project like this: BASE_DIR / 'subdir'.
+BASE_DIR = Path(__file__).resolve().parent.parent
+
+
+# Quick-start development settings - unsuitable for production
+# See https://docs.djangoproject.com/en/5.0/howto/deployment/checklist/
+
+# SECURITY WARNING: keep the secret key used in production secret!
+SECRET_KEY = os.environ.get("DJANGO_SECRET_KEY", "secret")
+
+# SECURITY WARNING: don't run with debug turned on in production!
+DEBUG = os.environ.get("DJANGO_DEBUG", "true") == "true"
+
+ALLOWED_HOSTS = json.loads(os.environ["DJANGO_ALLOWED_HOSTS"])
+
+
+INSTALLED_APPS = [
+    "django.contrib.admin",
+    "django.contrib.auth",
+    "django.contrib.contenttypes",
+    "django.contrib.sessions",
+    "django.contrib.messages",
+    "django.contrib.staticfiles",
+]
+
+MIDDLEWARE = [
+    "django.middleware.security.SecurityMiddleware",
+    "django.contrib.sessions.middleware.SessionMiddleware",
+    "django.middleware.common.CommonMiddleware",
+    "django.middleware.csrf.CsrfViewMiddleware",
+    "django.contrib.auth.middleware.AuthenticationMiddleware",
+    "django.contrib.messages.middleware.MessageMiddleware",
+    "django.middleware.clickjacking.XFrameOptionsMiddleware",
+]
+
+ROOT_URLCONF = "django_async_app.urls"
+
+TEMPLATES = [
+    {
+        "BACKEND": "django.template.backends.django.DjangoTemplates",
+        "DIRS": [],
+        "APP_DIRS": True,
+        "OPTIONS": {
+            "context_processors": [
+                "django.template.context_processors.debug",
+                "django.template.context_processors.request",
+                "django.contrib.auth.context_processors.auth",
+                "django.contrib.messages.context_processors.messages",
+            ],
+        },
+    },
+]
+
+WSGI_APPLICATION = "django_async_app.wsgi.application"
+
+
+# Database
+# https://docs.djangoproject.com/en/5.0/ref/settings/#databases
+DATABASES = {
+    "default": {
+        "ENGINE": "django.db.backends.postgresql",
+        "NAME": os.environ.get("POSTGRESQL_DB_NAME"),
+        "USER": os.environ.get("POSTGRESQL_DB_USERNAME"),
+        "PASSWORD": os.environ.get("POSTGRESQL_DB_PASSWORD"),
+        "HOST": os.environ.get("POSTGRESQL_DB_HOSTNAME"),
+        "PORT": os.environ.get("POSTGRESQL_DB_PORT", "5432"),
+    }
+}
+
+
+# Password validation
+# https://docs.djangoproject.com/en/5.0/ref/settings/#auth-password-validators
+
+AUTH_PASSWORD_VALIDATORS = [
+    {
+        "NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator",
+    },
+    {
+        "NAME": "django.contrib.auth.password_validation.MinimumLengthValidator",
+    },
+    {
+        "NAME": "django.contrib.auth.password_validation.CommonPasswordValidator",
+    },
+    {
+        "NAME": "django.contrib.auth.password_validation.NumericPasswordValidator",
+    },
+]
+
+
+# Internationalization
+# https://docs.djangoproject.com/en/5.0/topics/i18n/
+
+LANGUAGE_CODE = "en-us"
+
+TIME_ZONE = "UTC"
+
+USE_I18N = True
+
+USE_TZ = True
+
+
+# Static files (CSS, JavaScript, Images)
+# https://docs.djangoproject.com/en/5.0/howto/static-files/
+
+STATIC_URL = "static/"
+
+# Default primary key field type
+# https://docs.djangoproject.com/en/5.0/ref/settings/#default-auto-field
+
+DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField"

examples/django/django_async_app/django_async_app/django_async_app/urls.py

@@ -0,0 +1,28 @@
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+"""
+URL configuration for django_async_app project.
+
+The `urlpatterns` list routes URLs to views. For more information please see:
+    https://docs.djangoproject.com/en/5.0/topics/http/urls/
+Examples:
+Function views
+    1. Add an import:  from my_app import views
+    2. Add a URL to urlpatterns:  path('', views.home, name='home')
+Class-based views
+    1. Add an import:  from other_app.views import Home
+    2. Add a URL to urlpatterns:  path('', Home.as_view(), name='home')
+Including another URLconf
+    1. Import the include() function: from django.urls import include, path
+    2. Add a URL to urlpatterns:  path('blog/', include('blog.urls'))
+"""
+
+from django.contrib import admin
+from django.urls import path
+from testing.views import sleep
+
+urlpatterns = [
+    path("admin/", admin.site.urls),
+    path("sleep", sleep, name="sleep"),
+]

examples/django/django_async_app/django_async_app/django_async_app/wsgi.py

@@ -0,0 +1,19 @@
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+"""
+WSGI config for django_async_app project.
+
+It exposes the WSGI callable as a module-level variable named ``application``.
+
+For more information on this file, see
+https://docs.djangoproject.com/en/5.0/howto/deployment/wsgi/
+"""
+
+import os
+
+from django.core.wsgi import get_wsgi_application
+
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "django_async_app.settings")
+
+application = get_wsgi_application()

examples/django/django_async_app/django_async_app/manage.py

@@ -0,0 +1,26 @@
+#!/usr/bin/env python3
+
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+"""Django's command-line utility for administrative tasks."""
+import os
+import sys
+
+
+def main():
+    """Run administrative tasks."""
+    os.environ.setdefault("DJANGO_SETTINGS_MODULE", "django_async_app.settings")
+    try:
+        from django.core.management import execute_from_command_line
+    except ImportError as exc:
+        raise ImportError(
+            "Couldn't import Django. Are you sure it's installed and "
+            "available on your PYTHONPATH environment variable? Did you "
+            "forget to activate a virtual environment?"
+        ) from exc
+    execute_from_command_line(sys.argv)
+
+
+if __name__ == "__main__":
+    main()

examples/django/django_async_app/django_async_app/testing/__init__.py

@@ -0,0 +1,2 @@
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.

examples/django/django_async_app/django_async_app/testing/admin.py

@@ -0,0 +1,6 @@
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+from django.contrib import admin
+
+# Register your models here.

examples/django/django_async_app/django_async_app/testing/apps.py

@@ -0,0 +1,9 @@
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+from django.apps import AppConfig
+
+
+class TestingConfig(AppConfig):
+    default_auto_field = "django.db.models.BigAutoField"
+    name = "testing"

examples/django/django_async_app/django_async_app/testing/migrations/__init__.py

@@ -0,0 +1,2 @@
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.

examples/django/django_async_app/django_async_app/testing/models.py

@@ -0,0 +1,6 @@
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+from django.db import models
+
+# Create your models here.

examples/django/django_async_app/django_async_app/testing/tests.py

@@ -0,0 +1,6 @@
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+from django.test import TestCase
+
+# Create your tests here.

examples/django/django_async_app/django_async_app/testing/views.py

@@ -0,0 +1,12 @@
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+import time
+
+from django.http import HttpResponse
+
+
+def sleep(request):
+    duration = request.GET.get("duration")
+    time.sleep(int(duration))
+    return HttpResponse()

examples/django/django_async_app/requirements.txt

@@ -0,0 +1,4 @@
+Django
+tzdata
+psycopg2-binary
+gevent

examples/django/django_async_app/rockcraft.yaml

@@ -0,0 +1,14 @@
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+name: django-async-app
+summary: Example Async Django application image.
+description: Example Async Django application image.
+version: "0.1"
+base: [email protected]
+license: Apache-2.0
+platforms:
+  amd64:
+
+extensions:
+  - django-framework

examples/flask/charmcraft.yaml

@@ -57,9 +57,9 @@ config:
       type: string
     flask-secret-key-id:
       description: >-
-        This configuration is similar to `flask-secret-key`, but instead accepts a Juju user secret ID. 
-        The secret should contain a single key, "value", which maps to the actual Flask secret key. 
-        To create the secret, run the following command: 
+        This configuration is similar to `flask-secret-key`, but instead accepts a Juju user secret ID.
+        The secret should contain a single key, "value", which maps to the actual Flask secret key.
+        To create the secret, run the following command:
         `juju add-secret my-flask-secret-key value=<secret-string> && juju grant-secret my-flask-secret-key flask-k8s`,
         and use the outputted secret ID to configure this option.
       type: secret
@@ -82,6 +82,9 @@ config:
     webserver-workers:
       description: The number of webserver worker processes for handling requests.
       type: int
+    webserver-worker-class:
+      description: The method of webserver worker processes for handling requests. Can be either 'gevent' or 'sync'.
+      type: string
     secret-test:
       description: A test configuration option for testing user provided Juju secrets.
       type: secret

examples/flask/test_async_rock/app.py

@@ -0,0 +1,23 @@
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+import logging
+import os
+import socket
+import time
+
+from flask import Flask, g, jsonify, request
+
+app = Flask(__name__)
+
+
+@app.route("/")
+def hello_world():
+    return "Hello, World!"
+
+
+@app.route("/sleep")
+def sleep():
+    duration_seconds = int(request.args.get("duration"))
+    time.sleep(duration_seconds)
+    return ""

examples/flask/test_async_rock/requirements.txt

@@ -0,0 +1,2 @@
+Flask
+gevent