Cloud agnostic logger

.coveragerc

@@ -0,0 +1,20 @@
+[run]
+omit =
+    */.local/*
+    */apps.py
+    */admin.py
+    */journey_tests/*
+    */asgi.py
+    */wsgi.py
+    */settings*
+    */manage.py
+    */streamlit_main.py
+    */migrations/*
+source = django_google_structured_logger
+relative_files = True
+
+[html]
+directory = coverage-reports
+
+[xml]
+output = coverage-reports/coverage.xml

.github/workflows/pr-validation.yaml

@@ -0,0 +1,60 @@
+name: PR Validation
+
+on:
+  pull_request:
+    branches: main
+  push:
+    branches: main
+
+jobs:
+  quality-check:
+    runs-on: ubuntu-24.04
+    name: Source code quality check
+    steps:
+      - name: Get latest SHA on the branch
+        id: get_sha
+        run: |
+          echo "COMMIT_SHA=${{ github.event.pull_request.head.sha || github.sha }}" >> $GITHUB_OUTPUT
+
+      - name: Checkout Code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ steps.get_sha.outputs.COMMIT_SHA }}
+          fetch-depth: 0
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.13"
+
+      - name: Install Poetry and dependencies
+        run: |
+          pip install poetry==2.1.2
+          poetry sync -vv --all-groups --no-cache --no-interaction
+
+      - name: Run tests
+        run: |
+          echo '### tests result' >> $GITHUB_STEP_SUMMARY
+          echo '```' >> $GITHUB_STEP_SUMMARY
+          poetry run pytest >> $GITHUB_STEP_SUMMARY
+          echo '```' >> $GITHUB_STEP_SUMMARY
+
+      - name: Type checks
+        run: |
+          echo '### mypy checks' >> $GITHUB_STEP_SUMMARY
+          echo '```' >> $GITHUB_STEP_SUMMARY
+          poetry run mypy django_google_structured_logger >> $GITHUB_STEP_SUMMARY
+          poetry run mypy tests >> $GITHUB_STEP_SUMMARY
+          echo '```' >> $GITHUB_STEP_SUMMARY
+
+      - name: Lint
+        run: |
+          echo '### ruff linter checks' >> $GITHUB_STEP_SUMMARY
+          echo '```' >> $GITHUB_STEP_SUMMARY  
+          poetry run ruff check >> $GITHUB_STEP_SUMMARY
+          echo '```' >> $GITHUB_STEP_SUMMARY
+
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v5
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          files: ./coverage-reports/coverage.xml

.github/workflows/release.yml

@@ -18,7 +18,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: 3.9
+          python-version: 3.10
 
       - name: Install dependencies
         run: |

.gitignore

@@ -5,6 +5,7 @@
 *.pyc
 /reports
 /.coverage
+coverage-reports/
 /build
 .idea
 dist
@@ -183,3 +184,5 @@ cython_debug/
 
 # PyPI configuration file
 .pypirc
+
+.vscode/

README.md

@@ -3,6 +3,10 @@
 [![PyPI version](https://badge.fury.io/py/django-google-structured-logger.svg)](https://badge.fury.io/py/django-google-structured-logger)
 [![Python Versions](https://img.shields.io/pypi/pyversions/django-google-structured-logger)](https://pypi.org/project/django-google-structured-logger/)
 [![Django Versions](https://img.shields.io/pypi/djversions/django-google-structured-logger)](https://pypi.org/project/django-google-structured-logger/)
+[![codecov](https://codecov.io/gh/muehlemann-popp/django-google-structured-logger/graph/badge.svg?token=2X2RMRFOZO)](https://codecov.io/gh/muehlemann-popp/django-google-structured-logger)
+![Mypy Checked](https://img.shields.io/badge/checked%20with-mypy-blue.svg)
+[![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff)
+[![Taskfile](https://img.shields.io/badge/Task-Taskfile-blue?logo=data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHdpZHRoPSI1MDAiIGhlaWdodD0iNTAwIiB2aWV3Qm94PSIwIDAgMzc1IDM3NSI+PHBhdGggZmlsbD0iIzI5YmViMCIgZD0iTSAxODcuNTcwMzEyIDE5MC45MzM1OTQgTCAxODcuNTcwMzEyIDM3NSBMIDMwLjA3MDMxMiAyNzkuNTM1MTU2IEwgMzAuMDcwMzEyIDk1LjQ2NDg0NCBaIi8+PHBhdGggZmlsbD0iIzY5ZDJjOCIgZD0iTSAxODcuNTcwMzEyIDE5MC45MzM1OTQgTCAxODcuNTcwMzEyIDM3NSBMIDM0NS4wNzAzMTIgMjc5LjUzNTE1NiBMIDM0NS4wNzAzMTIgOTUuNDY0ODQ0IFoiLz48cGF0aCBmaWxsPSIjOTRkZmQ4IiBkPSJNIDE4Ny41NzAzMTIgMTkwLjkzMzU5NCBMIDMwLjA3MDMxMiA5NS40NjQ4NDQgTCAxODcuNTcwMzEyIDAgTCAzNDUuMDcwMzEyIDk1LjQ2NDg0NCBaIi8+PC9zdmc+)](https://taskfile.dev/)
 
 **Django Google Structured Logger** is a Django middleware designed to capture and log details from incoming requests and outgoing responses. It offers features to mask sensitive data, set default fields for Google Cloud Logging, and structure logs in a detailed and organized manner.
 
@@ -32,27 +36,76 @@ pip install django-google-structured-logger
 
 #### Configuration
 
-1. Add `GoogleFormatter` to your Django's `LOGGING` setting.
-   Example:
+1. Add a formatter to your Django's `LOGGING` setting.
+
+   **For standard JSON logging:**
    ```python
     LOGGING = {
         "version": 1,
         "disable_existing_loggers": False,
         "formatters": {
             "json": {
-                "()": "django_google_structured_logger.formatter.GoogleFormatter",
+                "()": "django_google_structured_logger.formatter.StandardJSONFormatter",
             },
         },
         "handlers": {
             "console": {
                 "level": "INFO",
                 "class": "logging.StreamHandler",
             },
-            "google-json-handler": {
+            "json-handler": {
                 "class": "logging.StreamHandler",
                 "formatter": "json",
             },
         },
+        "root": {
+            "handlers": [env.str("DJANGO_LOG_HANDLER", "json-handler")],
+            "level": env.str("ROOT_LOG_LEVEL", "INFO"),
+        },
+        "loggers": {
+            "()": {
+                "handlers": [env.str("DJANGO_LOG_HANDLER", "json-handler")],
+                "level": env.str("DJANGO_LOG_LEVEL", "INFO"),
+            },
+            "django": {
+                "handlers": [env.str("DJANGO_LOG_HANDLER", "json-handler")],
+                "level": env.str("DJANGO_LOG_LEVEL", "INFO"),
+                "propagate": False,
+            },
+            "django.server": {
+                "handlers": [env.str("DJANGO_LOG_HANDLER", "json-handler")],
+                "level": env.str("DJANGO_SERVER_LEVEL", "ERROR"),
+                "propagate": False,
+            },
+            "django.request": {
+                "handlers": [env.str("DJANGO_LOG_HANDLER", "json-handler")],
+                "level": env.str("DJANGO_REQUEST_LEVEL", "ERROR"),
+                "propagate": False,
+            },
+        },
+    }
+   ```
+
+   **For Google Cloud Logging integration:**
+   ```python
+    LOGGING = {
+        "version": 1,
+        "disable_existing_loggers": False,
+        "formatters": {
+            "google": {
+                "()": "django_google_structured_logger.formatter.GoogleCloudFormatter",
+            },
+        },
+        "handlers": {
+            "console": {
+                "level": "INFO",
+                "class": "logging.StreamHandler",
+            },
+            "google-json-handler": {
+                "class": "logging.StreamHandler",
+                "formatter": "google",
+            },
+        },
         "root": {
             "handlers": [env.str("DJANGO_LOG_HANDLER", "google-json-handler")],
             "level": env.str("ROOT_LOG_LEVEL", "INFO"),
@@ -80,7 +133,17 @@ pip install django-google-structured-logger
         },
     }
    ```
-2. Add `SetRequestToLoggerMiddleware` to your Django's `MIDDLEWARE` setting.
+
+   Alternatively, you can configure the formatter class via Django settings:
+   ```python
+   # For standard JSON logging (default)
+   LOG_FORMATTER_CLASS = "django_google_structured_logger.formatter.StandardJSONFormatter"
+
+   # For Google Cloud Logging
+   LOG_FORMATTER_CLASS = "django_google_structured_logger.formatter.GoogleCloudFormatter"
+   ```
+
+2. Add middleware to your Django's `MIDDLEWARE` setting.
 
     Django middleware:
     ```python
@@ -104,13 +167,15 @@ pip install django-google-structured-logger
 
 ### Key Components:
 
-#### 1. middleware.py
+#### 1. middlewares.py
 
-- **SetRequestToLoggerMiddleware**: This class contains methods to process incoming requests and outgoing responses and then log them. It supports features like abridging lengthy data and masking sensitive information.
+- **SetUserContextMiddleware**: Sets user context information for logging throughout the request lifecycle.
+- **LogRequestAndResponseMiddleware**: Processes incoming requests and outgoing responses and logs them. It supports features like abridging lengthy data and masking sensitive information.
 
 #### 2. formatter.py
 
-- **GoogleFormatter**: Extends `jsonlogger.JsonFormatter` to format logs specifically for Google Cloud Logging. It sets default fields such as severity, labels, operation, and source location based on Google's logging standards.
+- **StandardJSONFormatter**: A universal JSON log formatter that creates structured logs with fields like severity, source_location, labels, operation, and http_request/http_response. Suitable for any logging system that accepts JSON format.
+- **GoogleCloudFormatter**: Extends `StandardJSONFormatter` to format logs specifically for Google Cloud Logging. It remaps standard fields to Google Cloud's specific field names (e.g., `logging.googleapis.com/sourceLocation`) and adds trace correlation support.
 
 #### 3. settings.py
 
@@ -121,11 +186,12 @@ pip install django-google-structured-logger
 
 These are the settings that can be customized for the middleware:
 
+- `LOG_FORMATTER_CLASS`: Formatter class to use. Default is `"django_google_structured_logger.formatter.StandardJSONFormatter"`.
 - `LOG_MAX_STR_LEN`: Maximum string length before data is abridged. Default is `200`.
 - `LOG_MAX_LIST_LEN`: Maximum list length before data is abridged. Default is `10`.
 - `LOG_EXCLUDED_ENDPOINTS`: List of endpoints to exclude from logging. Default is an `empty list`.
 - `LOG_SENSITIVE_KEYS`: Regex patterns for keys which contain sensitive data. Defaults `DEFAULT_SENSITIVE_KEYS`.
-- `LOG_MASK_STYLE`: Style for masking sensitive data. Default is `"partially"`.
+- `LOG_MASK_STYLE`: Style for masking sensitive data. Default is `"partial"`.
 - `LOG_MIDDLEWARE_ENABLED`: Enable or disable the logging middleware. Default is `True`.
 - `LOG_EXCLUDED_HEADERS`: List of request headers to exclude from logging. Defaults `DEFAULT_SENSITIVE_HEADERS`.
 - `LOG_USER_ID_FIELD`: Field name for user ID. Default is `"id"`.
@@ -143,11 +209,11 @@ Note:
   ```
   will be logged as structured data in the `jsonPayload` field in Google Cloud Logging.
   Any data passed to extra kwargs will not be abridged or masked.
-- `extra` kwargs passed to logger may override any default fields set by `GoogleFormatter`.
+- `extra` kwargs passed to logger may override any default fields set by the formatters.
 
 
 ### Conclusion:
 
-**SetRequestToLoggerMiddleware** is a comprehensive solution for those seeking enhanced logging capabilities in their Django projects, with particular attention to sensitive data protection and compatibility with Google Cloud Logging.
+**Django Google Structured Logger** is a comprehensive solution for those seeking enhanced logging capabilities in their Django projects, with particular attention to sensitive data protection and compatibility with Google Cloud Logging.
 
 To get started, integrate the provided middleware, formatter, and settings into your Django project, customize as needed, and enjoy advanced logging capabilities!

Taskfile.yaml

@@ -0,0 +1,105 @@
+# https://taskfile.dev
+
+version: "3"
+
+vars:
+  GITHUB_REPOSITORY_OWNER: muehlemann-popp
+  GITHUB_REPOSITORY: django-google-structured-logger
+  GIT_SHA:
+    sh: git rev-parse --short=8 HEAD
+  CURRENT_DATETIME:
+    sh: date +"%Y-%m-%d_%H-%M-%S"
+  DIR_SRC: ./django_google_structured_logger
+  DIR_TESTS: ./tests
+
+tasks:
+  default:
+    desc: Display list of all available tasks
+    cmds:
+      - task --list-all
+    silent: true
+
+  # Local environment
+
+  poetry:install:
+    desc: Install dependencies without changes in lock file
+    cmds:
+      - poetry install -vv --all-groups
+
+  poetry:install:ci:
+    desc: Install dependencies like in CI pipeline
+    cmds:
+      - poetry sync -vv --without=dev --no-cache --no-interaction
+
+  # Python Specific
+
+  py:format:src:
+    desc: Format python files in src directory
+    dir: "{{.DIR_SRC}}"
+    cmds:
+      - poetry run ruff check --select I --fix
+      - poetry run ruff format
+
+  py:format:tests:
+    desc: Format python files in tests directory
+    dir: "{{.DIR_TESTS}}"
+    cmds:
+      - poetry run ruff check --select I --fix
+      - poetry run ruff format
+
+  py:format:all:
+    desc: Format all Python files in src and tests directories
+    cmds:
+      - task: py:format:src
+      - task: py:format:tests
+
+  checks:mypy:
+    desc: Run mypy check
+    cmds:
+      - poetry run mypy {{.DIR_SRC}}
+      - poetry run mypy {{.DIR_TESTS}}
+
+  checks:tests:
+    desc: Run tests
+    cmds:
+      - poetry run pytest
+
+  checks:ruff:
+    desc: Run ruff checks
+    cmds:
+      - poetry run ruff check --fix
+
+  checks:coverage:
+    desc: Run code coverage check
+    ignore_error: true
+    cmds:
+      - poetry run coverage run -m pytest -vv {{.CLI_ARGS}}
+
+  precommit:
+    desc: Run all checks
+    cmds:
+      - task: py:format:all
+      - task: checks:ruff
+      - task: checks:mypy
+      - task: checks:coverage
+
+  # GitHub CLI
+
+  github:repository:getid:
+    desc: Get repository id with GitHub CLI
+    cmds:
+      - "gh api -H 'Accept: application/vnd.github+json' repos/{{ .GITHUB_REPOSITORY_OWNER }}/{{ .CLI_ARGS | default .GITHUB_REPOSITORY_NAME }} | jq .id"
+
+  # git cli
+
+  git:diff:
+    desc: Prepare git diff for review
+    cmds:
+      - mkdir -p ./var
+      - git diff > ./var/git_diff
+
+  git:diff:main:
+    desc: Prepare comparison between current branch and main branch for review
+    cmds:
+      - mkdir -p ./var
+      - git diff main..HEAD > ./var/git_diff

django_google_structured_logger/apps.py

@@ -1,4 +1,4 @@
-from django.apps import AppConfig  # type: ignore
+from django.apps import AppConfig
 
 
 class DjangoMaterializedViewAppConfig(AppConfig):