Opentelemetry configuration

Initialise opentelemetry in the gunicorn post_fork hook for prod.
Includes an `OTEL_CONSOLE` env variable that can be used to
send traces to the console in local dev environments.

DEVELOPERS.md

@@ -98,6 +98,11 @@ And then update with the latest upstream assets:
 just assets/update
 ```
 
+## Opentelemetry
+
+To log opentelemetry traces to the console in local environments,
+set the `OTEL_EXPORTER_CONSOLE` environment variable in your `.env` file. 
+
 ## Testing
 
 ### Test categories

dotenv-sample

@@ -12,11 +12,24 @@ AIRLOCK_REQUEST_DIR=releases/
 # in AIRLOCK_API_TOKEN is set)
 AIRLOCK_DEV_USERS_FILE="dev_users.json"
 
-
 # Note: in development, these settings will be automatically updated by
 # `just job-server/{run,stop}
 # In production you can find this token for an existing backend in the
 # Job Server staff area at https://jobs.opensafely.org/staff/backends/ ,
 # and you can use the default value for the endpoint.
 AIRLOCK_API_TOKEN=
 AIRLOCK_API_ENDPOINT="https://localhost:9000/api/v2"
+
+# Opentelemetry
+export OTEL_EXPORTER_OTLP_ENDPOINT="https://api.honeycomb.io"
+# key will be for specific prod/test/dev honeycomb environment
+export OTEL_EXPORTER_OTLP_HEADERS="x-honeycomb-team=VALIDKEY1234"
+# this is the dataset that telemetry will go into in honeycomb
+export OTEL_SERVICE_NAME="airlock"
+
+# Set to True to log opentelemetry traces to the console in local env
+export OTEL_EXPORTER_CONSOLE=True
+
+# OTEL requires these, it needs to access settings before other stuff has initialised
+DJANGO_SETTINGS_MODULE="airlock.settings"
+PYTHONPATH="/usr/local/bin/python"

gunicorn.conf.py

@@ -1,6 +1,17 @@
+import tracing
+
+
 # workers
 workers = 4
 
 # listen
 port = 8000
 bind = "0.0.0.0"
+
+
+# Because of Gunicorn's pre-fork web server model, we need to initialise opentelemetry
+# in gunicorn's post_fork method in order to instrument our application process, see:
+# https://opentelemetry-python.readthedocs.io/en/latest/examples/fork-process-model/README.html
+def post_fork(server, worker):
+    server.log.info("Worker spawned (pid: %s)", worker.pid)
+    tracing.setup_default_tracing()

manage.py

@@ -4,10 +4,14 @@
 import os
 import sys
 
+import tracing
+
 
 def main():
     """Run administrative tasks."""
     os.environ.setdefault("DJANGO_SETTINGS_MODULE", "airlock.settings")
+    tracing.setup_default_tracing()
+
     try:
         from django.core.management import execute_from_command_line
     except ImportError as exc:

tracing.py

@@ -0,0 +1,48 @@
+import os
+
+from opentelemetry import trace
+from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.trace.export import BatchSpanProcessor, ConsoleSpanExporter
+
+
+def get_provider():
+    provider = TracerProvider()
+    trace.set_tracer_provider(provider)
+    return provider
+
+
+def add_exporter(provider, exporter, processor=BatchSpanProcessor):
+    """Utility method to add an exporter.
+
+    We use the BatchSpanProcessor by default, which is the default for
+    production. This is asynchronous, and queues and retries sending telemetry.
+
+    In testing, we instead use SimpleSpanProcessor, which is synchronous and
+    easy to inspect the output of within a test.
+    """
+    # Note: BatchSpanProcessor is configured via env vars:
+    # https://opentelemetry-python.readthedocs.io/en/latest/sdk/trace.export.html#opentelemetry.sdk.trace.export.BatchSpanProcessor
+    provider.add_span_processor(processor(exporter))
+
+
+def setup_default_tracing():
+    provider = get_provider()
+
+    """Inspect environment variables and set up exporters accordingly."""
+    if "OTEL_EXPORTER_OTLP_HEADERS" in os.environ:
+        if "OTEL_SERVICE_NAME" not in os.environ:
+            raise Exception(
+                "OTEL_EXPORTER_OTLP_HEADERS is configured, but missing OTEL_SERVICE_NAME"
+            )
+        if "OTEL_EXPORTER_OTLP_ENDPOINT" not in os.environ:
+            os.environ["OTEL_EXPORTER_OTLP_ENDPOINT"] = "https://api.honeycomb.io"
+
+        add_exporter(provider, OTLPSpanExporter())
+
+    if "OTEL_EXPORTER_CONSOLE" in os.environ:
+        add_exporter(provider, ConsoleSpanExporter())
+
+    from opentelemetry.instrumentation.auto_instrumentation import (  # noqa: F401
+        sitecustomize,
+    )