diff --git a/DEVELOPERS.md b/DEVELOPERS.md index 7691e211..d873b7a7 100644 --- a/DEVELOPERS.md +++ b/DEVELOPERS.md @@ -98,6 +98,11 @@ And then update with the latest upstream assets: just assets/update ``` +## Opentelemetry + +To log opentelemetry traces to the console in local environments, +set the `OTEL_EXPORTER_CONSOLE` environment variable in your `.env` file. + ## Testing ### Test categories diff --git a/dotenv-sample b/dotenv-sample index 7fd6f1d0..6f0c1dc9 100644 --- a/dotenv-sample +++ b/dotenv-sample @@ -12,7 +12,6 @@ AIRLOCK_REQUEST_DIR=releases/ # in AIRLOCK_API_TOKEN is set) AIRLOCK_DEV_USERS_FILE="dev_users.json" - # Note: in development, these settings will be automatically updated by # `just job-server/{run,stop} # In production you can find this token for an existing backend in the @@ -20,3 +19,17 @@ AIRLOCK_DEV_USERS_FILE="dev_users.json" # and you can use the default value for the endpoint. AIRLOCK_API_TOKEN= AIRLOCK_API_ENDPOINT="https://localhost:9000/api/v2" + +# Opentelemetry +export OTEL_EXPORTER_OTLP_ENDPOINT="https://api.honeycomb.io" +# key will be for specific prod/test/dev honeycomb environment +export OTEL_EXPORTER_OTLP_HEADERS="x-honeycomb-team=VALIDKEY1234" +# this is the dataset that telemetry will go into in honeycomb +export OTEL_SERVICE_NAME="airlock" + +# Set to True to log opentelemetry traces to the console in local env +export OTEL_EXPORTER_CONSOLE=True + +# OTEL requires these, it needs to access settings before other stuff has initialised +DJANGO_SETTINGS_MODULE="airlock.settings" +PYTHONPATH="/usr/local/bin/python" diff --git a/gunicorn.conf.py b/gunicorn.conf.py index f1a19cc6..f47e8194 100644 --- a/gunicorn.conf.py +++ b/gunicorn.conf.py @@ -1,6 +1,17 @@ +import tracing + + # workers workers = 4 # listen port = 8000 bind = "0.0.0.0" + + +# Because of Gunicorn's pre-fork web server model, we need to initialise opentelemetry +# in gunicorn's post_fork method in order to instrument our application process, see: +# https://opentelemetry-python.readthedocs.io/en/latest/examples/fork-process-model/README.html +def post_fork(server, worker): + server.log.info("Worker spawned (pid: %s)", worker.pid) + tracing.setup_default_tracing() diff --git a/manage.py b/manage.py index d7a800d9..94aa30c8 100755 --- a/manage.py +++ b/manage.py @@ -4,10 +4,14 @@ import os import sys +import tracing + def main(): """Run administrative tasks.""" os.environ.setdefault("DJANGO_SETTINGS_MODULE", "airlock.settings") + tracing.setup_default_tracing() + try: from django.core.management import execute_from_command_line except ImportError as exc: diff --git a/tracing.py b/tracing.py new file mode 100644 index 00000000..85536916 --- /dev/null +++ b/tracing.py @@ -0,0 +1,48 @@ +import os + +from opentelemetry import trace +from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import BatchSpanProcessor, ConsoleSpanExporter + + +def get_provider(): + provider = TracerProvider() + trace.set_tracer_provider(provider) + return provider + + +def add_exporter(provider, exporter, processor=BatchSpanProcessor): + """Utility method to add an exporter. + + We use the BatchSpanProcessor by default, which is the default for + production. This is asynchronous, and queues and retries sending telemetry. + + In testing, we instead use SimpleSpanProcessor, which is synchronous and + easy to inspect the output of within a test. + """ + # Note: BatchSpanProcessor is configured via env vars: + # https://opentelemetry-python.readthedocs.io/en/latest/sdk/trace.export.html#opentelemetry.sdk.trace.export.BatchSpanProcessor + provider.add_span_processor(processor(exporter)) + + +def setup_default_tracing(): + provider = get_provider() + + """Inspect environment variables and set up exporters accordingly.""" + if "OTEL_EXPORTER_OTLP_HEADERS" in os.environ: + if "OTEL_SERVICE_NAME" not in os.environ: + raise Exception( + "OTEL_EXPORTER_OTLP_HEADERS is configured, but missing OTEL_SERVICE_NAME" + ) + if "OTEL_EXPORTER_OTLP_ENDPOINT" not in os.environ: + os.environ["OTEL_EXPORTER_OTLP_ENDPOINT"] = "https://api.honeycomb.io" + + add_exporter(provider, OTLPSpanExporter()) + + if "OTEL_EXPORTER_CONSOLE" in os.environ: + add_exporter(provider, ConsoleSpanExporter()) + + from opentelemetry.instrumentation.auto_instrumentation import ( # noqa: F401 + sitecustomize, + )