Setup OpenTelemetry with Jaeger, Prometheus & Grafana.

.gitignore

@@ -0,0 +1,2 @@
+__pycache__/
+cloud-telemetry

README.md

@@ -14,3 +14,32 @@ A simple telemetry sidecar deployment might be a reasonable aim for a short proj
 * [Project Board](https://github.com/orgs/elixir-cloud-aai/projects/21)
 * Communication: ELIXIR Cloud Slack Channel, Topic #telemetry
 
+## Steps to run:
+
+### Create virtual environment
+`python3 -m venv cloud-telemetry`
+
+### Activate venv
+`source cloud-telemetry/bin/activate`
+
+### Install python requirements
+`pip install -r requirements.txt`
+
+### Run starlette application
+`uvicorn main:app`
+
+### Initiate components
+`docker compose up`
+<!-- ### Run open-telemetry collector
+`docker run -p 4317:4317 -p 4318:4318 --rm -v $(pwd)/collector-config.yaml:/etc/otelcol/config.yaml otel/opentelemetry-collector`
+
+OR
+
+### with jaeger collector:
+`docker run --name jaeger \ 
+  -e COLLECTOR_OTLP_ENABLED=true \
+  -p 16686:16686 \
+  -p 4317:4317 \
+  -p 4318:4318 \
+  jaegertracing/all-in-one:latest
+` -->

datasource.yml

@@ -0,0 +1,8 @@
+apiVersion: 1
+datasources:
+- name: Prometheus
+  type: prometheus
+  url: http://prometheus:9090
+  isDefault: true
+  access: proxy
+  editable: true

docker-compose.yml

@@ -0,0 +1,60 @@
+version: "3.5"
+services:
+  jaeger:
+    networks:
+      - backend
+    image: jaegertracing/all-in-one:${JAEGER_IMAGE_TAG:-latest}
+    volumes:
+      - "./jaeger-ui.json:/etc/jaeger/jaeger-ui.json"
+    command: --query.ui-config /etc/jaeger/jaeger-ui.json
+    environment:
+      - METRICS_STORAGE_TYPE=prometheus
+      - PROMETHEUS_SERVER_URL=http://prometheus:9090
+      - PROMETHEUS_QUERY_SUPPORT_SPANMETRICS_CONNECTOR=${PROMETHEUS_QUERY_SUPPORT_SPANMETRICS_CONNECTOR:-true}
+      - PROMETHEUS_QUERY_NAMESPACE=${PROMETHEUS_QUERY_NAMESPACE:-}
+      - PROMETHEUS_QUERY_DURATION_UNIT=${PROMETHEUS_QUERY_DURATION_UNIT:-}
+      - PROMETHEUS_QUERY_NORMALIZE_CALLS=true
+      - PROMETHEUS_QUERY_NORMALIZE_DURATION=true
+    ports:
+      - "16686:16686"
+
+  otel_collector:
+    networks:
+      backend:
+        # This is the host name used in Prometheus scrape configuration.
+        aliases: [spm_metrics_source]
+    image: otel/opentelemetry-collector-contrib:${OTEL_IMAGE_TAG:-0.89.0}
+    volumes:
+      - ${OTEL_CONFIG_SRC:-./otel-collector-config-connector.yml}:/etc/otelcol/otel-collector-config.yml
+    command: --config /etc/otelcol/otel-collector-config.yml
+    depends_on:
+      - jaeger
+    ports:
+      - "8889:8889"
+      - "4318:4318"
+
+  prometheus:
+    networks:
+      - backend
+    image: prom/prometheus:latest
+    volumes:
+      - "./prometheus.yml:/etc/prometheus/prometheus.yml"
+    ports:
+      - "9090:9090"
+
+  # grafana:
+  #   networks:
+  #     - backend
+  #   image: grafana/grafana:latest
+  #   volumes:
+  #     - ./grafana.ini:/etc/grafana/grafana.ini
+  #     - ./datasource.yml:/etc/grafana/provisioning/datasources/datasource.yaml
+  #   environment:
+  #     - GF_AUTH_ANONYMOUS_ENABLED=true
+  #     - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin
+  #     - GF_AUTH_DISABLE_LOGIN_FORM=true
+  #   ports:
+  #     - "3000:3000"
+
+networks:
+  backend:

jaeger-ui.json

@@ -0,0 +1,8 @@
+{
+    "monitor": {
+        "menuEnabled": true
+    },
+    "dependencies": {
+        "menuEnabled": true
+    }
+}

main.py

@@ -0,0 +1,50 @@
+from starlette.applications import Starlette
+from starlette.responses import JSONResponse
+from starlette.routing import Route
+import asyncio
+import random
+from middleware import OpenTelemetryInstrumentor
+
+# Testing APIs
+
+async def home(request):
+    return JSONResponse("hi")
+
+async def process_data(request):
+    await asyncio.sleep(random.uniform(1, 3))
+    return JSONResponse({"status": "Data processed successfully"})
+
+async def generate_report(request):
+    await asyncio.sleep(random.uniform(2, 5))
+    return JSONResponse({"status": "Report generated successfully"})
+
+async def perform_analysis(request):
+    await asyncio.sleep(random.uniform(3, 7))
+    return JSONResponse({"status": "Analysis completed"})
+
+async def backup_database(request):
+    await asyncio.sleep(random.uniform(5, 10))
+    return JSONResponse({"status": "Database backup completed"})
+
+async def optimize_system(request):
+    await asyncio.sleep(random.uniform(4, 8))
+    return JSONResponse({"status": "System optimization completed"})
+
+async def send_notifications(request):
+    await asyncio.sleep(random.uniform(1, 4))
+    return JSONResponse({"status": "Notifications sent successfully"})
+
+# Temp Routes for testing
+app = Starlette(
+    routes=[
+        Route("/", home),
+        Route("/process-data", process_data),
+        Route("/generate-report", generate_report),
+        Route("/perform-analysis", perform_analysis),
+        Route("/backup-database", backup_database),
+        Route("/optimize-system", optimize_system),
+        Route("/send-notifications", send_notifications),
+    ]
+)
+
+OpenTelemetryInstrumentor.instrument_app(app, service_name="ga4gh")

middleware.py

@@ -0,0 +1,63 @@
+from opentelemetry.instrumentation.starlette import StarletteInstrumentor
+from starlette.applications import Starlette
+from opentelemetry.sdk.resources import SERVICE_NAME, Resource
+from opentelemetry import trace
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.trace.export import BatchSpanProcessor, ConsoleSpanExporter
+
+from opentelemetry import metrics
+from opentelemetry.sdk.metrics import MeterProvider
+from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader, ConsoleMetricExporter
+# from prometheus_client import start_http_server
+
+class OpenTelemetryInstrumentor():
+    """Instrument a Starlette application with OpenTelemetry."""
+
+    def instrument_app(
+        app: Starlette,
+        service_name: str = "ga4gh",
+    ):
+        resource = Resource(attributes={
+            SERVICE_NAME: service_name
+        })
+        # Traces : Console
+        # traceProvider = TracerProvider(resource=resource)
+        # processor = BatchSpanProcessor(ConsoleSpanExporter())
+        # traceProvider.add_span_processor(processor)
+        # trace.set_tracer_provider(traceProvider)
+
+        # Jaeger || otel col setup : Exposing OTLP traces
+        # more read here: https://medium.com/jaegertracing/introducing-native-support-for-opentelemetry-in-jaeger-eb661be8183c
+        from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
+        traceProvider = TracerProvider(resource=resource)
+        processor = BatchSpanProcessor(OTLPSpanExporter())
+        traceProvider.add_span_processor(processor)
+        trace.set_tracer_provider(traceProvider)
+
+        # prometheus || otel col setup : Exposing OTLP metrics
+
+        from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
+        reader = PeriodicExportingMetricReader(OTLPMetricExporter())
+        meterProvider = MeterProvider(resource=resource, metric_readers=[reader])
+        metrics.set_meter_provider(meterProvider)
+
+        # Prometheus : Console
+
+        # reader = PeriodicExportingMetricReader(ConsoleMetricExporter())
+        # meterProvider = MeterProvider(resource=resource, metric_readers=[reader])
+        # metrics.set_meter_provider(meterProvider)
+        # Start Prometheus client
+
+        # Prometheus : HTTP
+
+        # start_http_server(port=9464, addr="localhost")
+        # Initialize PrometheusMetricReader which pulls metrics from the SDK
+        # on-demand to respond to scrape requests
+
+        # reader = PrometheusMetricReader()
+        # meterProvider = MeterProvider(resource=resource, metric_readers=[reader])
+        # metrics.set_meter_provider(meterProvider)
+
+        StarletteInstrumentor().instrument_app(app)
+
+

otel-collector-config-connector.yml

@@ -0,0 +1,37 @@
+receivers:
+  otlp:
+    protocols:
+      grpc:
+      http:
+        endpoint: 0.0.0.0:4318
+
+exporters:
+  debug:
+    verbosity: detailed
+
+  prometheus:
+    endpoint: "0.0.0.0:8889"
+
+  otlp:
+    endpoint: jaeger:4317
+    tls:
+      insecure: true
+
+connectors:
+  spanmetrics:
+
+processors:
+  batch:
+
+service:
+  pipelines:
+    traces:
+      receivers: [otlp]
+      processors: [batch]
+      exporters: [spanmetrics, otlp]
+
+    # The metrics pipeline receives generated span metrics from 'spanmetrics' connector
+    # and pushes to Prometheus exporter, which makes them available for scraping on :8889.
+    metrics/spanmetrics:
+      receivers: [spanmetrics]
+      exporters: [prometheus]

prometheus.yml

@@ -0,0 +1,9 @@
+global:
+  scrape_interval:     15s # Set the scrape interval to every 15 seconds. Default is every 1 minute.
+  evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute.
+  # scrape_timeout is set to the global default (10s).
+
+scrape_configs:
+  - job_name: aggregated-trace-metrics
+    static_configs:
+    - targets: ['spm_metrics_source:8889']

requirements.txt

@@ -0,0 +1,6 @@
+opentelemetry-api
+opentelemetry-sdk
+opentelemetry-exporter-otlp-proto-http
+opentelemetry-instrumentation-starlette
+starlette
+uvicorn