From 4d6b710491ec189be23911390e3966c4e4e32817 Mon Sep 17 00:00:00 2001 From: Reiley Yang Date: Mon, 20 May 2024 22:47:20 -0700 Subject: [PATCH 01/10] Improve exemplars tutorial --- docs/metrics/exemplars/Program.cs | 68 ++++++++ docs/metrics/exemplars/README.md | 164 +++++++++++------- docs/metrics/exemplars/docker-compose.yaml | 50 ------ docs/metrics/exemplars/exemplars.csproj | 5 + .../exemplars/grafana-datasources.yaml | 33 ---- docs/metrics/exemplars/otel-collector.yaml | 30 ---- docs/metrics/exemplars/prometheus.yaml | 8 - docs/metrics/exemplars/tempo.yaml | 17 -- 8 files changed, 177 insertions(+), 198 deletions(-) create mode 100644 docs/metrics/exemplars/Program.cs delete mode 100644 docs/metrics/exemplars/docker-compose.yaml create mode 100644 docs/metrics/exemplars/exemplars.csproj delete mode 100644 docs/metrics/exemplars/grafana-datasources.yaml delete mode 100644 docs/metrics/exemplars/otel-collector.yaml delete mode 100644 docs/metrics/exemplars/prometheus.yaml delete mode 100644 docs/metrics/exemplars/tempo.yaml diff --git a/docs/metrics/exemplars/Program.cs b/docs/metrics/exemplars/Program.cs new file mode 100644 index 00000000000..a55396a7a60 --- /dev/null +++ b/docs/metrics/exemplars/Program.cs @@ -0,0 +1,68 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +using System.Diagnostics; +using System.Diagnostics.Metrics; +using OpenTelemetry; +using OpenTelemetry.Exporter; +using OpenTelemetry.Metrics; +using OpenTelemetry.Trace; + +public class Program +{ + private static readonly ActivitySource MyActivitySource = new("OpenTelemetry.Demo.Exemplar"); + private static readonly Meter MyMeter = new("OpenTelemetry.Demo.Exemplar"); + private static readonly Counter MyFruitCounter = MyMeter.CreateCounter("MyFruitCounter"); + + public static void Main() + { + using var tracerProvider = Sdk.CreateTracerProviderBuilder() + .AddSource("OpenTelemetry.Demo.Exemplar") + .AddOtlpExporter() + .Build(); + + using var meterProvider = Sdk.CreateMeterProviderBuilder() + .AddMeter("OpenTelemetry.Demo.Exemplar") + .SetExemplarFilter(ExemplarFilterType.TraceBased) + .AddOtlpExporter((exporterOptions, metricReaderOptions) => + { + exporterOptions.Endpoint = new Uri("http://localhost:9090/api/v1/otlp/v1/metrics"); + exporterOptions.Protocol = OtlpExportProtocol.HttpProtobuf; + metricReaderOptions.PeriodicExportingMetricReaderOptions.ExportIntervalMilliseconds = 1000; + }) + .Build(); + + Console.WriteLine("Press any key to exit"); + + while (!Console.KeyAvailable) + { + using (var parent = MyActivitySource.StartActivity("Parent Operation")) + { + parent?.SetTag("key1", "value1"); + parent?.SetTag("key2", "value2"); + + using (var child = MyActivitySource.StartActivity("Child Operation")) + { + child?.SetTag("key3", "value3"); + child?.SetTag("key4", "value4"); + + MyFruitCounter.Add(1, new("name", "apple"), new("color", "red")); + MyFruitCounter.Add(1, new("name", "lemon"), new("color", "yellow")); + } + } + + MyFruitCounter.Add(2, new("name", "apple"), new("color", "green")); + MyFruitCounter.Add(4, new("name", "lemon"), new("color", "yellow")); + + Thread.Sleep(300); + } + + // Dispose meter provider before the application ends. + // This will flush the remaining metrics and shutdown the metrics pipeline. + meterProvider.Dispose(); + + // Dispose tracer provider before the application ends. + // This will flush the remaining spans and shutdown the tracing pipeline. + tracerProvider.Dispose(); + } +} diff --git a/docs/metrics/exemplars/README.md b/docs/metrics/exemplars/README.md index a95c817bd93..52602076e84 100644 --- a/docs/metrics/exemplars/README.md +++ b/docs/metrics/exemplars/README.md @@ -1,99 +1,143 @@ # Using Exemplars in OpenTelemetry .NET +- [Install and run Jaeger](#install-and-run-jaeger) +- [Install and run Prometheus](#install-and-run-prometheus) +- [Install and configure Grafana](#install-and-configure-grafana) +- [Export metrics and traces from the + application](#export-metrics-and-traces-from-the-application) +- [Use exemplars to navigate from metrics to + traces](#use-exemplars-to-navigate-from-metrics-to-traces) +- [Learn more](#learn-more) + Exemplars are example data points for aggregated data. They provide specific context to otherwise general aggregations. One common use case is to gain ability to correlate metrics to traces (and logs). While OpenTelemetry .NET supports Exemplars, it is only useful if the telemetry backend also supports the -capabilities. This tutorial uses well known open source backends to demonstrate -the concept. The following are the components involved: +capabilities. This tutorial uses well known open-source backends to demonstrate +the concept. The following components are involved: + +- [Program.cs](./Program.cs) - this application is instrumented with + OpenTelemetry, it sends metrics to Prometheus, and traces to Jaeger. +- [Prometheus](#install-and-run-prometheus) - Prometheus is used as the metrics + backend. +- [Jaeger](#install-and-run-jaeger) - Jaeger is used as the distributed tracing + backend. +- [Grafana](#install-and-run-grafana) - UI to query metrics from + Prometheus, traces from Jaeger, and to navigate between metrics and traces + using Exemplars. -* Test App - We use existing example app from the repo. This app is already -instrumented with OpenTelemetry for logs, metrics and traces, and is configured -to export them to the configured OTLP end point. -* OpenTelemetry Collector - An instance of collector is run, which receives -telemetry from the above app using OTLP. The collector then exports metrics to -Prometheus, traces to Tempo. -* Prometheus - Prometheus is used as the Metric backend. -* Tempo - Tempo is used as the Tracing backend. -* Grafana - UI to query metrics from Prometheus, traces from Tempo, and to - navigate between metrics and traces using Exemplar. +## Install and run Jaeger -All these components except the test app require additional configuration to -enable Exemplar feature. To make it easy for users, these components are -pre-configured to enable Exemplars, and a docker compose file is provided to - spun them all up, in the required configurations. +Download the [latest binary distribution +archive](https://www.jaegertracing.io/download/) of Jaeger. -## Pre-requisite +After finished downloading, extract it to a local location that's easy to +access. Run the `jaeger-all-in-one(.exe)` executable: -Install docker: +```sh +./jaeger-all-in-one --collector.otlp.enabled +``` -## Setup +## Install and run Prometheus -As mentioned in the intro, this tutorial uses OTel Collector, Prometheus, Tempo, -and Grafana, and they must be up and running before proceeding. The following -spins all of them with the correct configurations to support Exemplars. +Follow the [first steps](https://prometheus.io/docs/introduction/first_steps/) +to download the [latest release](https://prometheus.io/download/) of Prometheus. -Navigate to current directory and run the following: +After finished downloading, extract it to a local location that's easy to +access. Run the `prometheus(.exe)` server executable with feature flags +[exemplars +storage](https://prometheus.io/docs/prometheus/latest/feature_flags/#exemplars-storage) +and +[otlp-receiver](https://prometheus.io/docs/prometheus/latest/feature_flags/#otlp-receiver) +enabled: ```sh -docker compose up -d +./prometheus --enable-feature=exemplar-storage --enable-feature=otlp-write-receiver ``` -If the above step succeeds, all dependencies would be spun up and ready now. To -test, navigate to Grafana running at: `http://localhost:3000/`. +## Install and configure Grafana + +Follow the operating system specific instructions to [download and install +Grafana](https://grafana.com/docs/grafana/latest/setup-grafana/installation/#supported-operating-systems). + +After installation, start the standalone Grafana server (`grafana-server.exe` or +`./bin/grafana-server`, depending on the operating system). Then, use a +[supported web +browser](https://grafana.com/docs/grafana/latest/setup-grafana/installation/#supported-web-browsers) +to navigate to [http://localhost:3000/](http://localhost:3000/). + +Follow the instructions in the Grafana getting started +[doc](https://grafana.com/docs/grafana/latest/getting-started/getting-started/#step-2-log-in) +to log in. + +After successfully logging in, hover on the Configuration icon +on the panel at the left hand side, and click on Plugins. -## Run test app +Find and click on the Jaeger plugin. Next click on `Create a Jaeger data source` +button. Make the following changes: -Now that the required dependencies are ready, lets run the demo app. -This tutorial is using the existing ASP.NET Core app from the repo. +1. Set "URL" to `http://localhost:16686/`. +2. At the bottom of the page click `Save & test` to ensure the data source is + working. -Navigate to [Example Asp.Net Core App](../../../examples/AspNetCore/Program.cs) -directory and run the following command: +![Add Jaeger data +source](https://github.com/open-telemetry/opentelemetry-dotnet/assets/17327289/8356dc1d-dad2-4c82-9936-9a84b51d12fa) + +Find and click on the Prometheus plugin. Next click on +`Create a Prometheus data source` button. Make the following changes: + +1. Set "URL" to `http://localhost:9090`. +2. Under the "Exemplars" section, enable "Internal link", set "Data source" to + `Jaeger`, and set "Label name" to `trace_id`. +3. At the bottom of the page click `Save & test` to ensure the data source is + working. + +![Add Prometheus data +source](https://github.com/open-telemetry/opentelemetry-dotnet/assets/17327289/a137c4ac-dfd7-4d24-8811-208f66e67e37) + +## Export metrics and traces from the application + +Create a new console application and run it: ```sh +dotnet new console --output exemplars +cd exemplars dotnet run ``` -Once the application is running, navigate to -[http://localhost:5000/weatherforecast]("http://localhost:5000/weatherforecast") -from a web browser. You may use the following Powershell script to generate load -to the application. - -```powershell -while($true) -{ - Invoke-WebRequest http://localhost:5000/weatherforecast - Start-Sleep -Milliseconds 500 -} -``` +Add reference to [OTLP +Exporter](../../../src/OpenTelemetry.Exporter.OpenTelemetryProtocol/README.md): -## Use Exemplars to navigate from Metrics to Traces +```sh +dotnet add package OpenTelemetry.Exporter.OpenTelemetryProtocol +``` -The application sends metrics (with exemplars), and traces to the OTel -Collector, which export metrics and traces to Prometheus and Tempo -respectively. +Now copy the code from [Program.cs](./Program.cs) and run the application again. +The application will start sending metrics (with exemplars) to Prometheus and +traces to Jaeger. -Please wait for 2 minutes before continuing so that enough data is generated -and exported. +## Use exemplars to navigate from metrics to traces Open Grafana, select Explore, and select Prometheus as the source. Select the -metric named "http_server_duration_bucket", and plot the chart. Toggle on the -"Exemplar" option from the UI and hit refresh. +metric named "MyFruitCounter_total", and plot the chart. Toggle on the +"Exemplars" option from the UI and hit refresh. -![Enable Exemplar](https://user-images.githubusercontent.com/16979322/218627781-9886f837-11ae-4d52-94d3-f1821503209c.png) +![Enable Exemplars](https://github.com/open-telemetry/opentelemetry-dotnet/assets/17327289/fc11fd4b-5601-4728-acdd-5129d75fbc04) The Exemplars appear as special "diamond shaped dots" along with the metric -charts in the UI. Select any Exemplar to see the exemplar data, which includes +charts in the UI. Select any exemplar to see the exemplar data, which includes the timestamp when the measurement was recorded, the raw value, and trace context when the recording was done. The "trace_id" enables jumping to the -tracing backed (tempo). Click on the "Query with Tempo" button next to the -"trace_id" field to open the corresponding `Trace` in Tempo. +tracing backed (Jaeger in this case). Click on the "Query with Jaeger" button +next to the "trace_id" field to open the corresponding trace in Jaeger. ![Navigate to trace with exemplar](https://user-images.githubusercontent.com/16979322/218629999-1d1cd6ba-2385-4683-975a-d4797df8361a.png) -## References +## Learn more -* [Exemplar specification](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/metrics/sdk.md#exemplar) -* [Exemplars in Prometheus](https://prometheus.io/docs/prometheus/latest/feature_flags/#exemplars-storage) -* [Exemplars in Grafana](https://grafana.com/docs/grafana/latest/fundamentals/exemplars/) -* [Tempo](https://github.com/grafana/tempo) +- [What is Prometheus?](https://prometheus.io/docs/introduction/overview/) +- [Prometheus now supports OpenTelemetry + Metrics](https://horovits.medium.com/prometheus-now-supports-opentelemetry-metrics-83f85878e46a) +- [Jaeger Tracing](https://www.jaegertracing.io/) +- [Grafana support for + Prometheus](https://prometheus.io/docs/visualization/grafana/#creating-a-prometheus-graph) diff --git a/docs/metrics/exemplars/docker-compose.yaml b/docs/metrics/exemplars/docker-compose.yaml deleted file mode 100644 index c8cc94fa4b1..00000000000 --- a/docs/metrics/exemplars/docker-compose.yaml +++ /dev/null @@ -1,50 +0,0 @@ -version: "3" -services: - - # OTEL Collector to receive logs, metrics and traces from the application - otel-collector: - image: otel/opentelemetry-collector:0.70.0 - command: [ "--config=/etc/otel-collector.yaml" ] - volumes: - - ./otel-collector.yaml:/etc/otel-collector.yaml - ports: - - "4317:4317" - - "4318:4318" - - "9201:9201" - - # Exports Traces to Tempo - tempo: - image: grafana/tempo:latest - command: [ "-config.file=/etc/tempo.yaml" ] - volumes: - - ./tempo.yaml:/etc/tempo.yaml - - ./tempo-data:/tmp/tempo - ports: - - "3200" # tempo - - "4317" # otlp grpc - - "4318" # otlp http - -# Exports Metrics to Prometheus - prometheus: - image: prom/prometheus:latest - command: - - --config.file=/etc/prometheus.yaml - - --web.enable-remote-write-receiver - - --enable-feature=exemplar-storage - volumes: - - ./prometheus.yaml:/etc/prometheus.yaml - ports: - - "9090:9090" - -# UI to query traces and metrics - grafana: - image: grafana/grafana:9.3.2 - volumes: - - ./grafana-datasources.yaml:/etc/grafana/provisioning/datasources/datasources.yaml - environment: - - GF_AUTH_ANONYMOUS_ENABLED=true - - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin - - GF_AUTH_DISABLE_LOGIN_FORM=true - - GF_FEATURE_TOGGLES_ENABLE=traceqlEditor - ports: - - "3000:3000" diff --git a/docs/metrics/exemplars/exemplars.csproj b/docs/metrics/exemplars/exemplars.csproj new file mode 100644 index 00000000000..cce12eec60d --- /dev/null +++ b/docs/metrics/exemplars/exemplars.csproj @@ -0,0 +1,5 @@ + + + + + diff --git a/docs/metrics/exemplars/grafana-datasources.yaml b/docs/metrics/exemplars/grafana-datasources.yaml deleted file mode 100644 index 467975584ca..00000000000 --- a/docs/metrics/exemplars/grafana-datasources.yaml +++ /dev/null @@ -1,33 +0,0 @@ -apiVersion: 1 - -datasources: -- name: Prometheus - type: prometheus - uid: prometheus - access: proxy - orgId: 1 - url: http://prometheus:9090 - basicAuth: false - isDefault: true - version: 1 - editable: false - jsonData: - httpMethod: GET - exemplarTraceIdDestinations: - - name: trace_id - datasourceUid: Tempo -- name: Tempo - type: tempo - access: proxy - orgId: 1 - url: http://tempo:3200 - basicAuth: false - isDefault: false - version: 1 - editable: false - apiVersion: 1 - uid: tempo - jsonData: - httpMethod: GET - serviceMap: - datasourceUid: prometheus diff --git a/docs/metrics/exemplars/otel-collector.yaml b/docs/metrics/exemplars/otel-collector.yaml deleted file mode 100644 index bcf0cb5d6d2..00000000000 --- a/docs/metrics/exemplars/otel-collector.yaml +++ /dev/null @@ -1,30 +0,0 @@ -receivers: - otlp: - protocols: - grpc: - http: - -exporters: - logging: - loglevel: debug - prometheus: - endpoint: ":9201" - send_timestamps: true - metric_expiration: 180m - enable_open_metrics: true - otlp: - endpoint: tempo:4317 - tls: - insecure: true - -service: - pipelines: - traces: - receivers: [otlp] - exporters: [logging,otlp] - metrics: - receivers: [otlp] - exporters: [logging,prometheus] - logs: - receivers: [otlp] - exporters: [logging] diff --git a/docs/metrics/exemplars/prometheus.yaml b/docs/metrics/exemplars/prometheus.yaml deleted file mode 100644 index 4d6c5d18a6f..00000000000 --- a/docs/metrics/exemplars/prometheus.yaml +++ /dev/null @@ -1,8 +0,0 @@ -global: - scrape_interval: 15s - evaluation_interval: 15s - -scrape_configs: - - job_name: 'otel' - static_configs: - - targets: [ 'otel-collector:9201' ] diff --git a/docs/metrics/exemplars/tempo.yaml b/docs/metrics/exemplars/tempo.yaml deleted file mode 100644 index 0d46d4718e8..00000000000 --- a/docs/metrics/exemplars/tempo.yaml +++ /dev/null @@ -1,17 +0,0 @@ -server: - http_listen_port: 3200 - -distributor: - receivers: - otlp: - protocols: - http: - grpc: - -storage: - trace: - backend: local - wal: - path: /tmp/tempo/wal - local: - path: /tmp/tempo/blocks From 4b03720a1b66d594fab6c48f387db66e85dd4588 Mon Sep 17 00:00:00 2001 From: Reiley Yang Date: Mon, 20 May 2024 22:59:04 -0700 Subject: [PATCH 02/10] tweak --- docs/metrics/exemplars/Program.cs | 4 ++-- docs/metrics/exemplars/README.md | 13 +++++++++++-- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/docs/metrics/exemplars/Program.cs b/docs/metrics/exemplars/Program.cs index a55396a7a60..98ec7a66e7b 100644 --- a/docs/metrics/exemplars/Program.cs +++ b/docs/metrics/exemplars/Program.cs @@ -16,12 +16,12 @@ public class Program public static void Main() { - using var tracerProvider = Sdk.CreateTracerProviderBuilder() + var tracerProvider = Sdk.CreateTracerProviderBuilder() .AddSource("OpenTelemetry.Demo.Exemplar") .AddOtlpExporter() .Build(); - using var meterProvider = Sdk.CreateMeterProviderBuilder() + var meterProvider = Sdk.CreateMeterProviderBuilder() .AddMeter("OpenTelemetry.Demo.Exemplar") .SetExemplarFilter(ExemplarFilterType.TraceBased) .AddOtlpExporter((exporterOptions, metricReaderOptions) => diff --git a/docs/metrics/exemplars/README.md b/docs/metrics/exemplars/README.md index 52602076e84..bf637f83d2f 100644 --- a/docs/metrics/exemplars/README.md +++ b/docs/metrics/exemplars/README.md @@ -113,8 +113,17 @@ dotnet add package OpenTelemetry.Exporter.OpenTelemetryProtocol ``` Now copy the code from [Program.cs](./Program.cs) and run the application again. -The application will start sending metrics (with exemplars) to Prometheus and -traces to Jaeger. +The application will start sending metrics to Prometheus and traces to Jaeger. + +The application is configured with trace-based exemplar filter, which enables +the OpenTelemetry SDK to attach exemplars to metrics: + +```csharp +var meterProvider = Sdk.CreateMeterProviderBuilder() + ... + .SetExemplarFilter(ExemplarFilterType.TraceBased) + ... +``` ## Use exemplars to navigate from metrics to traces From daba90b6374074eb7b0533c83c79da513fdbbc82 Mon Sep 17 00:00:00 2001 From: Reiley Yang Date: Mon, 20 May 2024 23:03:28 -0700 Subject: [PATCH 03/10] fix link --- docs/metrics/exemplars/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/metrics/exemplars/README.md b/docs/metrics/exemplars/README.md index bf637f83d2f..4db5c987af0 100644 --- a/docs/metrics/exemplars/README.md +++ b/docs/metrics/exemplars/README.md @@ -22,7 +22,7 @@ the concept. The following components are involved: backend. - [Jaeger](#install-and-run-jaeger) - Jaeger is used as the distributed tracing backend. -- [Grafana](#install-and-run-grafana) - UI to query metrics from +- [Grafana](#install-and-configure-grafana) - UI to query metrics from Prometheus, traces from Jaeger, and to navigate between metrics and traces using Exemplars. From 03ff09b98c2de531931435e91db9debbbf1cf029 Mon Sep 17 00:00:00 2001 From: Reiley Yang Date: Mon, 20 May 2024 23:05:42 -0700 Subject: [PATCH 04/10] link --- docs/metrics/exemplars/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/metrics/exemplars/README.md b/docs/metrics/exemplars/README.md index 4db5c987af0..ebb94f0e98c 100644 --- a/docs/metrics/exemplars/README.md +++ b/docs/metrics/exemplars/README.md @@ -144,6 +144,8 @@ next to the "trace_id" field to open the corresponding trace in Jaeger. ## Learn more +- [Exemplar + specification](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/metrics/sdk.md#exemplar) - [What is Prometheus?](https://prometheus.io/docs/introduction/overview/) - [Prometheus now supports OpenTelemetry Metrics](https://horovits.medium.com/prometheus-now-supports-opentelemetry-metrics-83f85878e46a) From d1ba294e0a2c0c0c17a76362475a1604ff51460a Mon Sep 17 00:00:00 2001 From: Reiley Yang Date: Tue, 21 May 2024 10:03:02 -0700 Subject: [PATCH 05/10] polish --- docs/metrics/exemplars/Program.cs | 10 ++++------ docs/metrics/exemplars/README.md | 10 ++++++---- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/metrics/exemplars/Program.cs b/docs/metrics/exemplars/Program.cs index 98ec7a66e7b..cd63ac77ecc 100644 --- a/docs/metrics/exemplars/Program.cs +++ b/docs/metrics/exemplars/Program.cs @@ -12,7 +12,7 @@ public class Program { private static readonly ActivitySource MyActivitySource = new("OpenTelemetry.Demo.Exemplar"); private static readonly Meter MyMeter = new("OpenTelemetry.Demo.Exemplar"); - private static readonly Counter MyFruitCounter = MyMeter.CreateCounter("MyFruitCounter"); + private static readonly Histogram MyHistogram = MyMeter.CreateHistogram("MyHistogram"); public static void Main() { @@ -32,6 +32,8 @@ public static void Main() }) .Build(); + var random = new Random(); + Console.WriteLine("Press any key to exit"); while (!Console.KeyAvailable) @@ -46,14 +48,10 @@ public static void Main() child?.SetTag("key3", "value3"); child?.SetTag("key4", "value4"); - MyFruitCounter.Add(1, new("name", "apple"), new("color", "red")); - MyFruitCounter.Add(1, new("name", "lemon"), new("color", "yellow")); + MyHistogram.Record(random.NextDouble() * 100, new("tag1", "value1"), new("tag2", "value2")); } } - MyFruitCounter.Add(2, new("name", "apple"), new("color", "green")); - MyFruitCounter.Add(4, new("name", "lemon"), new("color", "yellow")); - Thread.Sleep(300); } diff --git a/docs/metrics/exemplars/README.md b/docs/metrics/exemplars/README.md index ebb94f0e98c..867c5bd5ed1 100644 --- a/docs/metrics/exemplars/README.md +++ b/docs/metrics/exemplars/README.md @@ -128,10 +128,11 @@ var meterProvider = Sdk.CreateMeterProviderBuilder() ## Use exemplars to navigate from metrics to traces Open Grafana, select Explore, and select Prometheus as the source. Select the -metric named "MyFruitCounter_total", and plot the chart. Toggle on the -"Exemplars" option from the UI and hit refresh. +metric named `MyHistogram_bucket`, and plot the chart. Toggle on the "Exemplars" +option from the UI and hit refresh. -![Enable Exemplars](https://github.com/open-telemetry/opentelemetry-dotnet/assets/17327289/fc11fd4b-5601-4728-acdd-5129d75fbc04) +![Enable +Exemplars](https://github.com/open-telemetry/opentelemetry-dotnet/assets/17327289/3c2dab2f-7a0a-419f-88fe-b8010b4c69f8) The Exemplars appear as special "diamond shaped dots" along with the metric charts in the UI. Select any exemplar to see the exemplar data, which includes @@ -140,7 +141,8 @@ context when the recording was done. The "trace_id" enables jumping to the tracing backed (Jaeger in this case). Click on the "Query with Jaeger" button next to the "trace_id" field to open the corresponding trace in Jaeger. -![Navigate to trace with exemplar](https://user-images.githubusercontent.com/16979322/218629999-1d1cd6ba-2385-4683-975a-d4797df8361a.png) +![Navigate to trace with +exemplar](https://github.com/open-telemetry/opentelemetry-dotnet/assets/17327289/56bb5297-f744-41f3-bc35-8596392b8673) ## Learn more From 107440aa6561be8a1dbd94f5cf8a3978b54cf7ef Mon Sep 17 00:00:00 2001 From: Reiley Yang Date: Tue, 21 May 2024 10:09:13 -0700 Subject: [PATCH 06/10] ToC --- docs/metrics/exemplars/README.md | 37 +++++++++++++++++++------------- 1 file changed, 22 insertions(+), 15 deletions(-) diff --git a/docs/metrics/exemplars/README.md b/docs/metrics/exemplars/README.md index 867c5bd5ed1..e08283f593b 100644 --- a/docs/metrics/exemplars/README.md +++ b/docs/metrics/exemplars/README.md @@ -1,13 +1,20 @@ # Using Exemplars in OpenTelemetry .NET -- [Install and run Jaeger](#install-and-run-jaeger) -- [Install and run Prometheus](#install-and-run-prometheus) -- [Install and configure Grafana](#install-and-configure-grafana) -- [Export metrics and traces from the + +
+Table of Contents + +* [Install and run Jaeger](#install-and-run-jaeger) +* [Install and run Prometheus](#install-and-run-prometheus) +* [Install and configure Grafana](#install-and-configure-grafana) +* [Export metrics and traces from the application](#export-metrics-and-traces-from-the-application) -- [Use exemplars to navigate from metrics to +* [Use exemplars to navigate from metrics to traces](#use-exemplars-to-navigate-from-metrics-to-traces) -- [Learn more](#learn-more) +* [Learn more](#learn-more) + +
+ Exemplars are example data points for aggregated data. They provide specific context to otherwise general aggregations. One common use case is to gain @@ -16,13 +23,13 @@ supports Exemplars, it is only useful if the telemetry backend also supports the capabilities. This tutorial uses well known open-source backends to demonstrate the concept. The following components are involved: -- [Program.cs](./Program.cs) - this application is instrumented with +* [Program.cs](./Program.cs) - this application is instrumented with OpenTelemetry, it sends metrics to Prometheus, and traces to Jaeger. -- [Prometheus](#install-and-run-prometheus) - Prometheus is used as the metrics +* [Prometheus](#install-and-run-prometheus) - Prometheus is used as the metrics backend. -- [Jaeger](#install-and-run-jaeger) - Jaeger is used as the distributed tracing +* [Jaeger](#install-and-run-jaeger) - Jaeger is used as the distributed tracing backend. -- [Grafana](#install-and-configure-grafana) - UI to query metrics from +* [Grafana](#install-and-configure-grafana) - UI to query metrics from Prometheus, traces from Jaeger, and to navigate between metrics and traces using Exemplars. @@ -146,11 +153,11 @@ exemplar](https://github.com/open-telemetry/opentelemetry-dotnet/assets/17327289 ## Learn more -- [Exemplar +* [Exemplar specification](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/metrics/sdk.md#exemplar) -- [What is Prometheus?](https://prometheus.io/docs/introduction/overview/) -- [Prometheus now supports OpenTelemetry +* [What is Prometheus?](https://prometheus.io/docs/introduction/overview/) +* [Prometheus now supports OpenTelemetry Metrics](https://horovits.medium.com/prometheus-now-supports-opentelemetry-metrics-83f85878e46a) -- [Jaeger Tracing](https://www.jaegertracing.io/) -- [Grafana support for +* [Jaeger Tracing](https://www.jaegertracing.io/) +* [Grafana support for Prometheus](https://prometheus.io/docs/visualization/grafana/#creating-a-prometheus-graph) From d55b453e3a293a7a62fd056fa16a9f5c99f773b0 Mon Sep 17 00:00:00 2001 From: Reiley Yang Date: Tue, 21 May 2024 10:11:39 -0700 Subject: [PATCH 07/10] link to exemplar configuration --- docs/metrics/exemplars/README.md | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/docs/metrics/exemplars/README.md b/docs/metrics/exemplars/README.md index e08283f593b..ed15c883bd5 100644 --- a/docs/metrics/exemplars/README.md +++ b/docs/metrics/exemplars/README.md @@ -16,12 +16,13 @@ -Exemplars are example data points for aggregated data. They provide specific -context to otherwise general aggregations. One common use case is to gain -ability to correlate metrics to traces (and logs). While OpenTelemetry .NET -supports Exemplars, it is only useful if the telemetry backend also supports the -capabilities. This tutorial uses well known open-source backends to demonstrate -the concept. The following components are involved: +[Exemplars](../customizing-the-sdk/README.md#exemplars) are example data points +for aggregated data. They provide specific context to otherwise general +aggregations. One common use case is to gain ability to correlate metrics to +traces (and logs). While OpenTelemetry .NET supports Exemplars, it is only +useful if the telemetry backend also supports the capabilities. This tutorial +uses well known open-source backends to demonstrate the concept. The following +components are involved: * [Program.cs](./Program.cs) - this application is instrumented with OpenTelemetry, it sends metrics to Prometheus, and traces to Jaeger. From 4fbd2985102305edf938dbf45c7d3b29cbbc2e94 Mon Sep 17 00:00:00 2001 From: Reiley Yang Date: Tue, 21 May 2024 10:30:41 -0700 Subject: [PATCH 08/10] address comments --- docs/metrics/exemplars/README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/metrics/exemplars/README.md b/docs/metrics/exemplars/README.md index ed15c883bd5..a677a012bff 100644 --- a/docs/metrics/exemplars/README.md +++ b/docs/metrics/exemplars/README.md @@ -133,6 +133,10 @@ var meterProvider = Sdk.CreateMeterProviderBuilder() ... ``` +For more details about the `SetExemplarFilter` API see: [Customizing +OpenTelemetry .NET SDK for Metrics > +ExemplarFilter](../customizing-the-sdk/README.md#exemplarfilter). + ## Use exemplars to navigate from metrics to traces Open Grafana, select Explore, and select Prometheus as the source. Select the @@ -140,7 +144,7 @@ metric named `MyHistogram_bucket`, and plot the chart. Toggle on the "Exemplars" option from the UI and hit refresh. ![Enable -Exemplars](https://github.com/open-telemetry/opentelemetry-dotnet/assets/17327289/3c2dab2f-7a0a-419f-88fe-b8010b4c69f8) +Exemplars](https://github.com/open-telemetry/opentelemetry-dotnet/assets/17327289/bc461c6d-a0b9-49b7-a91d-94b07c3f417f) The Exemplars appear as special "diamond shaped dots" along with the metric charts in the UI. Select any exemplar to see the exemplar data, which includes From d82ae9dc0522db82aee5103ff0895e9c642ee7a7 Mon Sep 17 00:00:00 2001 From: Reiley Yang Date: Tue, 21 May 2024 10:43:52 -0700 Subject: [PATCH 09/10] clean up --- docs/metrics/exemplars/README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/metrics/exemplars/README.md b/docs/metrics/exemplars/README.md index a677a012bff..89f88866560 100644 --- a/docs/metrics/exemplars/README.md +++ b/docs/metrics/exemplars/README.md @@ -1,6 +1,5 @@ # Using Exemplars in OpenTelemetry .NET -
Table of Contents @@ -14,7 +13,6 @@ * [Learn more](#learn-more)
- [Exemplars](../customizing-the-sdk/README.md#exemplars) are example data points for aggregated data. They provide specific context to otherwise general From d91587b22e93e3ee942ecf42bac6a2c4554e7a60 Mon Sep 17 00:00:00 2001 From: Reiley Yang Date: Tue, 21 May 2024 16:14:59 -0700 Subject: [PATCH 10/10] Restore Docker files --- docs/metrics/exemplars/docker-compose.yaml | 50 +++++++++++++++++++ .../exemplars/grafana-datasources.yaml | 33 ++++++++++++ docs/metrics/exemplars/otel-collector.yaml | 30 +++++++++++ docs/metrics/exemplars/prometheus.yaml | 8 +++ docs/metrics/exemplars/tempo.yaml | 17 +++++++ 5 files changed, 138 insertions(+) create mode 100644 docs/metrics/exemplars/docker-compose.yaml create mode 100644 docs/metrics/exemplars/grafana-datasources.yaml create mode 100644 docs/metrics/exemplars/otel-collector.yaml create mode 100644 docs/metrics/exemplars/prometheus.yaml create mode 100644 docs/metrics/exemplars/tempo.yaml diff --git a/docs/metrics/exemplars/docker-compose.yaml b/docs/metrics/exemplars/docker-compose.yaml new file mode 100644 index 00000000000..c8cc94fa4b1 --- /dev/null +++ b/docs/metrics/exemplars/docker-compose.yaml @@ -0,0 +1,50 @@ +version: "3" +services: + + # OTEL Collector to receive logs, metrics and traces from the application + otel-collector: + image: otel/opentelemetry-collector:0.70.0 + command: [ "--config=/etc/otel-collector.yaml" ] + volumes: + - ./otel-collector.yaml:/etc/otel-collector.yaml + ports: + - "4317:4317" + - "4318:4318" + - "9201:9201" + + # Exports Traces to Tempo + tempo: + image: grafana/tempo:latest + command: [ "-config.file=/etc/tempo.yaml" ] + volumes: + - ./tempo.yaml:/etc/tempo.yaml + - ./tempo-data:/tmp/tempo + ports: + - "3200" # tempo + - "4317" # otlp grpc + - "4318" # otlp http + +# Exports Metrics to Prometheus + prometheus: + image: prom/prometheus:latest + command: + - --config.file=/etc/prometheus.yaml + - --web.enable-remote-write-receiver + - --enable-feature=exemplar-storage + volumes: + - ./prometheus.yaml:/etc/prometheus.yaml + ports: + - "9090:9090" + +# UI to query traces and metrics + grafana: + image: grafana/grafana:9.3.2 + volumes: + - ./grafana-datasources.yaml:/etc/grafana/provisioning/datasources/datasources.yaml + environment: + - GF_AUTH_ANONYMOUS_ENABLED=true + - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin + - GF_AUTH_DISABLE_LOGIN_FORM=true + - GF_FEATURE_TOGGLES_ENABLE=traceqlEditor + ports: + - "3000:3000" diff --git a/docs/metrics/exemplars/grafana-datasources.yaml b/docs/metrics/exemplars/grafana-datasources.yaml new file mode 100644 index 00000000000..467975584ca --- /dev/null +++ b/docs/metrics/exemplars/grafana-datasources.yaml @@ -0,0 +1,33 @@ +apiVersion: 1 + +datasources: +- name: Prometheus + type: prometheus + uid: prometheus + access: proxy + orgId: 1 + url: http://prometheus:9090 + basicAuth: false + isDefault: true + version: 1 + editable: false + jsonData: + httpMethod: GET + exemplarTraceIdDestinations: + - name: trace_id + datasourceUid: Tempo +- name: Tempo + type: tempo + access: proxy + orgId: 1 + url: http://tempo:3200 + basicAuth: false + isDefault: false + version: 1 + editable: false + apiVersion: 1 + uid: tempo + jsonData: + httpMethod: GET + serviceMap: + datasourceUid: prometheus diff --git a/docs/metrics/exemplars/otel-collector.yaml b/docs/metrics/exemplars/otel-collector.yaml new file mode 100644 index 00000000000..bcf0cb5d6d2 --- /dev/null +++ b/docs/metrics/exemplars/otel-collector.yaml @@ -0,0 +1,30 @@ +receivers: + otlp: + protocols: + grpc: + http: + +exporters: + logging: + loglevel: debug + prometheus: + endpoint: ":9201" + send_timestamps: true + metric_expiration: 180m + enable_open_metrics: true + otlp: + endpoint: tempo:4317 + tls: + insecure: true + +service: + pipelines: + traces: + receivers: [otlp] + exporters: [logging,otlp] + metrics: + receivers: [otlp] + exporters: [logging,prometheus] + logs: + receivers: [otlp] + exporters: [logging] diff --git a/docs/metrics/exemplars/prometheus.yaml b/docs/metrics/exemplars/prometheus.yaml new file mode 100644 index 00000000000..4d6c5d18a6f --- /dev/null +++ b/docs/metrics/exemplars/prometheus.yaml @@ -0,0 +1,8 @@ +global: + scrape_interval: 15s + evaluation_interval: 15s + +scrape_configs: + - job_name: 'otel' + static_configs: + - targets: [ 'otel-collector:9201' ] diff --git a/docs/metrics/exemplars/tempo.yaml b/docs/metrics/exemplars/tempo.yaml new file mode 100644 index 00000000000..0d46d4718e8 --- /dev/null +++ b/docs/metrics/exemplars/tempo.yaml @@ -0,0 +1,17 @@ +server: + http_listen_port: 3200 + +distributor: + receivers: + otlp: + protocols: + http: + grpc: + +storage: + trace: + backend: local + wal: + path: /tmp/tempo/wal + local: + path: /tmp/tempo/blocks