diff --git a/src/OpenTelemetry.Exporter.Prometheus.AspNetCore/CHANGELOG.md b/src/OpenTelemetry.Exporter.Prometheus.AspNetCore/CHANGELOG.md index 703074b0be6..c4d7b083aba 100644 --- a/src/OpenTelemetry.Exporter.Prometheus.AspNetCore/CHANGELOG.md +++ b/src/OpenTelemetry.Exporter.Prometheus.AspNetCore/CHANGELOG.md @@ -2,6 +2,9 @@ ## Unreleased +* Fixed issue with OpenMetrics counter suffixes for Prometheus + ([#5623](https://github.com/open-telemetry/opentelemetry-dotnet/pull/5623)) + ## 1.9.0-alpha.1 Released 2024-May-20 diff --git a/src/OpenTelemetry.Exporter.Prometheus.HttpListener/CHANGELOG.md b/src/OpenTelemetry.Exporter.Prometheus.HttpListener/CHANGELOG.md index abb4d7287fc..19b77b88a2d 100644 --- a/src/OpenTelemetry.Exporter.Prometheus.HttpListener/CHANGELOG.md +++ b/src/OpenTelemetry.Exporter.Prometheus.HttpListener/CHANGELOG.md @@ -2,6 +2,9 @@ ## Unreleased +* Fixed issue with OpenMetrics counter suffixes for Prometheus + ([#5623](https://github.com/open-telemetry/opentelemetry-dotnet/pull/5623)) + ## 1.9.0-alpha.1 Released 2024-May-20 diff --git a/src/OpenTelemetry.Exporter.Prometheus.HttpListener/Internal/PrometheusMetric.cs b/src/OpenTelemetry.Exporter.Prometheus.HttpListener/Internal/PrometheusMetric.cs index a39a426136a..688c2e81b16 100644 --- a/src/OpenTelemetry.Exporter.Prometheus.HttpListener/Internal/PrometheusMetric.cs +++ b/src/OpenTelemetry.Exporter.Prometheus.HttpListener/Internal/PrometheusMetric.cs @@ -27,6 +27,7 @@ public PrometheusMetric(string name, string unit, PrometheusType type, bool disa // consecutive `_` characters MUST be replaced with a single `_` character. // https://github.com/open-telemetry/opentelemetry-specification/blob/b2f923fb1650dde1f061507908b834035506a796/specification/compatibility/prometheus_and_openmetrics.md#L230-L233 var sanitizedName = SanitizeMetricName(name); + var openMetricsName = SanitizeOpenMetricsName(sanitizedName); string sanitizedUnit = null; if (!string.IsNullOrEmpty(unit)) @@ -41,10 +42,31 @@ public PrometheusMetric(string name, string unit, PrometheusType type, bool disa // https://github.com/open-telemetry/opentelemetry-specification/blob/b2f923fb1650dde1f061507908b834035506a796/specification/compatibility/prometheus_and_openmetrics.md#L242-L246 if (!sanitizedName.Contains(sanitizedUnit)) { - sanitizedName = sanitizedName + "_" + sanitizedUnit; + sanitizedName += $"_{sanitizedUnit}"; } + + // OpenMetrics name MUST be suffixed with '_{unit}', regardless of whether the unit name appears within the text. + // Note that this may change in the future, however for the moment Prometheus will fail to read the metric using + // OpenMetrics format unless the suffix matches the unit. + // https://github.com/OpenObservability/OpenMetrics/blob/main/specification/OpenMetrics.md#unit + if (!openMetricsName.EndsWith(sanitizedUnit)) + { + openMetricsName += $"_{sanitizedUnit}"; + } + } + + // Special case: Converting "1" to "ratio". + // https://github.com/open-telemetry/opentelemetry-specification/blob/b2f923fb1650dde1f061507908b834035506a796/specification/compatibility/prometheus_and_openmetrics.md#L239 + if (type == PrometheusType.Gauge && unit == "1" && !sanitizedName.Contains("ratio")) + { + sanitizedName += "_ratio"; + openMetricsName += "_ratio"; } + // For TYPE, HELP and UNIT declarations for counters, the suffix '_total' is omitted in OpenMetrics format. + // https://github.com/OpenObservability/OpenMetrics/blob/main/specification/OpenMetrics.md#counter-1 + this.OpenMetricsMetadataName = openMetricsName; + // If the metric name for monotonic Sum metric points does not end in a suffix of `_total` a suffix of `_total` MUST be added by default, otherwise the name MUST remain unchanged. // Exporters SHOULD provide a configuration option to disable the addition of `_total` suffixes. // https://github.com/open-telemetry/opentelemetry-specification/blob/b2f923fb1650dde1f061507908b834035506a796/specification/compatibility/prometheus_and_openmetrics.md#L286 @@ -53,20 +75,25 @@ public PrometheusMetric(string name, string unit, PrometheusType type, bool disa sanitizedName += "_total"; } - // Special case: Converting "1" to "ratio". - // https://github.com/open-telemetry/opentelemetry-specification/blob/b2f923fb1650dde1f061507908b834035506a796/specification/compatibility/prometheus_and_openmetrics.md#L239 - if (type == PrometheusType.Gauge && unit == "1" && !sanitizedName.Contains("ratio")) + // For counters requested using OpenMetrics format, the MetricFamily name MUST be suffixed with '_total', regardless of the setting to disable the 'total' suffix. + // https://github.com/OpenObservability/OpenMetrics/blob/main/specification/OpenMetrics.md#counter-1 + if (type == PrometheusType.Counter && !openMetricsName.EndsWith("_total")) { - sanitizedName += "_ratio"; + openMetricsName += "_total"; } this.Name = sanitizedName; + this.OpenMetricsName = openMetricsName; this.Unit = sanitizedUnit; this.Type = type; } public string Name { get; } + public string OpenMetricsName { get; } + + public string OpenMetricsMetadataName { get; } + public string Unit { get; } public PrometheusType Type { get; } @@ -159,6 +186,16 @@ internal static string RemoveAnnotations(string unit) return sb.ToString(); } + private static string SanitizeOpenMetricsName(string metricName) + { + if (metricName.EndsWith("_total")) + { + return metricName.Substring(0, metricName.Length - 6); + } + + return metricName; + } + private static string GetUnit(string unit) { // Dropping the portions of the Unit within brackets (e.g. {packet}). Brackets MUST NOT be included in the resulting unit. A "count of foo" is considered unitless in Prometheus. diff --git a/src/OpenTelemetry.Exporter.Prometheus.HttpListener/Internal/PrometheusSerializer.cs b/src/OpenTelemetry.Exporter.Prometheus.HttpListener/Internal/PrometheusSerializer.cs index 719f21a0c61..3c142d91233 100644 --- a/src/OpenTelemetry.Exporter.Prometheus.HttpListener/Internal/PrometheusSerializer.cs +++ b/src/OpenTelemetry.Exporter.Prometheus.HttpListener/Internal/PrometheusSerializer.cs @@ -230,12 +230,29 @@ static string GetLabelValueString(object labelValue) } [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static int WriteMetricName(byte[] buffer, int cursor, PrometheusMetric metric) + public static int WriteMetricName(byte[] buffer, int cursor, PrometheusMetric metric, bool openMetricsRequested) { // Metric name has already been escaped. - for (int i = 0; i < metric.Name.Length; i++) + var name = openMetricsRequested ? metric.OpenMetricsName : metric.Name; + + for (int i = 0; i < name.Length; i++) + { + var ordinal = (ushort)name[i]; + buffer[cursor++] = unchecked((byte)ordinal); + } + + return cursor; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int WriteMetricMetadataName(byte[] buffer, int cursor, PrometheusMetric metric, bool openMetricsRequested) + { + // Metric name has already been escaped. + var name = openMetricsRequested ? metric.OpenMetricsMetadataName : metric.Name; + + for (int i = 0; i < name.Length; i++) { - var ordinal = (ushort)metric.Name[i]; + var ordinal = (ushort)name[i]; buffer[cursor++] = unchecked((byte)ordinal); } @@ -252,7 +269,7 @@ public static int WriteEof(byte[] buffer, int cursor) } [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static int WriteHelpMetadata(byte[] buffer, int cursor, PrometheusMetric metric, string metricDescription) + public static int WriteHelpMetadata(byte[] buffer, int cursor, PrometheusMetric metric, string metricDescription, bool openMetricsRequested) { if (string.IsNullOrEmpty(metricDescription)) { @@ -260,7 +277,7 @@ public static int WriteHelpMetadata(byte[] buffer, int cursor, PrometheusMetric } cursor = WriteAsciiStringNoEscape(buffer, cursor, "# HELP "); - cursor = WriteMetricName(buffer, cursor, metric); + cursor = WriteMetricMetadataName(buffer, cursor, metric, openMetricsRequested); if (!string.IsNullOrEmpty(metricDescription)) { @@ -274,14 +291,14 @@ public static int WriteHelpMetadata(byte[] buffer, int cursor, PrometheusMetric } [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static int WriteTypeMetadata(byte[] buffer, int cursor, PrometheusMetric metric) + public static int WriteTypeMetadata(byte[] buffer, int cursor, PrometheusMetric metric, bool openMetricsRequested) { var metricType = MapPrometheusType(metric.Type); Debug.Assert(!string.IsNullOrEmpty(metricType), $"{nameof(metricType)} should not be null or empty."); cursor = WriteAsciiStringNoEscape(buffer, cursor, "# TYPE "); - cursor = WriteMetricName(buffer, cursor, metric); + cursor = WriteMetricMetadataName(buffer, cursor, metric, openMetricsRequested); buffer[cursor++] = unchecked((byte)' '); cursor = WriteAsciiStringNoEscape(buffer, cursor, metricType); @@ -291,7 +308,7 @@ public static int WriteTypeMetadata(byte[] buffer, int cursor, PrometheusMetric } [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static int WriteUnitMetadata(byte[] buffer, int cursor, PrometheusMetric metric) + public static int WriteUnitMetadata(byte[] buffer, int cursor, PrometheusMetric metric, bool openMetricsRequested) { if (string.IsNullOrEmpty(metric.Unit)) { @@ -299,7 +316,7 @@ public static int WriteUnitMetadata(byte[] buffer, int cursor, PrometheusMetric } cursor = WriteAsciiStringNoEscape(buffer, cursor, "# UNIT "); - cursor = WriteMetricName(buffer, cursor, metric); + cursor = WriteMetricMetadataName(buffer, cursor, metric, openMetricsRequested); buffer[cursor++] = unchecked((byte)' '); diff --git a/src/OpenTelemetry.Exporter.Prometheus.HttpListener/Internal/PrometheusSerializerExt.cs b/src/OpenTelemetry.Exporter.Prometheus.HttpListener/Internal/PrometheusSerializerExt.cs index 1523ef7c160..70d67f468bf 100644 --- a/src/OpenTelemetry.Exporter.Prometheus.HttpListener/Internal/PrometheusSerializerExt.cs +++ b/src/OpenTelemetry.Exporter.Prometheus.HttpListener/Internal/PrometheusSerializerExt.cs @@ -24,9 +24,9 @@ public static bool CanWriteMetric(Metric metric) public static int WriteMetric(byte[] buffer, int cursor, Metric metric, PrometheusMetric prometheusMetric, bool openMetricsRequested = false) { - cursor = WriteTypeMetadata(buffer, cursor, prometheusMetric); - cursor = WriteUnitMetadata(buffer, cursor, prometheusMetric); - cursor = WriteHelpMetadata(buffer, cursor, prometheusMetric, metric.Description); + cursor = WriteTypeMetadata(buffer, cursor, prometheusMetric, openMetricsRequested); + cursor = WriteUnitMetadata(buffer, cursor, prometheusMetric, openMetricsRequested); + cursor = WriteHelpMetadata(buffer, cursor, prometheusMetric, metric.Description, openMetricsRequested); if (!metric.MetricType.IsHistogram()) { @@ -35,7 +35,7 @@ public static int WriteMetric(byte[] buffer, int cursor, Metric metric, Promethe var timestamp = metricPoint.EndTime.ToUnixTimeMilliseconds(); // Counter and Gauge - cursor = WriteMetricName(buffer, cursor, prometheusMetric); + cursor = WriteMetricName(buffer, cursor, prometheusMetric, openMetricsRequested); cursor = WriteTags(buffer, cursor, metric, metricPoint.Tags); buffer[cursor++] = unchecked((byte)' '); @@ -85,7 +85,7 @@ public static int WriteMetric(byte[] buffer, int cursor, Metric metric, Promethe { totalCount += histogramMeasurement.BucketCount; - cursor = WriteMetricName(buffer, cursor, prometheusMetric); + cursor = WriteMetricName(buffer, cursor, prometheusMetric, openMetricsRequested); cursor = WriteAsciiStringNoEscape(buffer, cursor, "_bucket{"); cursor = WriteTags(buffer, cursor, metric, tags, writeEnclosingBraces: false); @@ -111,7 +111,7 @@ public static int WriteMetric(byte[] buffer, int cursor, Metric metric, Promethe } // Histogram sum - cursor = WriteMetricName(buffer, cursor, prometheusMetric); + cursor = WriteMetricName(buffer, cursor, prometheusMetric, openMetricsRequested); cursor = WriteAsciiStringNoEscape(buffer, cursor, "_sum"); cursor = WriteTags(buffer, cursor, metric, metricPoint.Tags); @@ -125,7 +125,7 @@ public static int WriteMetric(byte[] buffer, int cursor, Metric metric, Promethe buffer[cursor++] = ASCII_LINEFEED; // Histogram count - cursor = WriteMetricName(buffer, cursor, prometheusMetric); + cursor = WriteMetricName(buffer, cursor, prometheusMetric, openMetricsRequested); cursor = WriteAsciiStringNoEscape(buffer, cursor, "_count"); cursor = WriteTags(buffer, cursor, metric, metricPoint.Tags); diff --git a/test/OpenTelemetry.Exporter.Prometheus.AspNetCore.Tests/PrometheusExporterMiddlewareTests.cs b/test/OpenTelemetry.Exporter.Prometheus.AspNetCore.Tests/PrometheusExporterMiddlewareTests.cs index 7386a5a9729..19b22e47be5 100644 --- a/test/OpenTelemetry.Exporter.Prometheus.AspNetCore.Tests/PrometheusExporterMiddlewareTests.cs +++ b/test/OpenTelemetry.Exporter.Prometheus.AspNetCore.Tests/PrometheusExporterMiddlewareTests.cs @@ -264,7 +264,7 @@ public async Task PrometheusExporterMiddlewareIntegration_CanServeOpenMetricsAnd var beginTimestamp = DateTimeOffset.Now.ToUnixTimeMilliseconds(); - var counter = meter.CreateCounter("counter_double"); + var counter = meter.CreateCounter("counter_double", unit: "By"); counter.Add(100.18D, tags); counter.Add(0.99D, tags); @@ -312,7 +312,7 @@ private static async Task RunPrometheusExporterMiddlewareIntegrationTest( var beginTimestamp = DateTimeOffset.Now.ToUnixTimeMilliseconds(); - var counter = meter.CreateCounter("counter_double"); + var counter = meter.CreateCounter("counter_double", unit: "By"); if (!skipMetrics) { counter.Add(100.18D, tags); @@ -368,14 +368,16 @@ private static async Task VerifyAsync(long beginTimestamp, long endTimestamp, Ht # TYPE otel_scope_info info # HELP otel_scope_info Scope metadata otel_scope_info{otel_scope_name="{{MeterName}}"} 1 - # TYPE counter_double_total counter - counter_double_total{otel_scope_name="{{MeterName}}",otel_scope_version="{{MeterVersion}}",key1="value1",key2="value2"} 101.17 (\d+\.\d{3}) + # TYPE counter_double_bytes counter + # UNIT counter_double_bytes bytes + counter_double_bytes_total{otel_scope_name="{{MeterName}}",otel_scope_version="{{MeterVersion}}",key1="value1",key2="value2"} 101.17 (\d+\.\d{3}) # EOF """.ReplaceLineEndings() : $$""" - # TYPE counter_double_total counter - counter_double_total{otel_scope_name="{{MeterName}}",otel_scope_version="{{MeterVersion}}",key1="value1",key2="value2"} 101.17 (\d+) + # TYPE counter_double_bytes_total counter + # UNIT counter_double_bytes_total bytes + counter_double_bytes_total{otel_scope_name="{{MeterName}}",otel_scope_version="{{MeterVersion}}",key1="value1",key2="value2"} 101.17 (\d+) # EOF """.ReplaceLineEndings(); diff --git a/test/OpenTelemetry.Exporter.Prometheus.HttpListener.Tests/PrometheusHttpListenerTests.cs b/test/OpenTelemetry.Exporter.Prometheus.HttpListener.Tests/PrometheusHttpListenerTests.cs index 3a4ca1c4153..9ef62de196f 100644 --- a/test/OpenTelemetry.Exporter.Prometheus.HttpListener.Tests/PrometheusHttpListenerTests.cs +++ b/test/OpenTelemetry.Exporter.Prometheus.HttpListener.Tests/PrometheusHttpListenerTests.cs @@ -202,7 +202,7 @@ private async Task RunPrometheusExporterHttpServerIntegrationTest(bool skipMetri new KeyValuePair("key2", "value2"), }; - var counter = meter.CreateCounter("counter_double"); + var counter = meter.CreateCounter("counter_double", unit: "By"); if (!skipMetrics) { counter.Add(100.18D, tags); @@ -241,11 +241,13 @@ private async Task RunPrometheusExporterHttpServerIntegrationTest(bool skipMetri + "# TYPE otel_scope_info info\n" + "# HELP otel_scope_info Scope metadata\n" + $"otel_scope_info{{otel_scope_name='{MeterName}'}} 1\n" - + "# TYPE counter_double_total counter\n" - + $"counter_double_total{{otel_scope_name='{MeterName}',otel_scope_version='{MeterVersion}',key1='value1',key2='value2'}} 101.17 (\\d+\\.\\d{{3}})\n" + + "# TYPE counter_double_bytes counter\n" + + "# UNIT counter_double_bytes bytes\n" + + $"counter_double_bytes_total{{otel_scope_name='{MeterName}',otel_scope_version='{MeterVersion}',key1='value1',key2='value2'}} 101.17 (\\d+\\.\\d{{3}})\n" + "# EOF\n" - : "# TYPE counter_double_total counter\n" - + $"counter_double_total{{otel_scope_name='{MeterName}',otel_scope_version='{MeterVersion}',key1='value1',key2='value2'}} 101.17 (\\d+)\n" + : "# TYPE counter_double_bytes_total counter\n" + + "# UNIT counter_double_bytes_total bytes\n" + + $"counter_double_bytes_total{{otel_scope_name='{MeterName}',otel_scope_version='{MeterVersion}',key1='value1',key2='value2'}} 101.17 (\\d+)\n" + "# EOF\n"; Assert.Matches(("^" + expected + "$").Replace('\'', '"'), content); diff --git a/test/OpenTelemetry.Exporter.Prometheus.HttpListener.Tests/PrometheusMetricTests.cs b/test/OpenTelemetry.Exporter.Prometheus.HttpListener.Tests/PrometheusMetricTests.cs index 81ce3e1f7da..d2990ceca70 100644 --- a/test/OpenTelemetry.Exporter.Prometheus.HttpListener.Tests/PrometheusMetricTests.cs +++ b/test/OpenTelemetry.Exporter.Prometheus.HttpListener.Tests/PrometheusMetricTests.cs @@ -26,7 +26,7 @@ public void SanitizeMetricName_SupportLeadingAndTrailingUnderscores() } [Fact] - public void SanitizeMetricName_RemoveUnsupportedChracters() + public void SanitizeMetricName_RemoveUnsupportedCharacters() { AssertSanitizeMetricName("metric_unit_$1000", "metric_unit_1000"); } @@ -116,13 +116,13 @@ public void Unit_AnnotationMismatch_Close() } [Fact] - public void Name_SpecialCaseGuage_AppendRatio() + public void Name_SpecialCaseGauge_AppendRatio() { AssertName("sample", "1", PrometheusType.Gauge, false, "sample_ratio"); } [Fact] - public void Name_GuageWithUnit_NoAppendRatio() + public void Name_GaugeWithUnit_NoAppendRatio() { AssertName("sample", "unit", PrometheusType.Gauge, false, "sample_unit"); } @@ -205,6 +205,66 @@ public void Name_StartWithNumber_UnderscoreStart() AssertName("2_metric_name", "By", PrometheusType.Summary, false, "_metric_name_bytes"); } + [Fact] + public void OpenMetricsName_UnitAlreadyPresentInName_Appended() + { + AssertOpenMetricsName("db_bytes_written", "By", PrometheusType.Gauge, false, "db_bytes_written_bytes"); + } + + [Fact] + public void OpenMetricsName_SuffixedWithUnit_NotAppended() + { + AssertOpenMetricsName("db_written_bytes", "By", PrometheusType.Gauge, false, "db_written_bytes"); + } + + [Fact] + public void OpenMetricsName_Counter_AppendTotal() + { + AssertOpenMetricsName("db_bytes_written", "By", PrometheusType.Counter, false, "db_bytes_written_bytes_total"); + } + + [Fact] + public void OpenMetricsName_Counter_DisableSuffixTotal_AppendTotal() + { + AssertOpenMetricsName("db_bytes_written", "By", PrometheusType.Counter, true, "db_bytes_written_bytes_total"); + } + + [Fact] + public void OpenMetricsName_CounterSuffixedWithTotal_AppendUnitAndTotal() + { + AssertOpenMetricsName("db_bytes_written_total", "By", PrometheusType.Counter, false, "db_bytes_written_bytes_total"); + } + + [Fact] + public void OpenMetricsName_CounterSuffixedWithTotal_DisableSuffixTotal_AppendTotal() + { + AssertOpenMetricsName("db_bytes_written_total", "By", PrometheusType.Counter, false, "db_bytes_written_bytes_total"); + } + + [Fact] + public void OpenMetricsName_SpecialCaseGauge_AppendRatio() + { + AssertOpenMetricsName("sample", "1", PrometheusType.Gauge, false, "sample_ratio"); + } + + [Fact] + public void OpenMetricsMetadataName_Counter_NotAppendTotal() + { + AssertOpenMetricsMetadataName("db_bytes_written", "By", PrometheusType.Counter, false, "db_bytes_written_bytes"); + } + + [Fact] + public void OpenMetricsMetadataName_Counter_DisableSuffixTotal_NotAppendTotal() + { + AssertOpenMetricsMetadataName("db_bytes_written", "By", PrometheusType.Counter, true, "db_bytes_written_bytes"); + } + + [Fact] + public void OpenMetricsMetadataName_SpecialCaseGauge_AppendRatio() + { + AssertOpenMetricsMetadataName("sample", "1", PrometheusType.Gauge, false, "sample_ratio"); + } + private static void AssertName( string name, string unit, PrometheusType type, bool disableTotalNameSuffixForCounters, string expected) { @@ -217,4 +277,18 @@ private static void AssertSanitizeMetricName(string name, string expected) var sanatizedName = PrometheusMetric.SanitizeMetricName(name); Assert.Equal(expected, sanatizedName); } + + private static void AssertOpenMetricsName( + string name, string unit, PrometheusType type, bool disableTotalNameSuffixForCounters, string expected) + { + var prometheusMetric = new PrometheusMetric(name, unit, type, disableTotalNameSuffixForCounters); + Assert.Equal(expected, prometheusMetric.OpenMetricsName); + } + + private static void AssertOpenMetricsMetadataName( + string name, string unit, PrometheusType type, bool disableTotalNameSuffixForCounters, string expected) + { + var prometheusMetric = new PrometheusMetric(name, unit, type, disableTotalNameSuffixForCounters); + Assert.Equal(expected, prometheusMetric.OpenMetricsMetadataName); + } }