Fix OpenMetrics format suffixes for Prometheus

src/OpenTelemetry.Exporter.Prometheus.AspNetCore/CHANGELOG.md

@@ -2,6 +2,9 @@
 
 ## Unreleased
 
+* Fixed issue with OpenMetrics counter suffixes for Prometheus
+  ([#5623](https://github.com/open-telemetry/opentelemetry-dotnet/pull/5623))
+
 ## 1.9.0-alpha.1
 
 Released 2024-May-20

src/OpenTelemetry.Exporter.Prometheus.HttpListener/CHANGELOG.md

@@ -2,6 +2,9 @@
 
 ## Unreleased
 
+* Fixed issue with OpenMetrics counter suffixes for Prometheus
+  ([#5623](https://github.com/open-telemetry/opentelemetry-dotnet/pull/5623))
+
 ## 1.9.0-alpha.1
 
 Released 2024-May-20

src/OpenTelemetry.Exporter.Prometheus.HttpListener/Internal/PrometheusMetric.cs

@@ -27,6 +27,7 @@ public PrometheusMetric(string name, string unit, PrometheusType type, bool disa
         // consecutive `_` characters MUST be replaced with a single `_` character.
         // https://github.com/open-telemetry/opentelemetry-specification/blob/b2f923fb1650dde1f061507908b834035506a796/specification/compatibility/prometheus_and_openmetrics.md#L230-L233
         var sanitizedName = SanitizeMetricName(name);
+        var openMetricsName = SanitizeOpenMetricsName(sanitizedName);
 
         string sanitizedUnit = null;
         if (!string.IsNullOrEmpty(unit))
@@ -41,10 +42,31 @@ public PrometheusMetric(string name, string unit, PrometheusType type, bool disa
             // https://github.com/open-telemetry/opentelemetry-specification/blob/b2f923fb1650dde1f061507908b834035506a796/specification/compatibility/prometheus_and_openmetrics.md#L242-L246
             if (!sanitizedName.Contains(sanitizedUnit))
             {
-                sanitizedName = sanitizedName + "_" + sanitizedUnit;
+                sanitizedName += $"_{sanitizedUnit}";
             }
+
+            // OpenMetrics name MUST be suffixed with '_{unit}', regardless of whether the unit name appears within the text.
+            // Note that this may change in the future, however for the moment Prometheus will fail to read the metric using
+            // OpenMetrics format unless the suffix matches the unit.
+            // https://github.com/OpenObservability/OpenMetrics/blob/main/specification/OpenMetrics.md#unit
+            if (!openMetricsName.EndsWith(sanitizedUnit))
+            {
+                openMetricsName += $"_{sanitizedUnit}";
+            }
+        }
+
+        // Special case: Converting "1" to "ratio".
+        // https://github.com/open-telemetry/opentelemetry-specification/blob/b2f923fb1650dde1f061507908b834035506a796/specification/compatibility/prometheus_and_openmetrics.md#L239
+        if (type == PrometheusType.Gauge && unit == "1" && !sanitizedName.Contains("ratio"))
+        {
+            sanitizedName += "_ratio";
+            openMetricsName += "_ratio";
         }
 
+        // For TYPE, HELP and UNIT declarations for counters, the suffix '_total' is omitted in OpenMetrics format.
+        // https://github.com/OpenObservability/OpenMetrics/blob/main/specification/OpenMetrics.md#counter-1
+        this.OpenMetricsMetadataName = openMetricsName;
+
         // If the metric name for monotonic Sum metric points does not end in a suffix of `_total` a suffix of `_total` MUST be added by default, otherwise the name MUST remain unchanged.
         // Exporters SHOULD provide a configuration option to disable the addition of `_total` suffixes.
         // https://github.com/open-telemetry/opentelemetry-specification/blob/b2f923fb1650dde1f061507908b834035506a796/specification/compatibility/prometheus_and_openmetrics.md#L286
@@ -53,20 +75,25 @@ public PrometheusMetric(string name, string unit, PrometheusType type, bool disa
             sanitizedName += "_total";
         }
 
-        // Special case: Converting "1" to "ratio".
-        // https://github.com/open-telemetry/opentelemetry-specification/blob/b2f923fb1650dde1f061507908b834035506a796/specification/compatibility/prometheus_and_openmetrics.md#L239
-        if (type == PrometheusType.Gauge && unit == "1" && !sanitizedName.Contains("ratio"))
+        // For counters requested using OpenMetrics format, the MetricFamily name MUST be suffixed with '_total', regardless of the setting to disable the 'total' suffix.
+        // https://github.com/OpenObservability/OpenMetrics/blob/main/specification/OpenMetrics.md#counter-1
+        if (type == PrometheusType.Counter && !openMetricsName.EndsWith("_total"))
         {
-            sanitizedName += "_ratio";
+            openMetricsName += "_total";
         }
 
         this.Name = sanitizedName;
+        this.OpenMetricsName = openMetricsName;
         this.Unit = sanitizedUnit;
         this.Type = type;
     }
 
     public string Name { get; }
 
+    public string OpenMetricsName { get; }
+
+    public string OpenMetricsMetadataName { get; }
+
     public string Unit { get; }
 
     public PrometheusType Type { get; }
@@ -159,6 +186,16 @@ internal static string RemoveAnnotations(string unit)
         return sb.ToString();
     }
 
+    private static string SanitizeOpenMetricsName(string metricName)
+    {
+        if (metricName.EndsWith("_total"))
+        {
+            return metricName.Substring(0, metricName.Length - 6);
+        }
+
+        return metricName;
+    }
+
     private static string GetUnit(string unit)
     {
         // Dropping the portions of the Unit within brackets (e.g. {packet}). Brackets MUST NOT be included in the resulting unit. A "count of foo" is considered unitless in Prometheus.

src/OpenTelemetry.Exporter.Prometheus.HttpListener/Internal/PrometheusSerializer.cs

@@ -230,12 +230,29 @@ static string GetLabelValueString(object labelValue)
     }
 
     [MethodImpl(MethodImplOptions.AggressiveInlining)]
-    public static int WriteMetricName(byte[] buffer, int cursor, PrometheusMetric metric)
+    public static int WriteMetricName(byte[] buffer, int cursor, PrometheusMetric metric, bool openMetricsRequested)
     {
         // Metric name has already been escaped.
-        for (int i = 0; i < metric.Name.Length; i++)
+        var name = openMetricsRequested ? metric.OpenMetricsName : metric.Name;
+
+        for (int i = 0; i < name.Length; i++)
+        {
+            var ordinal = (ushort)name[i];
+            buffer[cursor++] = unchecked((byte)ordinal);
+        }
+
+        return cursor;
+    }
+
+    [MethodImpl(MethodImplOptions.AggressiveInlining)]
+    public static int WriteMetricMetadataName(byte[] buffer, int cursor, PrometheusMetric metric, bool openMetricsRequested)
+    {
+        // Metric name has already been escaped.
+        var name = openMetricsRequested ? metric.OpenMetricsMetadataName : metric.Name;
+
+        for (int i = 0; i < name.Length; i++)
         {
-            var ordinal = (ushort)metric.Name[i];
+            var ordinal = (ushort)name[i];
             buffer[cursor++] = unchecked((byte)ordinal);
         }
 
@@ -252,15 +269,15 @@ public static int WriteEof(byte[] buffer, int cursor)
     }
 
     [MethodImpl(MethodImplOptions.AggressiveInlining)]
-    public static int WriteHelpMetadata(byte[] buffer, int cursor, PrometheusMetric metric, string metricDescription)
+    public static int WriteHelpMetadata(byte[] buffer, int cursor, PrometheusMetric metric, string metricDescription, bool openMetricsRequested)
     {
         if (string.IsNullOrEmpty(metricDescription))
         {
             return cursor;
         }
 
         cursor = WriteAsciiStringNoEscape(buffer, cursor, "# HELP ");
-        cursor = WriteMetricName(buffer, cursor, metric);
+        cursor = WriteMetricMetadataName(buffer, cursor, metric, openMetricsRequested);
 
         if (!string.IsNullOrEmpty(metricDescription))
         {
@@ -274,14 +291,14 @@ public static int WriteHelpMetadata(byte[] buffer, int cursor, PrometheusMetric
     }
 
     [MethodImpl(MethodImplOptions.AggressiveInlining)]
-    public static int WriteTypeMetadata(byte[] buffer, int cursor, PrometheusMetric metric)
+    public static int WriteTypeMetadata(byte[] buffer, int cursor, PrometheusMetric metric, bool openMetricsRequested)
     {
         var metricType = MapPrometheusType(metric.Type);
 
         Debug.Assert(!string.IsNullOrEmpty(metricType), $"{nameof(metricType)} should not be null or empty.");
 
         cursor = WriteAsciiStringNoEscape(buffer, cursor, "# TYPE ");
-        cursor = WriteMetricName(buffer, cursor, metric);
+        cursor = WriteMetricMetadataName(buffer, cursor, metric, openMetricsRequested);
         buffer[cursor++] = unchecked((byte)' ');
         cursor = WriteAsciiStringNoEscape(buffer, cursor, metricType);
 
@@ -291,15 +308,15 @@ public static int WriteTypeMetadata(byte[] buffer, int cursor, PrometheusMetric
     }
 
     [MethodImpl(MethodImplOptions.AggressiveInlining)]
-    public static int WriteUnitMetadata(byte[] buffer, int cursor, PrometheusMetric metric)
+    public static int WriteUnitMetadata(byte[] buffer, int cursor, PrometheusMetric metric, bool openMetricsRequested)
     {
         if (string.IsNullOrEmpty(metric.Unit))
         {
             return cursor;
         }
 
         cursor = WriteAsciiStringNoEscape(buffer, cursor, "# UNIT ");
-        cursor = WriteMetricName(buffer, cursor, metric);
+        cursor = WriteMetricMetadataName(buffer, cursor, metric, openMetricsRequested);
 
         buffer[cursor++] = unchecked((byte)' ');
 

src/OpenTelemetry.Exporter.Prometheus.HttpListener/Internal/PrometheusSerializerExt.cs

@@ -24,9 +24,9 @@ public static bool CanWriteMetric(Metric metric)
 
     public static int WriteMetric(byte[] buffer, int cursor, Metric metric, PrometheusMetric prometheusMetric, bool openMetricsRequested = false)
     {
-        cursor = WriteTypeMetadata(buffer, cursor, prometheusMetric);
-        cursor = WriteUnitMetadata(buffer, cursor, prometheusMetric);
-        cursor = WriteHelpMetadata(buffer, cursor, prometheusMetric, metric.Description);
+        cursor = WriteTypeMetadata(buffer, cursor, prometheusMetric, openMetricsRequested);
+        cursor = WriteUnitMetadata(buffer, cursor, prometheusMetric, openMetricsRequested);
+        cursor = WriteHelpMetadata(buffer, cursor, prometheusMetric, metric.Description, openMetricsRequested);
 
         if (!metric.MetricType.IsHistogram())
         {
@@ -35,7 +35,7 @@ public static int WriteMetric(byte[] buffer, int cursor, Metric metric, Promethe
                 var timestamp = metricPoint.EndTime.ToUnixTimeMilliseconds();
 
                 // Counter and Gauge
-                cursor = WriteMetricName(buffer, cursor, prometheusMetric);
+                cursor = WriteMetricName(buffer, cursor, prometheusMetric, openMetricsRequested);
                 cursor = WriteTags(buffer, cursor, metric, metricPoint.Tags);
 
                 buffer[cursor++] = unchecked((byte)' ');
@@ -85,7 +85,7 @@ public static int WriteMetric(byte[] buffer, int cursor, Metric metric, Promethe
                 {
                     totalCount += histogramMeasurement.BucketCount;
 
-                    cursor = WriteMetricName(buffer, cursor, prometheusMetric);
+                    cursor = WriteMetricName(buffer, cursor, prometheusMetric, openMetricsRequested);
                     cursor = WriteAsciiStringNoEscape(buffer, cursor, "_bucket{");
                     cursor = WriteTags(buffer, cursor, metric, tags, writeEnclosingBraces: false);
 
@@ -111,7 +111,7 @@ public static int WriteMetric(byte[] buffer, int cursor, Metric metric, Promethe
                 }
 
                 // Histogram sum
-                cursor = WriteMetricName(buffer, cursor, prometheusMetric);
+                cursor = WriteMetricName(buffer, cursor, prometheusMetric, openMetricsRequested);
                 cursor = WriteAsciiStringNoEscape(buffer, cursor, "_sum");
                 cursor = WriteTags(buffer, cursor, metric, metricPoint.Tags);
 
@@ -125,7 +125,7 @@ public static int WriteMetric(byte[] buffer, int cursor, Metric metric, Promethe
                 buffer[cursor++] = ASCII_LINEFEED;
 
                 // Histogram count
-                cursor = WriteMetricName(buffer, cursor, prometheusMetric);
+                cursor = WriteMetricName(buffer, cursor, prometheusMetric, openMetricsRequested);
                 cursor = WriteAsciiStringNoEscape(buffer, cursor, "_count");
                 cursor = WriteTags(buffer, cursor, metric, metricPoint.Tags);
 

test/OpenTelemetry.Exporter.Prometheus.AspNetCore.Tests/PrometheusExporterMiddlewareTests.cs

@@ -264,7 +264,7 @@ public async Task PrometheusExporterMiddlewareIntegration_CanServeOpenMetricsAnd
 
         var beginTimestamp = DateTimeOffset.Now.ToUnixTimeMilliseconds();
 
-        var counter = meter.CreateCounter<double>("counter_double");
+        var counter = meter.CreateCounter<double>("counter_double", unit: "By");
         counter.Add(100.18D, tags);
         counter.Add(0.99D, tags);
 
@@ -312,7 +312,7 @@ private static async Task RunPrometheusExporterMiddlewareIntegrationTest(
 
         var beginTimestamp = DateTimeOffset.Now.ToUnixTimeMilliseconds();
 
-        var counter = meter.CreateCounter<double>("counter_double");
+        var counter = meter.CreateCounter<double>("counter_double", unit: "By");
         if (!skipMetrics)
         {
             counter.Add(100.18D, tags);
@@ -368,14 +368,16 @@ private static async Task VerifyAsync(long beginTimestamp, long endTimestamp, Ht
                     # TYPE otel_scope_info info
                     # HELP otel_scope_info Scope metadata
                     otel_scope_info{otel_scope_name="{{MeterName}}"} 1
-                    # TYPE counter_double_total counter
-                    counter_double_total{otel_scope_name="{{MeterName}}",otel_scope_version="{{MeterVersion}}",key1="value1",key2="value2"} 101.17 (\d+\.\d{3})
+                    # TYPE counter_double_bytes counter
+                    # UNIT counter_double_bytes bytes
+                    counter_double_bytes_total{otel_scope_name="{{MeterName}}",otel_scope_version="{{MeterVersion}}",key1="value1",key2="value2"} 101.17 (\d+\.\d{3})
                     # EOF
 
                     """.ReplaceLineEndings()
             : $$"""
-                    # TYPE counter_double_total counter
-                    counter_double_total{otel_scope_name="{{MeterName}}",otel_scope_version="{{MeterVersion}}",key1="value1",key2="value2"} 101.17 (\d+)
+                    # TYPE counter_double_bytes_total counter
+                    # UNIT counter_double_bytes_total bytes
+                    counter_double_bytes_total{otel_scope_name="{{MeterName}}",otel_scope_version="{{MeterVersion}}",key1="value1",key2="value2"} 101.17 (\d+)
                     # EOF
 
                     """.ReplaceLineEndings();

test/OpenTelemetry.Exporter.Prometheus.HttpListener.Tests/PrometheusHttpListenerTests.cs

@@ -202,7 +202,7 @@ private async Task RunPrometheusExporterHttpServerIntegrationTest(bool skipMetri
             new KeyValuePair<string, object>("key2", "value2"),
         };
 
-        var counter = meter.CreateCounter<double>("counter_double");
+        var counter = meter.CreateCounter<double>("counter_double", unit: "By");
         if (!skipMetrics)
         {
             counter.Add(100.18D, tags);
@@ -241,11 +241,13 @@ private async Task RunPrometheusExporterHttpServerIntegrationTest(bool skipMetri
                   + "# TYPE otel_scope_info info\n"
                   + "# HELP otel_scope_info Scope metadata\n"
                   + $"otel_scope_info{{otel_scope_name='{MeterName}'}} 1\n"
-                  + "# TYPE counter_double_total counter\n"
-                  + $"counter_double_total{{otel_scope_name='{MeterName}',otel_scope_version='{MeterVersion}',key1='value1',key2='value2'}} 101.17 (\\d+\\.\\d{{3}})\n"
+                  + "# TYPE counter_double_bytes counter\n"
+                  + "# UNIT counter_double_bytes bytes\n"
+                  + $"counter_double_bytes_total{{otel_scope_name='{MeterName}',otel_scope_version='{MeterVersion}',key1='value1',key2='value2'}} 101.17 (\\d+\\.\\d{{3}})\n"
                   + "# EOF\n"
-                : "# TYPE counter_double_total counter\n"
-                  + $"counter_double_total{{otel_scope_name='{MeterName}',otel_scope_version='{MeterVersion}',key1='value1',key2='value2'}} 101.17 (\\d+)\n"
+                : "# TYPE counter_double_bytes_total counter\n"
+                  + "# UNIT counter_double_bytes_total bytes\n"
+                  + $"counter_double_bytes_total{{otel_scope_name='{MeterName}',otel_scope_version='{MeterVersion}',key1='value1',key2='value2'}} 101.17 (\\d+)\n"
                   + "# EOF\n";
 
             Assert.Matches(("^" + expected + "$").Replace('\'', '"'), content);