support native histograms

Signed-off-by: Jan Fajerski <[email protected]>

.circleci/config.yml

@@ -6,7 +6,7 @@ executors:
   # Whenever the Go version is updated here, .promu.yml should also be updated.
   golang:
     docker:
-      - image: cimg/go:1.20
+      - image: cimg/go:1.21
 jobs:
   test:
     executor: golang

.promu.yml

@@ -2,7 +2,7 @@
 go:
     # Whenever the Go version is updated here
     # .circle/config.yml should also be updated.
-    version: 1.20
+    version: 1.21
 repository:
     path: github.com/prometheus/prom2json
 build:

README.md

@@ -82,6 +82,9 @@ Example input from stdin:
 Note that all numbers are encoded as strings. Some parsers want it
 that way. Also, Prometheus allows sample values like `NaN` or `+Inf`,
 which cannot be encoded as JSON numbers.
+Native histograms are formated similarly as [the query 
+API](https://prometheus.io/docs/prometheus/latest/querying/api/#native-histograms) 
+would return.
 
 ```json
 [
@@ -129,6 +132,40 @@ which cannot be encoded as JSON numbers.
         "value": "1063110"
       }
     ]
+  },
+  {
+    "name": "http_request_duration_seconds",
+    "type": "HISTOGRAM",
+    "help": "More HTTP request latencies in seconds.",
+    "metrics": [
+      {
+        "labels": {
+        "method": "GET",
+        },
+        "buckets": [
+          [
+            0,
+            "17.448123722644123",
+            "19.027313840043536",
+            "139"
+          ],
+          [
+            0,
+            "19.027313840043536",
+            "20.749432874416154",
+            "85"
+          ],
+          [
+            0,
+            "20.749432874416154",
+            "22.62741699796952",
+            "70"
+          ],
+        ],
+        "count": "1000",
+        "sum": "29969.50000000001"
+      }
+    ]
   }
 ]
 ```

go.mod

@@ -1,15 +1,16 @@
 module github.com/prometheus/prom2json
 
-go 1.17
+go 1.21
 
 require (
-	github.com/davecgh/go-spew v1.1.1
+	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc
 	github.com/matttproud/golang_protobuf_extensions v1.0.4
 	github.com/prometheus/client_model v0.6.1
-	github.com/prometheus/common v0.53.0
+	github.com/prometheus/common v0.54.0
+	github.com/prometheus/prometheus v0.53.0
 )
 
 require (
-	github.com/golang/protobuf v1.5.3 // indirect
-	google.golang.org/protobuf v1.33.0 // indirect
+	github.com/golang/protobuf v1.5.4 // indirect
+	google.golang.org/protobuf v1.34.1 // indirect
 )

histogram/prometheus_model.go

@@ -0,0 +1,153 @@
+// Copyright 2020 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package histogram
+
+import (
+	"fmt"
+
+	dto "github.com/prometheus/client_model/go"
+	model "github.com/prometheus/prometheus/model/histogram"
+)
+
+type APIBucket[BC model.BucketCount] struct {
+	Boundaries   uint64
+	Lower, Upper float64
+	Count        BC
+}
+
+func NewModelHistogram(ch *dto.Histogram) (*model.Histogram, *model.FloatHistogram) {
+	if ch.GetSampleCountFloat() > 0 || ch.GetZeroCountFloat() > 0 {
+		// It is a float histogram.
+		fh := model.FloatHistogram{
+			Count:           ch.GetSampleCountFloat(),
+			Sum:             ch.GetSampleSum(),
+			ZeroThreshold:   ch.GetZeroThreshold(),
+			ZeroCount:       ch.GetZeroCountFloat(),
+			Schema:          ch.GetSchema(),
+			PositiveSpans:   make([]model.Span, len(ch.GetPositiveSpan())),
+			PositiveBuckets: ch.GetPositiveCount(),
+			NegativeSpans:   make([]model.Span, len(ch.GetNegativeSpan())),
+			NegativeBuckets: ch.GetNegativeCount(),
+		}
+		for i, span := range ch.GetPositiveSpan() {
+			fh.PositiveSpans[i].Offset = span.GetOffset()
+			fh.PositiveSpans[i].Length = span.GetLength()
+		}
+		for i, span := range ch.GetNegativeSpan() {
+			fh.NegativeSpans[i].Offset = span.GetOffset()
+			fh.NegativeSpans[i].Length = span.GetLength()
+		}
+		return nil, &fh
+	}
+	h := model.Histogram{
+		Count:           ch.GetSampleCount(),
+		Sum:             ch.GetSampleSum(),
+		ZeroThreshold:   ch.GetZeroThreshold(),
+		ZeroCount:       ch.GetZeroCount(),
+		Schema:          ch.GetSchema(),
+		PositiveSpans:   make([]model.Span, len(ch.GetPositiveSpan())),
+		PositiveBuckets: ch.GetPositiveDelta(),
+		NegativeSpans:   make([]model.Span, len(ch.GetNegativeSpan())),
+		NegativeBuckets: ch.GetNegativeDelta(),
+	}
+	for i, span := range ch.GetPositiveSpan() {
+		h.PositiveSpans[i].Offset = span.GetOffset()
+		h.PositiveSpans[i].Length = span.GetLength()
+	}
+	for i, span := range ch.GetNegativeSpan() {
+		h.NegativeSpans[i].Offset = span.GetOffset()
+		h.NegativeSpans[i].Length = span.GetLength()
+	}
+	return &h, nil
+}
+
+func BucketsAsJson[BC model.BucketCount](buckets []APIBucket[BC]) [][]interface{} {
+	ret := make([][]interface{}, len(buckets))
+	for i, b := range buckets {
+		ret[i] = []interface{}{b.Boundaries, fmt.Sprintf("%v", b.Lower), fmt.Sprintf("%v", b.Upper), fmt.Sprintf("%v", b.Count)}
+	}
+	return ret
+}
+
+func GetAPIBuckets(h *model.Histogram) []APIBucket[uint64] {
+	var apiBuckets []APIBucket[uint64]
+	var nBuckets []model.Bucket[uint64]
+	for it := h.NegativeBucketIterator(); it.Next(); {
+		bucket := it.At()
+		if bucket.Count != 0 {
+			nBuckets = append(nBuckets, it.At())
+		}
+	}
+	for i := len(nBuckets) - 1; i >= 0; i-- {
+		apiBuckets = append(apiBuckets, makeBucket[uint64](nBuckets[i]))
+	}
+
+	if h.ZeroCount != 0 {
+		apiBuckets = append(apiBuckets, makeBucket[uint64](h.ZeroBucket()))
+	}
+
+	for it := h.PositiveBucketIterator(); it.Next(); {
+		bucket := it.At()
+		if bucket.Count != 0 {
+			apiBuckets = append(apiBuckets, makeBucket[uint64](bucket))
+		}
+	}
+	return apiBuckets
+}
+
+func GetAPIFloatBuckets(h *model.FloatHistogram) []APIBucket[float64] {
+	var apiBuckets []APIBucket[float64]
+	var nBuckets []model.Bucket[float64]
+	for it := h.NegativeBucketIterator(); it.Next(); {
+		bucket := it.At()
+		if bucket.Count != 0 {
+			nBuckets = append(nBuckets, it.At())
+		}
+	}
+	for i := len(nBuckets) - 1; i >= 0; i-- {
+		apiBuckets = append(apiBuckets, makeBucket[float64](nBuckets[i]))
+	}
+
+	if h.ZeroCount != 0 {
+		apiBuckets = append(apiBuckets, makeBucket[float64](h.ZeroBucket()))
+	}
+
+	for it := h.PositiveBucketIterator(); it.Next(); {
+		bucket := it.At()
+		if bucket.Count != 0 {
+			apiBuckets = append(apiBuckets, makeBucket[float64](bucket))
+		}
+	}
+	return apiBuckets
+}
+
+func makeBucket[BC model.BucketCount](bucket model.Bucket[BC]) APIBucket[BC] {
+	boundaries := uint64(2) // () Exclusive on both sides AKA open interval.
+	if bucket.LowerInclusive {
+		if bucket.UpperInclusive {
+			boundaries = 3 // [] Inclusive on both sides AKA closed interval.
+		} else {
+			boundaries = 1 // [) Inclusive only on lower end AKA right open.
+		}
+	} else {
+		if bucket.UpperInclusive {
+			boundaries = 0 // (] Inclusive only on upper end AKA left open.
+		}
+	}
+	return APIBucket[BC]{
+		Boundaries: boundaries,
+		Lower:      bucket.Lower,
+		Upper:      bucket.Upper,
+		Count:      bucket.Count,
+	}
+}

prom2json.go

@@ -23,6 +23,7 @@ import (
 	"github.com/prometheus/common/expfmt"
 
 	dto "github.com/prometheus/client_model/go"
+	"github.com/prometheus/prom2json/histogram"
 )
 
 const acceptHeader = `application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=delimited;q=0.7,text/plain;version=0.0.4;q=0.3`
@@ -56,7 +57,7 @@ type Summary struct {
 type Histogram struct {
 	Labels      map[string]string `json:"labels,omitempty"`
 	TimestampMs string            `json:"timestamp_ms,omitempty"`
-	Buckets     map[string]string `json:"buckets,omitempty"`
+	Buckets     interface{}       `json:"buckets,omitempty"`
 	Count       string            `json:"count"`
 	Sum         string            `json:"sum"`
 }
@@ -81,13 +82,7 @@ func NewFamily(dtoMF *dto.MetricFamily) *Family {
 				Sum:         fmt.Sprint(m.GetSummary().GetSampleSum()),
 			}
 		case dto.MetricType_HISTOGRAM:
-			mf.Metrics[i] = Histogram{
-				Labels:      makeLabels(m),
-				TimestampMs: makeTimestamp(m),
-				Buckets:     makeBuckets(m),
-				Count:       fmt.Sprint(m.GetHistogram().GetSampleCount()),
-				Sum:         fmt.Sprint(m.GetHistogram().GetSampleSum()),
-			}
+			mf.Metrics[i] = makeHistogram(m)
 		default:
 			mf.Metrics[i] = Metric{
 				Labels:      makeLabels(m),
@@ -112,6 +107,27 @@ func getValue(m *dto.Metric) float64 {
 	}
 }
 
+func makeHistogram(m *dto.Metric) Histogram {
+	hist := Histogram{
+		Labels:      makeLabels(m),
+		TimestampMs: makeTimestamp(m),
+		Count:       fmt.Sprint(m.GetHistogram().GetSampleCount()),
+		Sum:         fmt.Sprint(m.GetHistogram().GetSampleSum()),
+	}
+	if b := makeBuckets(m); len(b) > 0 {
+		hist.Buckets = b
+	} else {
+		h, fh := histogram.NewModelHistogram(m.GetHistogram())
+		if h == nil {
+			// float histogram
+			hist.Buckets = histogram.BucketsAsJson[float64](histogram.GetAPIFloatBuckets(fh))
+		} else {
+			hist.Buckets = histogram.BucketsAsJson[uint64](histogram.GetAPIBuckets(h))
+		}
+	}
+	return hist
+}
+
 func makeLabels(m *dto.Metric) map[string]string {
 	result := map[string]string{}
 	for _, lp := range m.Label {