From 25319d9118dc514888d0c20cfe6c08469eb5716c Mon Sep 17 00:00:00 2001 From: beorn7 Date: Thu, 29 Sep 2022 18:33:02 +0200 Subject: [PATCH 1/5] protobuf: Improve naming of `oneof` fields The name after the `oneof` doesn't show up in the explicit getter for a particular field, e.g. for the sum of a `HistogramValue` as a double, the getter is called `GetDoubleValue` in Go. Nothing in this getter tells you that you are getting the sum. For one, this is confusing. But you will also get a name collision if any of the other fields becomes a `oneof` (e.g. the count for a float histogram). With this commit, the getter will be called `GetDoubleSum` etc. Signed-off-by: beorn7 --- proto/openmetrics_data_model.proto | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/proto/openmetrics_data_model.proto b/proto/openmetrics_data_model.proto index a95942d..3e811b1 100644 --- a/proto/openmetrics_data_model.proto +++ b/proto/openmetrics_data_model.proto @@ -110,8 +110,8 @@ message GaugeValue { message CounterValue { // Required. oneof total { - double double_value = 1; - uint64 int_value = 2; + double double_total = 1; + uint64 int_total = 2; } // The time values began being collected for this counter. @@ -126,8 +126,8 @@ message CounterValue { message HistogramValue { // Optional. oneof sum { - double double_value = 1; - int64 int_value = 2; + double double_sum = 1; + int64 int_sum = 2; } // Optional. @@ -190,8 +190,8 @@ message InfoValue { message SummaryValue { // Optional. oneof sum { - double double_value = 1; - int64 int_value = 2; + double double_sum = 1; + int64 int_sum = 2; } // Optional. From da0400be5196692f8e0c379405bb41b8d812dd75 Mon Sep 17 00:00:00 2001 From: beorn7 Date: Wed, 5 Oct 2022 16:06:37 +0200 Subject: [PATCH 2/5] protobuf: Add "float histogram" capability There are rare use cases of "weighted" or "scaled" histogram, where the total count and the count in buckets are actually floating point values. The need for this "float histogram" becomes more obvious with the introduction of Native Histograms. With Histograms being first class citizens also within Prometheus, a recording rule that outputs Histograms will generally output Histograms where the counts are floating point numbers again (because PromQL operates on floats only, and because the result of a `rate` operation is interpolated and therefore not integral anymore, even if the input is). If such a result of a recording rule is federated, we need to represent "float histograms" in the exposition format. This commit introduces the "float histogram" capability to the conventional Histogram, which is also needed to add Native Histograms later. Signed-off-by: beorn7 --- proto/openmetrics_data_model.proto | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/proto/openmetrics_data_model.proto b/proto/openmetrics_data_model.proto index 3e811b1..564b3ea 100644 --- a/proto/openmetrics_data_model.proto +++ b/proto/openmetrics_data_model.proto @@ -131,7 +131,10 @@ message HistogramValue { } // Optional. - uint64 count = 3; + oneof count { + double double_count = 6; + uint64 int_count = 3; + } // The time values began being collected for this histogram. // Optional. @@ -144,7 +147,10 @@ message HistogramValue { // with an optional exemplar. message Bucket { // Required. - uint64 count = 1; + oneof count { + double double_count = 4; + uint64 int_count = 1; + } // Optional. double upper_bound = 2; From f8b544129b13d38d73b2ddbf84ce2ec6af145e41 Mon Sep 17 00:00:00 2001 From: beorn7 Date: Wed, 5 Oct 2022 16:55:45 +0200 Subject: [PATCH 3/5] protobuf: Add native histogram support This is still experimental. It is not clear yet how to represent native histograms in the text format, but with this commit, implementers of OpenMetrics using the protobuf format can already gather experience with native histograms. Signed-off-by: beorn7 --- proto/openmetrics_data_model.proto | 56 ++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/proto/openmetrics_data_model.proto b/proto/openmetrics_data_model.proto index 564b3ea..f96a902 100644 --- a/proto/openmetrics_data_model.proto +++ b/proto/openmetrics_data_model.proto @@ -158,6 +158,62 @@ message HistogramValue { // Optional. Exemplar exemplar = 3; } + + // Everything below here is for native histograms (also known as sparse histograms). + + // schema defines the bucket schema. Currently, valid numbers are -4 <= n <= 8. + // They are all for base-2 bucket schemas, where 1 is a bucket boundary in each case, and + // then each power of two is divided into 2^n logarithmic buckets. + // Or in other words, each bucket boundary is the previous boundary times 2^(2^-n). + // In the future, more bucket schemas may be added using numbers < -4 or > 8. + // Optional. + sint32 schema = 7; + + // Breadth of the zero bucket. + // Optional. + double zero_threshold = 8; + + // Count in zero bucket. + // Optional. + oneof zero_count { + double double_zero_count = 9; + uint64 int_zero_count = 10; + } + + // Negative buckets for the native histogram. + // Optional. + repeated BucketSpan negative_span = 11; + // Use either "negative_delta" or "negative_count", the former for + // regular histograms with integer counts, the latter for float + // histograms. + repeated sint64 negative_delta = 12; // Count delta of each bucket compared to previous one (or to zero for 1st bucket). + repeated double negative_count = 13; // Absolute count of each bucket. + + // Positive buckets for the native histogram. + // Optional. + repeated BucketSpan positive_span = 14; + // Use either "positive_delta" or "positive_count", the former for + // regular histograms with integer counts, the latter for float + // histograms. + repeated sint64 positive_delta = 15; // Count delta of each bucket compared to previous one (or to zero for 1st bucket). + repeated double positive_count = 16; // Absolute count of each bucket. + + // A BucketSpan defines a number of consecutive buckets in a native + // histogram with their offset. Logically, it would be more + // straightforward to include the bucket counts in the Span. However, + // the protobuf representation is more compact in the way the data is + // structured here (with all the buckets in a single array separate + // from the Spans). + message BucketSpan { + + // Gap to previous span, or starting point for 1st span (which can be negative). + // Required. + sint32 offset = 1; + + // Length of consecutive buckets. + // Required. + uint32 length = 2; + } } message Exemplar { From cabfdb2f3773b670101da201ac5326a4228e1838 Mon Sep 17 00:00:00 2001 From: beorn7 Date: Wed, 21 Feb 2024 14:55:54 +0100 Subject: [PATCH 4/5] proto: Add exemplars for native histograms This commit mirrors https://github.com/prometheus/client_model/pull/80 . Signed-off-by: beorn7 --- proto/openmetrics_data_model.proto | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/proto/openmetrics_data_model.proto b/proto/openmetrics_data_model.proto index f96a902..4c3c527 100644 --- a/proto/openmetrics_data_model.proto +++ b/proto/openmetrics_data_model.proto @@ -198,6 +198,10 @@ message HistogramValue { repeated sint64 positive_delta = 15; // Count delta of each bucket compared to previous one (or to zero for 1st bucket). repeated double positive_count = 16; // Absolute count of each bucket. + // Only used for native histograms. These exemplars MUST have a timestamp. + // Optional. + repeated Exemplar exemplars = 17; + // A BucketSpan defines a number of consecutive buckets in a native // histogram with their offset. Logically, it would be more // straightforward to include the bucket counts in the Span. However, From b8199dba2341556770652157663aa6d05c35216a Mon Sep 17 00:00:00 2001 From: beorn7 Date: Wed, 21 Feb 2024 15:08:18 +0100 Subject: [PATCH 5/5] Fix Makefile and proto file to enable Go code generation The proto file was missing the `go_package` option. Furthermore, the generated Go file was put into the `bin` directory, which is weird. Signed-off-by: beorn7 --- Makefile | 2 +- proto/openmetrics_data_model.proto | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index b8791eb..26974ed 100644 --- a/Makefile +++ b/Makefile @@ -31,7 +31,7 @@ test_open_metrics_validator: .PHONY: proto_go proto_go: setup - protoc --go_out=$(BUILD) --go_opt=paths=source_relative ./proto/*.proto + protoc --go_out=. --go_opt=paths=source_relative ./proto/*.proto .PHONY: setup setup: diff --git a/proto/openmetrics_data_model.proto b/proto/openmetrics_data_model.proto index 4c3c527..34a9c79 100644 --- a/proto/openmetrics_data_model.proto +++ b/proto/openmetrics_data_model.proto @@ -5,6 +5,8 @@ syntax = "proto3"; // All string fields MUST be UTF-8 encoded strings. package openmetrics; +option go_package = "github.com/OpenObservability/OpenMetrics/proto;openmetrics"; + import "google/protobuf/timestamp.proto"; // The top-level container type that is encoded and sent over the wire.