From ca3322536fd4121048ce5e880719117c425584fc Mon Sep 17 00:00:00 2001 From: Ivan <2103732+codebien@users.noreply.github.com> Date: Wed, 5 Jul 2023 17:35:00 +0200 Subject: [PATCH 1/8] cloudv2/pbcloud: Compact histogram --- output/cloud/expv2/pbcloud/metric.pb.go | 240 ++++++++++++++++-------- output/cloud/expv2/pbcloud/metric.proto | 34 ++-- 2 files changed, 180 insertions(+), 94 deletions(-) diff --git a/output/cloud/expv2/pbcloud/metric.pb.go b/output/cloud/expv2/pbcloud/metric.pb.go index 76e053b5086..6b472355419 100644 --- a/output/cloud/expv2/pbcloud/metric.pb.go +++ b/output/cloud/expv2/pbcloud/metric.pb.go @@ -797,6 +797,65 @@ func (x *RateValue) GetTotalCount() uint32 { return 0 } +// BucketSpan represents the consecutive and significant (non-zero) buckets. +// The concept is imported from the Prometheus' native histogram. +// It should simplify an eventual transition to Prometheus. +// https://github.com/prometheus/prometheus/blob/f399f386cef3b19c48157e9678a4f50997db3f41/prompb/io/prometheus/client/metrics.proto#L115-L124 +type BucketSpan struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Offset uint32 `protobuf:"varint,1,opt,name=offset,proto3" json:"offset,omitempty"` // Gap to previous span, or starting point for 1st span. + Length uint32 `protobuf:"varint,2,opt,name=length,proto3" json:"length,omitempty"` // Length of the next consecutive buckets. +} + +func (x *BucketSpan) Reset() { + *x = BucketSpan{} + if protoimpl.UnsafeEnabled { + mi := &file_metric_proto_msgTypes[11] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *BucketSpan) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*BucketSpan) ProtoMessage() {} + +func (x *BucketSpan) ProtoReflect() protoreflect.Message { + mi := &file_metric_proto_msgTypes[11] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use BucketSpan.ProtoReflect.Descriptor instead. +func (*BucketSpan) Descriptor() ([]byte, []int) { + return file_metric_proto_rawDescGZIP(), []int{11} +} + +func (x *BucketSpan) GetOffset() uint32 { + if x != nil { + return x.Offset + } + return 0 +} + +func (x *BucketSpan) GetLength() uint32 { + if x != nil { + return x.Length + } + return 0 +} + type TrendHdrValue struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -804,32 +863,31 @@ type TrendHdrValue struct { // Required. Time *timestamppb.Timestamp `protobuf:"bytes,1,opt,name=time,proto3" json:"time,omitempty"` - // buckets - Counters []uint32 `protobuf:"varint,2,rep,packed,name=counters,proto3" json:"counters,omitempty"` - // index of first bucket in `counters` - LowerCounterIndex uint32 `protobuf:"varint,3,opt,name=lower_counter_index,json=lowerCounterIndex,proto3" json:"lower_counter_index,omitempty"` + // positive buckets starting from bucket 1, each counter is an absolute value. + Counters []uint32 `protobuf:"varint,2,rep,packed,name=counters,proto3" json:"counters,omitempty"` + Spans []*BucketSpan `protobuf:"bytes,3,rep,name=spans,proto3" json:"spans,omitempty"` // sum of observations - Sum float64 `protobuf:"fixed64,4,opt,name=sum,proto3" json:"sum,omitempty"` + Sum float64 `protobuf:"fixed64,5,opt,name=sum,proto3" json:"sum,omitempty"` // count of observations - Count uint32 `protobuf:"varint,5,opt,name=count,proto3" json:"count,omitempty"` + Count uint32 `protobuf:"varint,6,opt,name=count,proto3" json:"count,omitempty"` // smallest and largest observed value - MinValue float64 `protobuf:"fixed64,6,opt,name=min_value,json=minValue,proto3" json:"min_value,omitempty"` - MaxValue float64 `protobuf:"fixed64,7,opt,name=max_value,json=maxValue,proto3" json:"max_value,omitempty"` - // counters for zero and infinity buckets - ExtraLowValuesCounter *uint32 `protobuf:"varint,8,opt,name=extra_low_values_counter,json=extraLowValuesCounter,proto3,oneof" json:"extra_low_values_counter,omitempty"` - ExtraHighValuesCounter *uint32 `protobuf:"varint,9,opt,name=extra_high_values_counter,json=extraHighValuesCounter,proto3,oneof" json:"extra_high_values_counter,omitempty"` + MinValue float64 `protobuf:"fixed64,7,opt,name=min_value,json=minValue,proto3" json:"min_value,omitempty"` + MaxValue float64 `protobuf:"fixed64,8,opt,name=max_value,json=maxValue,proto3" json:"max_value,omitempty"` + // counters for zero- and infinity-buckets + ExtraLowValuesCounter *uint32 `protobuf:"varint,9,opt,name=extra_low_values_counter,json=extraLowValuesCounter,proto3,oneof" json:"extra_low_values_counter,omitempty"` + ExtraHighValuesCounter *uint32 `protobuf:"varint,10,opt,name=extra_high_values_counter,json=extraHighValuesCounter,proto3,oneof" json:"extra_high_values_counter,omitempty"` // histogram parameter - value multiplier aka smallest value // default = 1.0 - MinResolution *float64 `protobuf:"fixed64,10,opt,name=min_resolution,json=minResolution,proto3,oneof" json:"min_resolution,omitempty"` + MinResolution *float64 `protobuf:"fixed64,11,opt,name=min_resolution,json=minResolution,proto3,oneof" json:"min_resolution,omitempty"` // histogram parameter - number of significant digits used to calculate buckets formula // default = 2 - SignificantDigits *uint32 `protobuf:"varint,11,opt,name=significant_digits,json=significantDigits,proto3,oneof" json:"significant_digits,omitempty"` + SignificantDigits *uint32 `protobuf:"varint,12,opt,name=significant_digits,json=significantDigits,proto3,oneof" json:"significant_digits,omitempty"` } func (x *TrendHdrValue) Reset() { *x = TrendHdrValue{} if protoimpl.UnsafeEnabled { - mi := &file_metric_proto_msgTypes[11] + mi := &file_metric_proto_msgTypes[12] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -842,7 +900,7 @@ func (x *TrendHdrValue) String() string { func (*TrendHdrValue) ProtoMessage() {} func (x *TrendHdrValue) ProtoReflect() protoreflect.Message { - mi := &file_metric_proto_msgTypes[11] + mi := &file_metric_proto_msgTypes[12] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -855,7 +913,7 @@ func (x *TrendHdrValue) ProtoReflect() protoreflect.Message { // Deprecated: Use TrendHdrValue.ProtoReflect.Descriptor instead. func (*TrendHdrValue) Descriptor() ([]byte, []int) { - return file_metric_proto_rawDescGZIP(), []int{11} + return file_metric_proto_rawDescGZIP(), []int{12} } func (x *TrendHdrValue) GetTime() *timestamppb.Timestamp { @@ -872,11 +930,11 @@ func (x *TrendHdrValue) GetCounters() []uint32 { return nil } -func (x *TrendHdrValue) GetLowerCounterIndex() uint32 { +func (x *TrendHdrValue) GetSpans() []*BucketSpan { if x != nil { - return x.LowerCounterIndex + return x.Spans } - return 0 + return nil } func (x *TrendHdrValue) GetSum() float64 { @@ -1020,54 +1078,58 @@ var file_metric_proto_rawDesc = []byte{ 0x6f, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0c, 0x6e, 0x6f, 0x6e, 0x7a, 0x65, 0x72, 0x6f, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x1f, 0x0a, 0x0b, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, - 0x52, 0x0a, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x22, 0xb0, 0x04, 0x0a, - 0x0d, 0x54, 0x72, 0x65, 0x6e, 0x64, 0x48, 0x64, 0x72, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x2e, - 0x0a, 0x04, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, - 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, - 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x04, 0x74, 0x69, 0x6d, 0x65, 0x12, 0x1a, - 0x0a, 0x08, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x65, 0x72, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0d, - 0x52, 0x08, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x65, 0x72, 0x73, 0x12, 0x2e, 0x0a, 0x13, 0x6c, 0x6f, - 0x77, 0x65, 0x72, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x65, 0x72, 0x5f, 0x69, 0x6e, 0x64, 0x65, - 0x78, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x11, 0x6c, 0x6f, 0x77, 0x65, 0x72, 0x43, 0x6f, - 0x75, 0x6e, 0x74, 0x65, 0x72, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x12, 0x10, 0x0a, 0x03, 0x73, 0x75, - 0x6d, 0x18, 0x04, 0x20, 0x01, 0x28, 0x01, 0x52, 0x03, 0x73, 0x75, 0x6d, 0x12, 0x14, 0x0a, 0x05, - 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x05, 0x63, 0x6f, 0x75, - 0x6e, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x6d, 0x69, 0x6e, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, - 0x06, 0x20, 0x01, 0x28, 0x01, 0x52, 0x08, 0x6d, 0x69, 0x6e, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, - 0x1b, 0x0a, 0x09, 0x6d, 0x61, 0x78, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x07, 0x20, 0x01, - 0x28, 0x01, 0x52, 0x08, 0x6d, 0x61, 0x78, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x3c, 0x0a, 0x18, - 0x65, 0x78, 0x74, 0x72, 0x61, 0x5f, 0x6c, 0x6f, 0x77, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x73, - 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x65, 0x72, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0d, 0x48, 0x00, - 0x52, 0x15, 0x65, 0x78, 0x74, 0x72, 0x61, 0x4c, 0x6f, 0x77, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x73, - 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x65, 0x72, 0x88, 0x01, 0x01, 0x12, 0x3e, 0x0a, 0x19, 0x65, 0x78, - 0x74, 0x72, 0x61, 0x5f, 0x68, 0x69, 0x67, 0x68, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x5f, - 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x65, 0x72, 0x18, 0x09, 0x20, 0x01, 0x28, 0x0d, 0x48, 0x01, 0x52, - 0x16, 0x65, 0x78, 0x74, 0x72, 0x61, 0x48, 0x69, 0x67, 0x68, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x73, - 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x65, 0x72, 0x88, 0x01, 0x01, 0x12, 0x2a, 0x0a, 0x0e, 0x6d, 0x69, - 0x6e, 0x5f, 0x72, 0x65, 0x73, 0x6f, 0x6c, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x0a, 0x20, 0x01, - 0x28, 0x01, 0x48, 0x02, 0x52, 0x0d, 0x6d, 0x69, 0x6e, 0x52, 0x65, 0x73, 0x6f, 0x6c, 0x75, 0x74, - 0x69, 0x6f, 0x6e, 0x88, 0x01, 0x01, 0x12, 0x32, 0x0a, 0x12, 0x73, 0x69, 0x67, 0x6e, 0x69, 0x66, - 0x69, 0x63, 0x61, 0x6e, 0x74, 0x5f, 0x64, 0x69, 0x67, 0x69, 0x74, 0x73, 0x18, 0x0b, 0x20, 0x01, - 0x28, 0x0d, 0x48, 0x03, 0x52, 0x11, 0x73, 0x69, 0x67, 0x6e, 0x69, 0x66, 0x69, 0x63, 0x61, 0x6e, - 0x74, 0x44, 0x69, 0x67, 0x69, 0x74, 0x73, 0x88, 0x01, 0x01, 0x42, 0x1b, 0x0a, 0x19, 0x5f, 0x65, - 0x78, 0x74, 0x72, 0x61, 0x5f, 0x6c, 0x6f, 0x77, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x5f, - 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x65, 0x72, 0x42, 0x1c, 0x0a, 0x1a, 0x5f, 0x65, 0x78, 0x74, 0x72, - 0x61, 0x5f, 0x68, 0x69, 0x67, 0x68, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x5f, 0x63, 0x6f, - 0x75, 0x6e, 0x74, 0x65, 0x72, 0x42, 0x11, 0x0a, 0x0f, 0x5f, 0x6d, 0x69, 0x6e, 0x5f, 0x72, 0x65, - 0x73, 0x6f, 0x6c, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x42, 0x15, 0x0a, 0x13, 0x5f, 0x73, 0x69, 0x67, - 0x6e, 0x69, 0x66, 0x69, 0x63, 0x61, 0x6e, 0x74, 0x5f, 0x64, 0x69, 0x67, 0x69, 0x74, 0x73, 0x2a, - 0x86, 0x01, 0x0a, 0x0a, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x54, 0x79, 0x70, 0x65, 0x12, 0x1b, - 0x0a, 0x17, 0x4d, 0x45, 0x54, 0x52, 0x49, 0x43, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x55, 0x4e, - 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x17, 0x0a, 0x13, 0x4d, - 0x45, 0x54, 0x52, 0x49, 0x43, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x43, 0x4f, 0x55, 0x4e, 0x54, - 0x45, 0x52, 0x10, 0x01, 0x12, 0x15, 0x0a, 0x11, 0x4d, 0x45, 0x54, 0x52, 0x49, 0x43, 0x5f, 0x54, - 0x59, 0x50, 0x45, 0x5f, 0x47, 0x41, 0x55, 0x47, 0x45, 0x10, 0x02, 0x12, 0x14, 0x0a, 0x10, 0x4d, - 0x45, 0x54, 0x52, 0x49, 0x43, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x52, 0x41, 0x54, 0x45, 0x10, - 0x03, 0x12, 0x15, 0x0a, 0x11, 0x4d, 0x45, 0x54, 0x52, 0x49, 0x43, 0x5f, 0x54, 0x59, 0x50, 0x45, - 0x5f, 0x54, 0x52, 0x45, 0x4e, 0x44, 0x10, 0x04, 0x42, 0x28, 0x5a, 0x26, 0x67, 0x6f, 0x2e, 0x6b, - 0x36, 0x2e, 0x69, 0x6f, 0x2f, 0x6b, 0x36, 0x2f, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x2f, 0x63, - 0x6c, 0x6f, 0x75, 0x64, 0x2f, 0x65, 0x78, 0x70, 0x76, 0x32, 0x2f, 0x70, 0x62, 0x63, 0x6c, 0x6f, - 0x75, 0x64, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x52, 0x0a, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x22, 0x3c, 0x0a, 0x0a, + 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x53, 0x70, 0x61, 0x6e, 0x12, 0x16, 0x0a, 0x06, 0x6f, 0x66, + 0x66, 0x73, 0x65, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x06, 0x6f, 0x66, 0x66, 0x73, + 0x65, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x0d, 0x52, 0x06, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x22, 0xab, 0x04, 0x0a, 0x0d, 0x54, + 0x72, 0x65, 0x6e, 0x64, 0x48, 0x64, 0x72, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x2e, 0x0a, 0x04, + 0x74, 0x69, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, + 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, + 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x04, 0x74, 0x69, 0x6d, 0x65, 0x12, 0x1a, 0x0a, 0x08, + 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x65, 0x72, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0d, 0x52, 0x08, + 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x65, 0x72, 0x73, 0x12, 0x29, 0x0a, 0x05, 0x73, 0x70, 0x61, 0x6e, + 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, + 0x73, 0x2e, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x53, 0x70, 0x61, 0x6e, 0x52, 0x05, 0x73, 0x70, + 0x61, 0x6e, 0x73, 0x12, 0x10, 0x0a, 0x03, 0x73, 0x75, 0x6d, 0x18, 0x05, 0x20, 0x01, 0x28, 0x01, + 0x52, 0x03, 0x73, 0x75, 0x6d, 0x12, 0x14, 0x0a, 0x05, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x06, + 0x20, 0x01, 0x28, 0x0d, 0x52, 0x05, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x6d, + 0x69, 0x6e, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x01, 0x52, 0x08, + 0x6d, 0x69, 0x6e, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x6d, 0x61, 0x78, 0x5f, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x08, 0x20, 0x01, 0x28, 0x01, 0x52, 0x08, 0x6d, 0x61, 0x78, + 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x3c, 0x0a, 0x18, 0x65, 0x78, 0x74, 0x72, 0x61, 0x5f, 0x6c, + 0x6f, 0x77, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x65, + 0x72, 0x18, 0x09, 0x20, 0x01, 0x28, 0x0d, 0x48, 0x00, 0x52, 0x15, 0x65, 0x78, 0x74, 0x72, 0x61, + 0x4c, 0x6f, 0x77, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x65, 0x72, + 0x88, 0x01, 0x01, 0x12, 0x3e, 0x0a, 0x19, 0x65, 0x78, 0x74, 0x72, 0x61, 0x5f, 0x68, 0x69, 0x67, + 0x68, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x65, 0x72, + 0x18, 0x0a, 0x20, 0x01, 0x28, 0x0d, 0x48, 0x01, 0x52, 0x16, 0x65, 0x78, 0x74, 0x72, 0x61, 0x48, + 0x69, 0x67, 0x68, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x65, 0x72, + 0x88, 0x01, 0x01, 0x12, 0x2a, 0x0a, 0x0e, 0x6d, 0x69, 0x6e, 0x5f, 0x72, 0x65, 0x73, 0x6f, 0x6c, + 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x01, 0x48, 0x02, 0x52, 0x0d, 0x6d, + 0x69, 0x6e, 0x52, 0x65, 0x73, 0x6f, 0x6c, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x88, 0x01, 0x01, 0x12, + 0x32, 0x0a, 0x12, 0x73, 0x69, 0x67, 0x6e, 0x69, 0x66, 0x69, 0x63, 0x61, 0x6e, 0x74, 0x5f, 0x64, + 0x69, 0x67, 0x69, 0x74, 0x73, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x0d, 0x48, 0x03, 0x52, 0x11, 0x73, + 0x69, 0x67, 0x6e, 0x69, 0x66, 0x69, 0x63, 0x61, 0x6e, 0x74, 0x44, 0x69, 0x67, 0x69, 0x74, 0x73, + 0x88, 0x01, 0x01, 0x42, 0x1b, 0x0a, 0x19, 0x5f, 0x65, 0x78, 0x74, 0x72, 0x61, 0x5f, 0x6c, 0x6f, + 0x77, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x65, 0x72, + 0x42, 0x1c, 0x0a, 0x1a, 0x5f, 0x65, 0x78, 0x74, 0x72, 0x61, 0x5f, 0x68, 0x69, 0x67, 0x68, 0x5f, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x65, 0x72, 0x42, 0x11, + 0x0a, 0x0f, 0x5f, 0x6d, 0x69, 0x6e, 0x5f, 0x72, 0x65, 0x73, 0x6f, 0x6c, 0x75, 0x74, 0x69, 0x6f, + 0x6e, 0x42, 0x15, 0x0a, 0x13, 0x5f, 0x73, 0x69, 0x67, 0x6e, 0x69, 0x66, 0x69, 0x63, 0x61, 0x6e, + 0x74, 0x5f, 0x64, 0x69, 0x67, 0x69, 0x74, 0x73, 0x2a, 0x86, 0x01, 0x0a, 0x0a, 0x4d, 0x65, 0x74, + 0x72, 0x69, 0x63, 0x54, 0x79, 0x70, 0x65, 0x12, 0x1b, 0x0a, 0x17, 0x4d, 0x45, 0x54, 0x52, 0x49, + 0x43, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, + 0x45, 0x44, 0x10, 0x00, 0x12, 0x17, 0x0a, 0x13, 0x4d, 0x45, 0x54, 0x52, 0x49, 0x43, 0x5f, 0x54, + 0x59, 0x50, 0x45, 0x5f, 0x43, 0x4f, 0x55, 0x4e, 0x54, 0x45, 0x52, 0x10, 0x01, 0x12, 0x15, 0x0a, + 0x11, 0x4d, 0x45, 0x54, 0x52, 0x49, 0x43, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x41, 0x55, + 0x47, 0x45, 0x10, 0x02, 0x12, 0x14, 0x0a, 0x10, 0x4d, 0x45, 0x54, 0x52, 0x49, 0x43, 0x5f, 0x54, + 0x59, 0x50, 0x45, 0x5f, 0x52, 0x41, 0x54, 0x45, 0x10, 0x03, 0x12, 0x15, 0x0a, 0x11, 0x4d, 0x45, + 0x54, 0x52, 0x49, 0x43, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x54, 0x52, 0x45, 0x4e, 0x44, 0x10, + 0x04, 0x42, 0x28, 0x5a, 0x26, 0x67, 0x6f, 0x2e, 0x6b, 0x36, 0x2e, 0x69, 0x6f, 0x2f, 0x6b, 0x36, + 0x2f, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x2f, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2f, 0x65, 0x78, + 0x70, 0x76, 0x32, 0x2f, 0x70, 0x62, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x62, 0x06, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x33, } var ( @@ -1083,7 +1145,7 @@ func file_metric_proto_rawDescGZIP() []byte { } var file_metric_proto_enumTypes = make([]protoimpl.EnumInfo, 1) -var file_metric_proto_msgTypes = make([]protoimpl.MessageInfo, 12) +var file_metric_proto_msgTypes = make([]protoimpl.MessageInfo, 13) var file_metric_proto_goTypes = []interface{}{ (MetricType)(0), // 0: metrics.MetricType (*MetricSet)(nil), // 1: metrics.MetricSet @@ -1097,8 +1159,9 @@ var file_metric_proto_goTypes = []interface{}{ (*CounterValue)(nil), // 9: metrics.CounterValue (*GaugeValue)(nil), // 10: metrics.GaugeValue (*RateValue)(nil), // 11: metrics.RateValue - (*TrendHdrValue)(nil), // 12: metrics.TrendHdrValue - (*timestamppb.Timestamp)(nil), // 13: google.protobuf.Timestamp + (*BucketSpan)(nil), // 12: metrics.BucketSpan + (*TrendHdrValue)(nil), // 13: metrics.TrendHdrValue + (*timestamppb.Timestamp)(nil), // 14: google.protobuf.Timestamp } var file_metric_proto_depIdxs = []int32{ 2, // 0: metrics.MetricSet.metrics:type_name -> metrics.Metric @@ -1112,16 +1175,17 @@ var file_metric_proto_depIdxs = []int32{ 9, // 8: metrics.CounterSamples.values:type_name -> metrics.CounterValue 10, // 9: metrics.GaugeSamples.values:type_name -> metrics.GaugeValue 11, // 10: metrics.RateSamples.values:type_name -> metrics.RateValue - 12, // 11: metrics.TrendHdrSamples.values:type_name -> metrics.TrendHdrValue - 13, // 12: metrics.CounterValue.time:type_name -> google.protobuf.Timestamp - 13, // 13: metrics.GaugeValue.time:type_name -> google.protobuf.Timestamp - 13, // 14: metrics.RateValue.time:type_name -> google.protobuf.Timestamp - 13, // 15: metrics.TrendHdrValue.time:type_name -> google.protobuf.Timestamp - 16, // [16:16] is the sub-list for method output_type - 16, // [16:16] is the sub-list for method input_type - 16, // [16:16] is the sub-list for extension type_name - 16, // [16:16] is the sub-list for extension extendee - 0, // [0:16] is the sub-list for field type_name + 13, // 11: metrics.TrendHdrSamples.values:type_name -> metrics.TrendHdrValue + 14, // 12: metrics.CounterValue.time:type_name -> google.protobuf.Timestamp + 14, // 13: metrics.GaugeValue.time:type_name -> google.protobuf.Timestamp + 14, // 14: metrics.RateValue.time:type_name -> google.protobuf.Timestamp + 14, // 15: metrics.TrendHdrValue.time:type_name -> google.protobuf.Timestamp + 12, // 16: metrics.TrendHdrValue.spans:type_name -> metrics.BucketSpan + 17, // [17:17] is the sub-list for method output_type + 17, // [17:17] is the sub-list for method input_type + 17, // [17:17] is the sub-list for extension type_name + 17, // [17:17] is the sub-list for extension extendee + 0, // [0:17] is the sub-list for field type_name } func init() { file_metric_proto_init() } @@ -1263,6 +1327,18 @@ func file_metric_proto_init() { } } file_metric_proto_msgTypes[11].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*BucketSpan); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_metric_proto_msgTypes[12].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*TrendHdrValue); i { case 0: return &v.state @@ -1281,14 +1357,14 @@ func file_metric_proto_init() { (*TimeSeries_RateSamples)(nil), (*TimeSeries_TrendHdrSamples)(nil), } - file_metric_proto_msgTypes[11].OneofWrappers = []interface{}{} + file_metric_proto_msgTypes[12].OneofWrappers = []interface{}{} type x struct{} out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_metric_proto_rawDesc, NumEnums: 1, - NumMessages: 12, + NumMessages: 13, NumExtensions: 0, NumServices: 0, }, diff --git a/output/cloud/expv2/pbcloud/metric.proto b/output/cloud/expv2/pbcloud/metric.proto index 6aefdc57db3..4e66cae57e3 100644 --- a/output/cloud/expv2/pbcloud/metric.proto +++ b/output/cloud/expv2/pbcloud/metric.proto @@ -115,30 +115,40 @@ message RateValue { uint32 total_count = 3; } +// BucketSpan represents the consecutive and significant (non-zero) buckets. +// The concept is imported from the Prometheus' native histogram. +// It should simplify an eventual transition to Prometheus. +// https://github.com/prometheus/prometheus/blob/f399f386cef3b19c48157e9678a4f50997db3f41/prompb/io/prometheus/client/metrics.proto#L115-L124 +message BucketSpan { + uint32 offset = 1; // Gap to previous span, or starting point for 1st span. + uint32 length = 2; // Length of the next consecutive buckets. +} + message TrendHdrValue { // Required. google.protobuf.Timestamp time = 1; - // buckets + // positive buckets starting from bucket 1, each counter is an absolute value. repeated uint32 counters = 2; - // index of first bucket in `counters` - uint32 lower_counter_index = 3; + repeated BucketSpan spans = 3; + // sum of observations - double sum = 4; + double sum = 5; // count of observations - uint32 count = 5; + uint32 count = 6; // smallest and largest observed value - double min_value = 6; - double max_value = 7; + double min_value = 7; + double max_value = 8; - // counters for zero and infinity buckets - optional uint32 extra_low_values_counter = 8; - optional uint32 extra_high_values_counter = 9; + // counters for zero- and infinity-buckets + optional uint32 extra_low_values_counter = 9; + optional uint32 extra_high_values_counter = 10; // histogram parameter - value multiplier aka smallest value // default = 1.0 - optional double min_resolution = 10; + optional double min_resolution = 11; + // histogram parameter - number of significant digits used to calculate buckets formula // default = 2 - optional uint32 significant_digits = 11; + optional uint32 significant_digits = 12; } From 6f7548dd1ca5d49843799a3ec7d76a86d7b8300e Mon Sep 17 00:00:00 2001 From: Ivan <2103732+codebien@users.noreply.github.com> Date: Wed, 5 Jul 2023 19:08:30 +0200 Subject: [PATCH 2/8] hdr_test: Test more edge cases --- output/cloud/expv2/hdr_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/output/cloud/expv2/hdr_test.go b/output/cloud/expv2/hdr_test.go index 803dda2c2ce..2187eaebfda 100644 --- a/output/cloud/expv2/hdr_test.go +++ b/output/cloud/expv2/hdr_test.go @@ -21,6 +21,9 @@ func TestValueBacket(t *testing.T) { {in: -1029, exp: 0}, {in: -12, exp: 0}, {in: -0.82673, exp: 0}, + {in: 0, exp: 0}, + {in: 0.12, exp: 1}, + {in: 1.91, exp: 2}, {in: 10, exp: 10}, {in: 12, exp: 12}, {in: 12.5, exp: 13}, From 0f676c5b3eac0ddcf3b97c6ae8715d25ee92860f Mon Sep 17 00:00:00 2001 From: Ivan <2103732+codebien@users.noreply.github.com> Date: Thu, 6 Jul 2023 10:02:07 +0200 Subject: [PATCH 3/8] cloudv2: Use a compact version for histogram The histogram now uses a more compact solution for storing the distribution. It tracks only the significant buckets. --- output/cloud/expv2/hdr.go | 149 +++++++---------- output/cloud/expv2/hdr_test.go | 288 +++++++++++++++++---------------- 2 files changed, 209 insertions(+), 228 deletions(-) diff --git a/output/cloud/expv2/hdr.go b/output/cloud/expv2/hdr.go index 54ff939e4a1..ec3ce23830e 100644 --- a/output/cloud/expv2/hdr.go +++ b/output/cloud/expv2/hdr.go @@ -3,6 +3,7 @@ package expv2 import ( "math" "math/bits" + "sort" "go.k6.io/k6/output/cloud/expv2/pbcloud" ) @@ -37,14 +38,13 @@ const ( // The current version is: f(N = 25, m = 7) = 3200. type histogram struct { // Buckets stores the counters for each bin of the histogram. - // It does not include the first and the last absolute bucket, - // because they contain exception cases - // and they requires to be tracked in a dedicated way. - // - // It is expected to start and end with a non-zero bucket, - // in this way we can avoid extra allocation for not significant buckets. - // All the zero buckets in between are preserved. - Buckets []uint32 + // It does not include counters for the untrackable values, + // because they contain exception cases and require to be tracked in a dedicated way. + Buckets map[uint32]uint32 + + // Indexes keeps an ordered slice of unique-seen buckets' indexes. + // It allows to iterate the buckets in order. It uses an ascendent order. + Indexes []uint32 // ExtraLowBucket counts occurrences of observed values smaller // than the minimum trackable value. @@ -54,16 +54,6 @@ type histogram struct { // than the maximum trackable value. ExtraHighBucket uint32 - // FirstNotZeroBucket represents the index of the first bucket - // with a significant counter in the Buckets slice (a not zero value). - // In this way, all the buckets before can be omitted. - FirstNotZeroBucket uint32 - - // LastNotZeroBucket represents the index of the last bucket - // with a significant counter in the Buckets slice (a not zero value). - // In this way, all the buckets after can be omitted. - LastNotZeroBucket uint32 - // Max is the absolute maximum observed value. Max float64 @@ -104,92 +94,72 @@ func (h *histogram) addToBucket(v float64) { return } - index := resolveBucketIndex(v) + ix := resolveBucketIndex(v) + if _, contains := h.Buckets[ix]; !contains { + h.trackBucket(ix) + } - // they grow the current Buckets slice if there isn't enough capacity. - // - // An example with growRight: - // With Buckets [4, 1] and index equals to 5 - // then we expect a slice like [4,1,0,0,0,0] - // then the counter at 5th position will be incremented - // generating the final slice [4,1,0,0,0,1] - switch { - case len(h.Buckets) == 0: - h.init(index) - case index < h.FirstNotZeroBucket: - h.prependBuckets(index) - case index > h.LastNotZeroBucket: - h.appendBuckets(index) - default: - h.Buckets[index-h.FirstNotZeroBucket]++ + if h.Buckets == nil { + h.Buckets = make(map[uint32]uint32) } + h.Buckets[ix]++ } -func (h *histogram) init(index uint32) { - h.FirstNotZeroBucket = index - h.LastNotZeroBucket = index - h.Buckets = make([]uint32, 1, 32) - h.Buckets[0] = 1 -} +// trackBucket stores the unique seen buckets. +func (h *histogram) trackBucket(index uint32) { + i := sort.Search(len(h.Indexes), func(i int) bool { + return h.Indexes[i] > index + }) -// prependBuckets expands the buckets slice with zeros up to the required index, -// then it increments the required bucket. -func (h *histogram) prependBuckets(index uint32) { - if h.FirstNotZeroBucket <= index { - panic("buckets is already contains the requested index") + // insert at the end + if len(h.Indexes) == i { + h.Indexes = append(h.Indexes, index) + return } - newLen := (h.FirstNotZeroBucket - index) + uint32(len(h.Buckets)) - - // TODO: we may consider to swap by sub-groups - // e.g [4, 1] => [4, 1, 0, 0] => [0, 0, 4, 1] - // It requires a benchmark if it is better than just copy it. - - newBuckets := make([]uint32, newLen) - copy(newBuckets[h.FirstNotZeroBucket-index:], h.Buckets) - h.Buckets = newBuckets - - // Update the stats - h.Buckets[0] = 1 - h.FirstNotZeroBucket = index + // insert in the middle + h.Indexes = append(h.Indexes, 0) // expand the slice + copy(h.Indexes[i+1:], h.Indexes[i:]) // make the space + h.Indexes[i] = index // set the index } -// appendBuckets expands the buckets slice with zeros buckets till the required index, -// then it increments the required bucket. -// If the slice has enough capacity then it reuses it without allocate. -func (h *histogram) appendBuckets(index uint32) { - if h.LastNotZeroBucket >= index { - panic("buckets is already bigger than requested index") +// histogramAsProto converts the histogram into the equivalent Protobuf version. +func histogramAsProto(h *histogram, time int64) *pbcloud.TrendHdrValue { + var ( + counters []uint32 + spans []*pbcloud.BucketSpan + ) + + // allocate only if at least one item is available + if len(h.Indexes) > 0 { + // init the counters + counters = make([]uint32, 1, len(h.Indexes)) + counters[0] = h.Buckets[h.Indexes[0]] + // open the first span + spans = append(spans, &pbcloud.BucketSpan{Offset: h.Indexes[0], Length: 1}) } - newLen := index - h.FirstNotZeroBucket + 1 + for i := 1; i < len(h.Buckets); i++ { + counters = append(counters, h.Buckets[h.Indexes[i]]) - if uint32(cap(h.Buckets)) > newLen { - // See https://go.dev/ref/spec#Slice_expressions - // "For slices, the upper index bound is - // the slice capacity cap(a) rather than the length" - h.Buckets = h.Buckets[:newLen] - } else { - newBuckets := make([]uint32, newLen) - copy(newBuckets, h.Buckets) - h.Buckets = newBuckets - } + // if the current and the previous indexes are not consecutive + // consider as closed the current on-going span and start a new one. + if diff := h.Indexes[i] - h.Indexes[i-1]; diff > 1 { + spans = append(spans, &pbcloud.BucketSpan{Offset: diff, Length: 1}) + continue + } - // Update the stats - h.Buckets[len(h.Buckets)-1] = 1 - h.LastNotZeroBucket = index -} + spans[len(spans)-1].Length++ + } -// histogramAsProto converts the histogram into the equivalent Protobuf version. -func histogramAsProto(h *histogram, time int64) *pbcloud.TrendHdrValue { hval := &pbcloud.TrendHdrValue{ - Time: timestampAsProto(time), - LowerCounterIndex: h.FirstNotZeroBucket, - MinValue: h.Min, - MaxValue: h.Max, - Sum: h.Sum, - Count: h.Count, - Counters: h.Buckets, + Time: timestampAsProto(time), + MinValue: h.Min, + MaxValue: h.Max, + Sum: h.Sum, + Count: h.Count, + Counters: counters, + Spans: spans, } if h.ExtraLowBucket > 0 { hval.ExtraLowValuesCounter = &h.ExtraLowBucket @@ -255,6 +225,7 @@ func resolveBucketIndex(val float64) uint32 { return (nkdiff << k) + (upscaled >> nkdiff) } +// Add implements the metricValue interface. func (h *histogram) Add(v float64) { h.addToBucket(v) } diff --git a/output/cloud/expv2/hdr_test.go b/output/cloud/expv2/hdr_test.go index 2187eaebfda..7dd2c622561 100644 --- a/output/cloud/expv2/hdr_test.go +++ b/output/cloud/expv2/hdr_test.go @@ -11,7 +11,7 @@ import ( "google.golang.org/protobuf/types/known/timestamppb" ) -func TestValueBacket(t *testing.T) { +func TestResolveBucketIndex(t *testing.T) { t.Parallel() tests := []struct { @@ -32,6 +32,7 @@ func TestValueBacket(t *testing.T) { {in: 256, exp: 256}, {in: 282.29, exp: 269}, {in: 1029, exp: 512}, + {in: 39751, exp: 1179}, {in: (1 << 30) - 1, exp: 3071}, {in: (1 << 30), exp: 3072}, {in: math.MaxInt32, exp: 3199}, @@ -41,181 +42,171 @@ func TestValueBacket(t *testing.T) { } } -func TestNewHistogramWithSimpleValue(t *testing.T) { +func TestHistogramAddWithSimpleValue(t *testing.T) { t.Parallel() // Zero as value res := histogram{} - res.addToBucket(0) + res.Add(0) exp := histogram{ - Buckets: []uint32{1}, - FirstNotZeroBucket: 0, - LastNotZeroBucket: 0, - ExtraLowBucket: 0, - ExtraHighBucket: 0, - Max: 0, - Min: 0, - Sum: 0, - Count: 1, + Buckets: map[uint32]uint32{0: 1}, + Indexes: []uint32{0}, + ExtraLowBucket: 0, + ExtraHighBucket: 0, + Max: 0, + Min: 0, + Sum: 0, + Count: 1, } require.Equal(t, exp, res) // Add a lower bucket index within slice capacity res = histogram{} - res.addToBucket(8) - res.addToBucket(5) + res.Add(8) + res.Add(5) exp = histogram{ - Buckets: []uint32{1, 0, 0, 1}, - FirstNotZeroBucket: 5, - LastNotZeroBucket: 8, - ExtraLowBucket: 0, - ExtraHighBucket: 0, - Max: 8, - Min: 5, - Sum: 13, - Count: 2, + Buckets: map[uint32]uint32{5: 1, 8: 1}, + Indexes: []uint32{5, 8}, + ExtraLowBucket: 0, + ExtraHighBucket: 0, + Max: 8, + Min: 5, + Sum: 13, + Count: 2, } require.Equal(t, exp, res) // Add a higher bucket index within slice capacity res = histogram{} - res.addToBucket(100) - res.addToBucket(101) + res.Add(100) + res.Add(101) exp = histogram{ - Buckets: []uint32{1, 1}, - FirstNotZeroBucket: 100, - LastNotZeroBucket: 101, - ExtraLowBucket: 0, - ExtraHighBucket: 0, - Max: 101, - Min: 100, - Sum: 201, - Count: 2, + Buckets: map[uint32]uint32{100: 1, 101: 1}, + Indexes: []uint32{100, 101}, + ExtraLowBucket: 0, + ExtraHighBucket: 0, + Max: 101, + Min: 100, + Sum: 201, + Count: 2, } require.Equal(t, exp, res) // Same case but reversed test check res = histogram{} - res.addToBucket(101) - res.addToBucket(100) + res.Add(101) + res.Add(100) exp = histogram{ - Buckets: []uint32{1, 1}, - FirstNotZeroBucket: 100, - LastNotZeroBucket: 101, - ExtraLowBucket: 0, - ExtraHighBucket: 0, - Max: 101, - Min: 100, - Sum: 201, - Count: 2, + Buckets: map[uint32]uint32{100: 1, 101: 1}, + Indexes: []uint32{100, 101}, + ExtraLowBucket: 0, + ExtraHighBucket: 0, + Max: 101, + Min: 100, + Sum: 201, + Count: 2, } assert.Equal(t, exp, res) // One more complex case with lower index and more than two indexes res = histogram{} - res.addToBucket(8) - res.addToBucket(9) - res.addToBucket(10) - res.addToBucket(5) + res.Add(8) + res.Add(9) + res.Add(10) + res.Add(5) exp = histogram{ - Buckets: []uint32{1, 0, 0, 1, 1, 1}, - FirstNotZeroBucket: 5, - LastNotZeroBucket: 10, - ExtraLowBucket: 0, - ExtraHighBucket: 0, - Max: 10, - Min: 5, - Sum: 32, - Count: 4, + Buckets: map[uint32]uint32{8: 1, 9: 1, 10: 1, 5: 1}, + Indexes: []uint32{5, 8, 9, 10}, + ExtraLowBucket: 0, + ExtraHighBucket: 0, + Max: 10, + Min: 5, + Sum: 32, + Count: 4, } assert.Equal(t, exp, res) } -func TestNewHistogramWithUntrackables(t *testing.T) { +func TestHistogramAddWithUntrackables(t *testing.T) { t.Parallel() res := histogram{} for _, v := range []float64{5, -3.14, 2 * 1e9, 1} { - res.addToBucket(v) + res.Add(v) } exp := histogram{ - Buckets: []uint32{1, 0, 0, 0, 1}, - FirstNotZeroBucket: 1, - LastNotZeroBucket: 5, - ExtraLowBucket: 1, - ExtraHighBucket: 1, - Max: 2 * 1e9, - Min: -3.14, - Sum: 2*1e9 + 5 + 1 - 3.14, - Count: 4, + Buckets: map[uint32]uint32{1: 1, 5: 1}, + Indexes: []uint32{1, 5}, + ExtraLowBucket: 1, + ExtraHighBucket: 1, + Max: 2 * 1e9, + Min: -3.14, + Sum: 2*1e9 + 5 + 1 - 3.14, + Count: 4, } assert.Equal(t, exp, res) } -func TestNewHistogramWithMultipleValues(t *testing.T) { +func TestHistogramAddWithMultipleOccurances(t *testing.T) { t.Parallel() res := histogram{} for _, v := range []float64{51.8, 103.6, 103.6, 103.6, 103.6} { - res.addToBucket(v) + res.Add(v) } exp := histogram{ - FirstNotZeroBucket: 52, - LastNotZeroBucket: 104, - Max: 103.6, - Min: 51.8, - ExtraLowBucket: 0, - ExtraHighBucket: 0, - Buckets: append(append([]uint32{1}, make([]uint32, 51)...), 4), - // Buckets = {1, 0 for 51 times, 4} - Sum: 466.20000000000005, - Count: 5, + Buckets: map[uint32]uint32{52: 1, 104: 4}, + Indexes: []uint32{52, 104}, + Max: 103.6, + Min: 51.8, + ExtraLowBucket: 0, + ExtraHighBucket: 0, + Sum: 466.20000000000005, + Count: 5, } assert.Equal(t, exp, res) } -func TestNewHistogramWithNegativeNum(t *testing.T) { +func TestHistogramAddWithNegativeNum(t *testing.T) { t.Parallel() res := histogram{} - res.addToBucket(-2.42314) + res.Add(-2.42314) exp := histogram{ - FirstNotZeroBucket: 0, - Max: -2.42314, - Min: -2.42314, - Buckets: nil, - ExtraLowBucket: 1, - ExtraHighBucket: 0, - Sum: -2.42314, - Count: 1, + Max: -2.42314, + Min: -2.42314, + Buckets: nil, + ExtraLowBucket: 1, + ExtraHighBucket: 0, + Sum: -2.42314, + Count: 1, } assert.Equal(t, exp, res) } -func TestNewHistogramWithMultipleNegativeNums(t *testing.T) { +func TestHistogramAddWithMultipleNegativeNums(t *testing.T) { t.Parallel() res := histogram{} for _, v := range []float64{-0.001, -0.001, -0.001} { - res.addToBucket(v) + res.Add(v) } exp := histogram{ - Buckets: nil, - FirstNotZeroBucket: 0, - ExtraLowBucket: 3, - ExtraHighBucket: 0, - Max: -0.001, - Min: -0.001, - Sum: -0.003, - Count: 3, + Buckets: nil, + ExtraLowBucket: 3, + ExtraHighBucket: 0, + Max: -0.001, + Min: -0.001, + Sum: -0.003, + Count: 3, } assert.Equal(t, exp, res) } @@ -225,41 +216,16 @@ func TestNewHistoramWithNoVals(t *testing.T) { res := histogram{} exp := histogram{ - Buckets: nil, - FirstNotZeroBucket: 0, - ExtraLowBucket: 0, - ExtraHighBucket: 0, - Max: 0, - Min: 0, - Sum: 0, + Buckets: nil, + ExtraLowBucket: 0, + ExtraHighBucket: 0, + Max: 0, + Min: 0, + Sum: 0, } assert.Equal(t, exp, res) } -func TestHistogramAppendBuckets(t *testing.T) { - t.Parallel() - h := histogram{} - - // the cap is smaller than requested index - // so it creates a new slice - h.appendBuckets(3) - assert.Len(t, h.Buckets, 4) - - // it must preserve already existing items - h.Buckets[2] = 101 - - // it appends to the same slice - h.appendBuckets(5) - assert.Len(t, h.Buckets, 6) - assert.Equal(t, uint32(101), h.Buckets[2]) - assert.Equal(t, uint32(1), h.Buckets[5]) - - // it is not possible to request an index smaller than - // the last already available index - h.LastNotZeroBucket = 5 - assert.Panics(t, func() { h.appendBuckets(4) }) -} - func TestHistogramAsProto(t *testing.T) { t.Parallel() @@ -280,11 +246,11 @@ func TestHistogramAsProto(t *testing.T) { name: "not trackable values", vals: []float64{-0.23, 1<<30 + 1}, exp: &pbcloud.TrendHdrValue{ - Count: 2, ExtraLowValuesCounter: uint32ptr(1), ExtraHighValuesCounter: uint32ptr(1), Counters: nil, - LowerCounterIndex: 0, + Spans: nil, + Count: 2, MinValue: -0.23, MaxValue: 1<<30 + 1, Sum: (1 << 30) + 1 - 0.23, @@ -298,10 +264,54 @@ func TestHistogramAsProto(t *testing.T) { ExtraLowValuesCounter: nil, ExtraHighValuesCounter: nil, Counters: []uint32{2, 1}, - LowerCounterIndex: 2, - MinValue: 1.1, - MaxValue: 3, - Sum: 6.1, + Spans: []*pbcloud.BucketSpan{ + { + Offset: 2, + Length: 2, + }, + }, + MinValue: 1.1, + MaxValue: 3, + Sum: 6.1, + }, + }, + { + name: "longer sequence", + vals: []float64{ + 2275, 52.25, 268.85, 383.47, 18.49, + 163.85, 4105, 835.27, 52, 18.28, 238.44, 39751, 18.86, + 967.05, 967.01, 967, 4123.5, 270.69, 677.27, + }, + // Sorted: + // 18.28,18.49,18.86,52,52.25,163.85, + // 238.44,268.85,270.69,383.47,677.27,835.27,967,967.01,967.05 + // 2275, 4105, 4123.5, 39751 + // Distribution + // - {x<256}: 19:3, 52:1, 53:1, 164:1, 239:1 + // - {x >= 256}: 262:1, 263:1, 320:1, 425:1, 465:1, 497:1 498:2 + // - {x > 1k}: 654:1, 768:2, 1179:1 + exp: &pbcloud.TrendHdrValue{ + Count: 19, + Counters: []uint32{3, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 2, 1}, + Spans: []*pbcloud.BucketSpan{ + {Offset: 19, Length: 1}, + {Offset: 33, Length: 2}, + {Offset: 111, Length: 1}, + {Offset: 75, Length: 1}, + {Offset: 23, Length: 2}, // 262 + {Offset: 57, Length: 1}, + {Offset: 105, Length: 1}, + {Offset: 40, Length: 1}, + {Offset: 32, Length: 2}, + {Offset: 156, Length: 1}, // 654 + {Offset: 114, Length: 1}, + {Offset: 411, Length: 1}, + }, + ExtraLowValuesCounter: nil, + ExtraHighValuesCounter: nil, + MinValue: 18.28, + MaxValue: 39751, + Sum: 56153.280000000006, }, }, } @@ -309,7 +319,7 @@ func TestHistogramAsProto(t *testing.T) { for _, tc := range cases { h := histogram{} for _, v := range tc.vals { - h.addToBucket(v) + h.Add(v) } tc.exp.Time = ×tamppb.Timestamp{Seconds: 1} assert.Equal(t, tc.exp, histogramAsProto(&h, time.Unix(1, 0).UnixNano()), tc.name) From b7cb2b3cb7431a58aef1915d3a3a8386b2b794d7 Mon Sep 17 00:00:00 2001 From: Ivan <2103732+codebien@users.noreply.github.com> Date: Thu, 6 Jul 2023 15:38:15 +0200 Subject: [PATCH 4/8] cloudv2/integration: Test updated accordingly --- output/cloud/expv2/integration/testdata/metricset.json | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/output/cloud/expv2/integration/testdata/metricset.json b/output/cloud/expv2/integration/testdata/metricset.json index 4720f352227..1bd42ebbdb4 100644 --- a/output/cloud/expv2/integration/testdata/metricset.json +++ b/output/cloud/expv2/integration/testdata/metricset.json @@ -92,7 +92,12 @@ "counters": [ 1 ], - "lowerCounterIndex": 6, + "spans": [ + { + "offset": 6, + "length": 1 + } + ], "maxValue": 6, "minValue": 6, "sum": 6 From e7bee240a75f0635f8ffb1a227c8c962810ae504 Mon Sep 17 00:00:00 2001 From: Ivan <2103732+codebien@users.noreply.github.com> Date: Fri, 7 Jul 2023 12:48:44 +0200 Subject: [PATCH 5/8] Address request changes --- output/cloud/expv2/hdr.go | 40 ++++--- output/cloud/expv2/hdr_test.go | 203 ++++++++++++++++---------------- output/cloud/expv2/sink.go | 2 +- output/cloud/expv2/sink_test.go | 7 +- 4 files changed, 131 insertions(+), 121 deletions(-) diff --git a/output/cloud/expv2/hdr.go b/output/cloud/expv2/hdr.go index ec3ce23830e..4166f94f11b 100644 --- a/output/cloud/expv2/hdr.go +++ b/output/cloud/expv2/hdr.go @@ -67,20 +67,28 @@ type histogram struct { Count uint32 } +func newHistogram() *histogram { + return &histogram{ + Buckets: make(map[uint32]uint32), + Max: -math.MaxFloat64, + Min: math.MaxFloat64, + } +} + // addToBucket increments the counter of the bucket of the provided value. // If the value is lower or higher than the trackable limits // then it is counted into specific buckets. All the stats are also updated accordingly. func (h *histogram) addToBucket(v float64) { - if h.Count == 0 { - h.Max, h.Min = v, v - } else { - if v > h.Max { - h.Max = v - } - if v < h.Min { - h.Min = v - } + // if h.Count == 0 { + // h.Max, h.Min = v, v + // } else { + if v > h.Max { + h.Max = v + } + if v < h.Min { + h.Min = v } + // } h.Count++ h.Sum += v @@ -99,9 +107,6 @@ func (h *histogram) addToBucket(v float64) { h.trackBucket(ix) } - if h.Buckets == nil { - h.Buckets = make(map[uint32]uint32) - } h.Buckets[ix]++ } @@ -117,10 +122,10 @@ func (h *histogram) trackBucket(index uint32) { return } - // insert in the middle - h.Indexes = append(h.Indexes, 0) // expand the slice - copy(h.Indexes[i+1:], h.Indexes[i:]) // make the space - h.Indexes[i] = index // set the index + // insert at specific `i` + h.Indexes = append(h.Indexes, 0) + copy(h.Indexes[i+1:], h.Indexes[i:]) + h.Indexes[i] = index } // histogramAsProto converts the histogram into the equivalent Protobuf version. @@ -130,7 +135,8 @@ func histogramAsProto(h *histogram, time int64) *pbcloud.TrendHdrValue { spans []*pbcloud.BucketSpan ) - // allocate only if at least one item is available + // allocate only if at least one item is available, in the case of only + // untrackable values, then Indexes and Buckets are expected to be empty. if len(h.Indexes) > 0 { // init the counters counters = make([]uint32, 1, len(h.Indexes)) diff --git a/output/cloud/expv2/hdr_test.go b/output/cloud/expv2/hdr_test.go index 7dd2c622561..4e4e211a4d2 100644 --- a/output/cloud/expv2/hdr_test.go +++ b/output/cloud/expv2/hdr_test.go @@ -2,11 +2,11 @@ package expv2 import ( "math" + "strconv" "testing" "time" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" "go.k6.io/k6/output/cloud/expv2/pbcloud" "google.golang.org/protobuf/types/known/timestamppb" ) @@ -42,105 +42,101 @@ func TestResolveBucketIndex(t *testing.T) { } } -func TestHistogramAddWithSimpleValue(t *testing.T) { +func TestHistogramAddWithSimpleValues(t *testing.T) { t.Parallel() - // Zero as value - res := histogram{} - res.Add(0) - exp := histogram{ - Buckets: map[uint32]uint32{0: 1}, - Indexes: []uint32{0}, - ExtraLowBucket: 0, - ExtraHighBucket: 0, - Max: 0, - Min: 0, - Sum: 0, - Count: 1, - } - require.Equal(t, exp, res) - - // Add a lower bucket index within slice capacity - res = histogram{} - res.Add(8) - res.Add(5) - - exp = histogram{ - Buckets: map[uint32]uint32{5: 1, 8: 1}, - Indexes: []uint32{5, 8}, - ExtraLowBucket: 0, - ExtraHighBucket: 0, - Max: 8, - Min: 5, - Sum: 13, - Count: 2, - } - require.Equal(t, exp, res) - - // Add a higher bucket index within slice capacity - res = histogram{} - res.Add(100) - res.Add(101) - - exp = histogram{ - Buckets: map[uint32]uint32{100: 1, 101: 1}, - Indexes: []uint32{100, 101}, - ExtraLowBucket: 0, - ExtraHighBucket: 0, - Max: 101, - Min: 100, - Sum: 201, - Count: 2, - } - require.Equal(t, exp, res) - - // Same case but reversed test check - res = histogram{} - res.Add(101) - res.Add(100) - - exp = histogram{ - Buckets: map[uint32]uint32{100: 1, 101: 1}, - Indexes: []uint32{100, 101}, - ExtraLowBucket: 0, - ExtraHighBucket: 0, - Max: 101, - Min: 100, - Sum: 201, - Count: 2, + cases := []struct { + vals []float64 + exp histogram + }{ + { + vals: []float64{0}, + exp: histogram{ + Buckets: map[uint32]uint32{0: 1}, + Indexes: []uint32{0}, + ExtraLowBucket: 0, + ExtraHighBucket: 0, + Max: 0, + Min: 0, + Sum: 0, + Count: 1, + }, + }, + { + vals: []float64{8, 5}, + exp: histogram{ + Buckets: map[uint32]uint32{5: 1, 8: 1}, + Indexes: []uint32{5, 8}, + ExtraLowBucket: 0, + ExtraHighBucket: 0, + Max: 8, + Min: 5, + Sum: 13, + Count: 2, + }, + }, + { + vals: []float64{8, 9, 10, 5}, + exp: histogram{ + Buckets: map[uint32]uint32{8: 1, 9: 1, 10: 1, 5: 1}, + Indexes: []uint32{5, 8, 9, 10}, + ExtraLowBucket: 0, + ExtraHighBucket: 0, + Max: 10, + Min: 5, + Sum: 32, + Count: 4, + }, + }, + { + vals: []float64{100, 101}, + exp: histogram{ + Buckets: map[uint32]uint32{100: 1, 101: 1}, + Indexes: []uint32{100, 101}, + ExtraLowBucket: 0, + ExtraHighBucket: 0, + Max: 101, + Min: 100, + Sum: 201, + Count: 2, + }, + }, + { + vals: []float64{101, 100}, + exp: histogram{ + Buckets: map[uint32]uint32{100: 1, 101: 1}, + Indexes: []uint32{100, 101}, + ExtraLowBucket: 0, + ExtraHighBucket: 0, + Max: 101, + Min: 100, + Sum: 201, + Count: 2, + }, + }, } - assert.Equal(t, exp, res) - // One more complex case with lower index and more than two indexes - res = histogram{} - res.Add(8) - res.Add(9) - res.Add(10) - res.Add(5) - - exp = histogram{ - Buckets: map[uint32]uint32{8: 1, 9: 1, 10: 1, 5: 1}, - Indexes: []uint32{5, 8, 9, 10}, - ExtraLowBucket: 0, - ExtraHighBucket: 0, - Max: 10, - Min: 5, - Sum: 32, - Count: 4, + for i, tc := range cases { + tc := tc + t.Run(strconv.Itoa(i), func(t *testing.T) { + h := newHistogram() + for _, v := range tc.vals { + h.Add(v) + } + assert.Equal(t, &tc.exp, h) + }) } - - assert.Equal(t, exp, res) } func TestHistogramAddWithUntrackables(t *testing.T) { t.Parallel() - res := histogram{} + res := newHistogram() for _, v := range []float64{5, -3.14, 2 * 1e9, 1} { res.Add(v) } - exp := histogram{ + exp := &histogram{ Buckets: map[uint32]uint32{1: 1, 5: 1}, Indexes: []uint32{1, 5}, ExtraLowBucket: 1, @@ -156,12 +152,12 @@ func TestHistogramAddWithUntrackables(t *testing.T) { func TestHistogramAddWithMultipleOccurances(t *testing.T) { t.Parallel() - res := histogram{} + res := newHistogram() for _, v := range []float64{51.8, 103.6, 103.6, 103.6, 103.6} { res.Add(v) } - exp := histogram{ + exp := &histogram{ Buckets: map[uint32]uint32{52: 1, 104: 4}, Indexes: []uint32{52, 104}, Max: 103.6, @@ -177,13 +173,13 @@ func TestHistogramAddWithMultipleOccurances(t *testing.T) { func TestHistogramAddWithNegativeNum(t *testing.T) { t.Parallel() - res := histogram{} + res := newHistogram() res.Add(-2.42314) - exp := histogram{ + exp := &histogram{ Max: -2.42314, Min: -2.42314, - Buckets: nil, + Buckets: map[uint32]uint32{}, ExtraLowBucket: 1, ExtraHighBucket: 0, Sum: -2.42314, @@ -194,13 +190,13 @@ func TestHistogramAddWithNegativeNum(t *testing.T) { func TestHistogramAddWithMultipleNegativeNums(t *testing.T) { t.Parallel() - res := histogram{} + res := newHistogram() for _, v := range []float64{-0.001, -0.001, -0.001} { res.Add(v) } - exp := histogram{ - Buckets: nil, + exp := &histogram{ + Buckets: map[uint32]uint32{}, ExtraLowBucket: 3, ExtraHighBucket: 0, Max: -0.001, @@ -214,13 +210,13 @@ func TestHistogramAddWithMultipleNegativeNums(t *testing.T) { func TestNewHistoramWithNoVals(t *testing.T) { t.Parallel() - res := histogram{} - exp := histogram{ - Buckets: nil, + res := newHistogram() + exp := &histogram{ + Buckets: map[uint32]uint32{}, ExtraLowBucket: 0, ExtraHighBucket: 0, - Max: 0, - Min: 0, + Max: -math.MaxFloat64, + Min: math.MaxFloat64, Sum: 0, } assert.Equal(t, exp, res) @@ -240,7 +236,10 @@ func TestHistogramAsProto(t *testing.T) { }{ { name: "empty histogram", - exp: &pbcloud.TrendHdrValue{}, + exp: &pbcloud.TrendHdrValue{ + MaxValue: -math.MaxFloat64, + MinValue: math.MaxFloat64, + }, }, { name: "not trackable values", @@ -317,11 +316,11 @@ func TestHistogramAsProto(t *testing.T) { } for _, tc := range cases { - h := histogram{} + h := newHistogram() for _, v := range tc.vals { h.Add(v) } tc.exp.Time = ×tamppb.Timestamp{Seconds: 1} - assert.Equal(t, tc.exp, histogramAsProto(&h, time.Unix(1, 0).UnixNano()), tc.name) + assert.Equal(t, tc.exp, histogramAsProto(h, time.Unix(1, 0).UnixNano()), tc.name) } } diff --git a/output/cloud/expv2/sink.go b/output/cloud/expv2/sink.go index e341d833271..185453c0bbc 100644 --- a/output/cloud/expv2/sink.go +++ b/output/cloud/expv2/sink.go @@ -22,7 +22,7 @@ func newMetricValue(mt metrics.MetricType) metricValue { case metrics.Rate: am = &rate{} case metrics.Trend: - am = &histogram{} + am = newHistogram() default: // Should not be possible to create // an invalid metric type except for specific diff --git a/output/cloud/expv2/sink_test.go b/output/cloud/expv2/sink_test.go index 5fb1fd6cdbf..70bdeb21b8e 100644 --- a/output/cloud/expv2/sink_test.go +++ b/output/cloud/expv2/sink_test.go @@ -1,6 +1,7 @@ package expv2 import ( + "math" "testing" "github.com/stretchr/testify/assert" @@ -16,7 +17,11 @@ func TestNewSink(t *testing.T) { {metrics.Counter, &counter{}}, {metrics.Gauge, &gauge{}}, {metrics.Rate, &rate{}}, - {metrics.Trend, &histogram{}}, + {metrics.Trend, &histogram{ + Buckets: map[uint32]uint32{}, + Max: -math.MaxFloat64, + Min: math.MaxFloat64}, + }, } for _, tc := range tests { assert.Equal(t, tc.exp, newMetricValue(tc.mt)) From 976a8ff4d125fe839318f701e7a6254ba58318f1 Mon Sep 17 00:00:00 2001 From: Ivan <2103732+codebien@users.noreply.github.com> Date: Fri, 7 Jul 2023 12:53:54 +0200 Subject: [PATCH 6/8] Test polishing --- output/cloud/expv2/hdr.go | 4 --- output/cloud/expv2/hdr_test.go | 47 ++++++++++++++++++--------------- output/cloud/expv2/sink_test.go | 11 +++++--- 3 files changed, 33 insertions(+), 29 deletions(-) diff --git a/output/cloud/expv2/hdr.go b/output/cloud/expv2/hdr.go index 4166f94f11b..120550ed3b8 100644 --- a/output/cloud/expv2/hdr.go +++ b/output/cloud/expv2/hdr.go @@ -79,16 +79,12 @@ func newHistogram() *histogram { // If the value is lower or higher than the trackable limits // then it is counted into specific buckets. All the stats are also updated accordingly. func (h *histogram) addToBucket(v float64) { - // if h.Count == 0 { - // h.Max, h.Min = v, v - // } else { if v > h.Max { h.Max = v } if v < h.Min { h.Min = v } - // } h.Count++ h.Sum += v diff --git a/output/cloud/expv2/hdr_test.go b/output/cloud/expv2/hdr_test.go index 4e4e211a4d2..797e3863a52 100644 --- a/output/cloud/expv2/hdr_test.go +++ b/output/cloud/expv2/hdr_test.go @@ -119,6 +119,7 @@ func TestHistogramAddWithSimpleValues(t *testing.T) { for i, tc := range cases { tc := tc t.Run(strconv.Itoa(i), func(t *testing.T) { + t.Parallel() h := newHistogram() for _, v := range tc.vals { h.Add(v) @@ -131,9 +132,9 @@ func TestHistogramAddWithSimpleValues(t *testing.T) { func TestHistogramAddWithUntrackables(t *testing.T) { t.Parallel() - res := newHistogram() + h := newHistogram() for _, v := range []float64{5, -3.14, 2 * 1e9, 1} { - res.Add(v) + h.Add(v) } exp := &histogram{ @@ -146,15 +147,15 @@ func TestHistogramAddWithUntrackables(t *testing.T) { Sum: 2*1e9 + 5 + 1 - 3.14, Count: 4, } - assert.Equal(t, exp, res) + assert.Equal(t, exp, h) } func TestHistogramAddWithMultipleOccurances(t *testing.T) { t.Parallel() - res := newHistogram() + h := newHistogram() for _, v := range []float64{51.8, 103.6, 103.6, 103.6, 103.6} { - res.Add(v) + h.Add(v) } exp := &histogram{ @@ -167,14 +168,14 @@ func TestHistogramAddWithMultipleOccurances(t *testing.T) { Sum: 466.20000000000005, Count: 5, } - assert.Equal(t, exp, res) + assert.Equal(t, exp, h) } func TestHistogramAddWithNegativeNum(t *testing.T) { t.Parallel() - res := newHistogram() - res.Add(-2.42314) + h := newHistogram() + h.Add(-2.42314) exp := &histogram{ Max: -2.42314, @@ -185,14 +186,14 @@ func TestHistogramAddWithNegativeNum(t *testing.T) { Sum: -2.42314, Count: 1, } - assert.Equal(t, exp, res) + assert.Equal(t, exp, h) } func TestHistogramAddWithMultipleNegativeNums(t *testing.T) { t.Parallel() - res := newHistogram() + h := newHistogram() for _, v := range []float64{-0.001, -0.001, -0.001} { - res.Add(v) + h.Add(v) } exp := &histogram{ @@ -204,13 +205,13 @@ func TestHistogramAddWithMultipleNegativeNums(t *testing.T) { Sum: -0.003, Count: 3, } - assert.Equal(t, exp, res) + assert.Equal(t, exp, h) } func TestNewHistoramWithNoVals(t *testing.T) { t.Parallel() - res := newHistogram() + h := newHistogram() exp := &histogram{ Buckets: map[uint32]uint32{}, ExtraLowBucket: 0, @@ -219,7 +220,7 @@ func TestNewHistoramWithNoVals(t *testing.T) { Min: math.MaxFloat64, Sum: 0, } - assert.Equal(t, exp, res) + assert.Equal(t, exp, h) } func TestHistogramAsProto(t *testing.T) { @@ -315,12 +316,16 @@ func TestHistogramAsProto(t *testing.T) { }, } - for _, tc := range cases { - h := newHistogram() - for _, v := range tc.vals { - h.Add(v) - } - tc.exp.Time = ×tamppb.Timestamp{Seconds: 1} - assert.Equal(t, tc.exp, histogramAsProto(h, time.Unix(1, 0).UnixNano()), tc.name) + for i, tc := range cases { + tc := tc + t.Run(strconv.Itoa(i), func(t *testing.T) { + t.Parallel() + h := newHistogram() + for _, v := range tc.vals { + h.Add(v) + } + tc.exp.Time = ×tamppb.Timestamp{Seconds: 1} + assert.Equal(t, tc.exp, histogramAsProto(h, time.Unix(1, 0).UnixNano()), tc.name) + }) } } diff --git a/output/cloud/expv2/sink_test.go b/output/cloud/expv2/sink_test.go index 70bdeb21b8e..5257e1ffb8f 100644 --- a/output/cloud/expv2/sink_test.go +++ b/output/cloud/expv2/sink_test.go @@ -17,10 +17,13 @@ func TestNewSink(t *testing.T) { {metrics.Counter, &counter{}}, {metrics.Gauge, &gauge{}}, {metrics.Rate, &rate{}}, - {metrics.Trend, &histogram{ - Buckets: map[uint32]uint32{}, - Max: -math.MaxFloat64, - Min: math.MaxFloat64}, + { + metrics.Trend, + &histogram{ + Buckets: map[uint32]uint32{}, + Max: -math.MaxFloat64, + Min: math.MaxFloat64, + }, }, } for _, tc := range tests { From 1ec8c978e2b5af6fe35a8eab65de888d3af4b1f1 Mon Sep 17 00:00:00 2001 From: Ivan <2103732+codebien@users.noreply.github.com> Date: Fri, 7 Jul 2023 14:16:37 +0200 Subject: [PATCH 7/8] Use the constructor --- output/cloud/expv2/sink_test.go | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/output/cloud/expv2/sink_test.go b/output/cloud/expv2/sink_test.go index 5257e1ffb8f..5d0fa243ee6 100644 --- a/output/cloud/expv2/sink_test.go +++ b/output/cloud/expv2/sink_test.go @@ -1,7 +1,6 @@ package expv2 import ( - "math" "testing" "github.com/stretchr/testify/assert" @@ -17,14 +16,7 @@ func TestNewSink(t *testing.T) { {metrics.Counter, &counter{}}, {metrics.Gauge, &gauge{}}, {metrics.Rate, &rate{}}, - { - metrics.Trend, - &histogram{ - Buckets: map[uint32]uint32{}, - Max: -math.MaxFloat64, - Min: math.MaxFloat64, - }, - }, + {metrics.Trend, newHistogram()}, } for _, tc := range tests { assert.Equal(t, tc.exp, newMetricValue(tc.mt)) From 4f49f360973443390b93735155fd78ae33bd2132 Mon Sep 17 00:00:00 2001 From: Ivan <2103732+codebien@users.noreply.github.com> Date: Fri, 7 Jul 2023 16:13:58 +0200 Subject: [PATCH 8/8] Sort the buckets during the Proto generation --- output/cloud/expv2/hdr.go | 52 +++++++++++----------------------- output/cloud/expv2/hdr_test.go | 7 ----- 2 files changed, 17 insertions(+), 42 deletions(-) diff --git a/output/cloud/expv2/hdr.go b/output/cloud/expv2/hdr.go index 120550ed3b8..8407a776463 100644 --- a/output/cloud/expv2/hdr.go +++ b/output/cloud/expv2/hdr.go @@ -42,10 +42,6 @@ type histogram struct { // because they contain exception cases and require to be tracked in a dedicated way. Buckets map[uint32]uint32 - // Indexes keeps an ordered slice of unique-seen buckets' indexes. - // It allows to iterate the buckets in order. It uses an ascendent order. - Indexes []uint32 - // ExtraLowBucket counts occurrences of observed values smaller // than the minimum trackable value. ExtraLowBucket uint32 @@ -98,55 +94,41 @@ func (h *histogram) addToBucket(v float64) { return } - ix := resolveBucketIndex(v) - if _, contains := h.Buckets[ix]; !contains { - h.trackBucket(ix) - } - - h.Buckets[ix]++ -} - -// trackBucket stores the unique seen buckets. -func (h *histogram) trackBucket(index uint32) { - i := sort.Search(len(h.Indexes), func(i int) bool { - return h.Indexes[i] > index - }) - - // insert at the end - if len(h.Indexes) == i { - h.Indexes = append(h.Indexes, index) - return - } - - // insert at specific `i` - h.Indexes = append(h.Indexes, 0) - copy(h.Indexes[i+1:], h.Indexes[i:]) - h.Indexes[i] = index + h.Buckets[resolveBucketIndex(v)]++ } // histogramAsProto converts the histogram into the equivalent Protobuf version. func histogramAsProto(h *histogram, time int64) *pbcloud.TrendHdrValue { var ( + indexes []uint32 counters []uint32 spans []*pbcloud.BucketSpan ) // allocate only if at least one item is available, in the case of only // untrackable values, then Indexes and Buckets are expected to be empty. - if len(h.Indexes) > 0 { + if len(h.Buckets) > 0 { + indexes = make([]uint32, 0, len(h.Buckets)) + for index := range h.Buckets { + indexes = append(indexes, index) + } + sort.Slice(indexes, func(i, j int) bool { + return indexes[i] < indexes[j] + }) + // init the counters - counters = make([]uint32, 1, len(h.Indexes)) - counters[0] = h.Buckets[h.Indexes[0]] + counters = make([]uint32, 1, len(h.Buckets)) + counters[0] = h.Buckets[indexes[0]] // open the first span - spans = append(spans, &pbcloud.BucketSpan{Offset: h.Indexes[0], Length: 1}) + spans = append(spans, &pbcloud.BucketSpan{Offset: indexes[0], Length: 1}) } - for i := 1; i < len(h.Buckets); i++ { - counters = append(counters, h.Buckets[h.Indexes[i]]) + for i := 1; i < len(indexes); i++ { + counters = append(counters, h.Buckets[indexes[i]]) // if the current and the previous indexes are not consecutive // consider as closed the current on-going span and start a new one. - if diff := h.Indexes[i] - h.Indexes[i-1]; diff > 1 { + if diff := indexes[i] - indexes[i-1]; diff > 1 { spans = append(spans, &pbcloud.BucketSpan{Offset: diff, Length: 1}) continue } diff --git a/output/cloud/expv2/hdr_test.go b/output/cloud/expv2/hdr_test.go index 797e3863a52..734510634d6 100644 --- a/output/cloud/expv2/hdr_test.go +++ b/output/cloud/expv2/hdr_test.go @@ -53,7 +53,6 @@ func TestHistogramAddWithSimpleValues(t *testing.T) { vals: []float64{0}, exp: histogram{ Buckets: map[uint32]uint32{0: 1}, - Indexes: []uint32{0}, ExtraLowBucket: 0, ExtraHighBucket: 0, Max: 0, @@ -66,7 +65,6 @@ func TestHistogramAddWithSimpleValues(t *testing.T) { vals: []float64{8, 5}, exp: histogram{ Buckets: map[uint32]uint32{5: 1, 8: 1}, - Indexes: []uint32{5, 8}, ExtraLowBucket: 0, ExtraHighBucket: 0, Max: 8, @@ -79,7 +77,6 @@ func TestHistogramAddWithSimpleValues(t *testing.T) { vals: []float64{8, 9, 10, 5}, exp: histogram{ Buckets: map[uint32]uint32{8: 1, 9: 1, 10: 1, 5: 1}, - Indexes: []uint32{5, 8, 9, 10}, ExtraLowBucket: 0, ExtraHighBucket: 0, Max: 10, @@ -92,7 +89,6 @@ func TestHistogramAddWithSimpleValues(t *testing.T) { vals: []float64{100, 101}, exp: histogram{ Buckets: map[uint32]uint32{100: 1, 101: 1}, - Indexes: []uint32{100, 101}, ExtraLowBucket: 0, ExtraHighBucket: 0, Max: 101, @@ -105,7 +101,6 @@ func TestHistogramAddWithSimpleValues(t *testing.T) { vals: []float64{101, 100}, exp: histogram{ Buckets: map[uint32]uint32{100: 1, 101: 1}, - Indexes: []uint32{100, 101}, ExtraLowBucket: 0, ExtraHighBucket: 0, Max: 101, @@ -139,7 +134,6 @@ func TestHistogramAddWithUntrackables(t *testing.T) { exp := &histogram{ Buckets: map[uint32]uint32{1: 1, 5: 1}, - Indexes: []uint32{1, 5}, ExtraLowBucket: 1, ExtraHighBucket: 1, Max: 2 * 1e9, @@ -160,7 +154,6 @@ func TestHistogramAddWithMultipleOccurances(t *testing.T) { exp := &histogram{ Buckets: map[uint32]uint32{52: 1, 104: 4}, - Indexes: []uint32{52, 104}, Max: 103.6, Min: 51.8, ExtraLowBucket: 0,