metrics: Add histograms for duration metrics

Deprecate `prometheus.Counter` analogs. Closes #2351.

Signed-off-by: Pavel Karpy <[email protected]>

CHANGELOG.md

@@ -38,6 +38,11 @@ on timeout, try increasing the value, for example, twice. Also note that the
 execution of commands with the `--await` flag and without an explicitly
 specified time period is now limited to 1 minute. This value can be changed with
 `--timeout` flag.
+Histogram (not counter) RPC/engine operation handling time metrics were added. For
+an old engine `*operation_name*_duration` a new `*operation_name*_time` is available.
+For an old `*operation_name*_req_duration` RPC a new `rpc_*operation_name*_time` is
+available. The old ones (the counters) have been deprecated and will be removed with
+the following minor release.
 
 Deprecated `morph.rpc_endpoint` SN and `morph.endpoint.client` IR configurations
 have been removed. Use `morph.endpoints` for both instead.

pkg/metrics/engine.go

@@ -8,103 +8,197 @@ import (
 
 type (
 	engineMetrics struct {
-		listContainersDuration        prometheus.Counter
-		estimateContainerSizeDuration prometheus.Counter
-		deleteDuration                prometheus.Counter
-		existsDuration                prometheus.Counter
-		getDuration                   prometheus.Counter
-		headDuration                  prometheus.Counter
-		inhumeDuration                prometheus.Counter
-		putDuration                   prometheus.Counter
-		rangeDuration                 prometheus.Counter
-		searchDuration                prometheus.Counter
-		listObjectsDuration           prometheus.Counter
-		containerSize                 prometheus.GaugeVec
-		payloadSize                   prometheus.GaugeVec
+		listContainersDuration_counter        prometheus.Counter
+		estimateContainerSizeDuration_counter prometheus.Counter
+		deleteDuration_counter                prometheus.Counter
+		existsDuration_counter                prometheus.Counter
+		getDuration_counter                   prometheus.Counter
+		headDuration_counter                  prometheus.Counter
+		inhumeDuration_counter                prometheus.Counter
+		putDuration_counter                   prometheus.Counter
+		rangeDuration_counter                 prometheus.Counter
+		searchDuration_counter                prometheus.Counter
+		listObjectsDuration_counter           prometheus.Counter
+
+		listContainersDuration        prometheus.Histogram
+		estimateContainerSizeDuration prometheus.Histogram
+		deleteDuration                prometheus.Histogram
+		existsDuration                prometheus.Histogram
+		getDuration                   prometheus.Histogram
+		headDuration                  prometheus.Histogram
+		inhumeDuration                prometheus.Histogram
+		putDuration                   prometheus.Histogram
+		rangeDuration                 prometheus.Histogram
+		searchDuration                prometheus.Histogram
+		listObjectsDuration           prometheus.Histogram
+
+		containerSize prometheus.GaugeVec
+		payloadSize   prometheus.GaugeVec
 	}
 )
 
 const engineSubsystem = "engine"
 
 func newEngineMetrics() engineMetrics {
 	var (
-		listContainersDuration = prometheus.NewCounter(prometheus.CounterOpts{
+		listContainersDuration = prometheus.NewHistogram(prometheus.HistogramOpts{
+			Namespace: storageNodeNameSpace,
+			Subsystem: engineSubsystem,
+			Name:      "list_containers_time",
+			Help:      "Engine list containers handling time",
+		})
+
+		estimateContainerSizeDuration = prometheus.NewHistogram(prometheus.HistogramOpts{
+			Namespace: storageNodeNameSpace,
+			Subsystem: engineSubsystem,
+			Name:      "estimate_container_size_time",
+			Help:      "Engine container size handling time",
+		})
+
+		deleteDuration = prometheus.NewHistogram(prometheus.HistogramOpts{
+			Namespace: storageNodeNameSpace,
+			Subsystem: engineSubsystem,
+			Name:      "delete_time",
+			Help:      "Engine delete operations handling time",
+		})
+
+		existsDuration = prometheus.NewHistogram(prometheus.HistogramOpts{
+			Namespace: storageNodeNameSpace,
+			Subsystem: engineSubsystem,
+			Name:      "exists_time",
+			Help:      "Engine exists operations handling time",
+		})
+
+		getDuration = prometheus.NewHistogram(prometheus.HistogramOpts{
+			Namespace: storageNodeNameSpace,
+			Subsystem: engineSubsystem,
+			Name:      "get_time",
+			Help:      "Engine get operations handling time",
+		})
+
+		headDuration = prometheus.NewHistogram(prometheus.HistogramOpts{
+			Namespace: storageNodeNameSpace,
+			Subsystem: engineSubsystem,
+			Name:      "head_time",
+			Help:      "Engine head operations handling time",
+		})
+
+		inhumeDuration = prometheus.NewHistogram(prometheus.HistogramOpts{
+			Namespace: storageNodeNameSpace,
+			Subsystem: engineSubsystem,
+			Name:      "inhume_time",
+			Help:      "Engine inhume operations handling time",
+		})
+
+		putDuration = prometheus.NewHistogram(prometheus.HistogramOpts{
+			Namespace: storageNodeNameSpace,
+			Subsystem: engineSubsystem,
+			Name:      "put_time",
+			Help:      "Engine put operations handling time",
+		})
+
+		rangeDuration = prometheus.NewHistogram(prometheus.HistogramOpts{
+			Namespace: storageNodeNameSpace,
+			Subsystem: engineSubsystem,
+			Name:      "range_time",
+			Help:      "Engine range operations handling time",
+		})
+
+		searchDuration = prometheus.NewHistogram(prometheus.HistogramOpts{
+			Namespace: storageNodeNameSpace,
+			Subsystem: engineSubsystem,
+			Name:      "search_time",
+			Help:      "Engine search operations handling time",
+		})
+
+		listObjectsDuration = prometheus.NewHistogram(prometheus.HistogramOpts{
+			Namespace: storageNodeNameSpace,
+			Subsystem: engineSubsystem,
+			Name:      "list_objects_time",
+			Help:      "Engine list objects operations handling time",
+		})
+	)
+
+	var (
+		listContainersDuration_counter = prometheus.NewCounter(prometheus.CounterOpts{
 			Namespace: storageNodeNameSpace,
 			Subsystem: engineSubsystem,
 			Name:      "list_containers_duration",
-			Help:      "Accumulated duration of engine list containers operations",
+			Help:      "Accumulated duration of engine list containers operations [DEPRECATED]",
 		})
 
-		estimateContainerSizeDuration = prometheus.NewCounter(prometheus.CounterOpts{
+		estimateContainerSizeDuration_counter = prometheus.NewCounter(prometheus.CounterOpts{
 			Namespace: storageNodeNameSpace,
 			Subsystem: engineSubsystem,
 			Name:      "estimate_container_size_duration",
-			Help:      "Accumulated duration of engine container size estimate operations",
+			Help:      "Accumulated duration of engine container size estimate operations [DEPRECATED]",
 		})
 
-		deleteDuration = prometheus.NewCounter(prometheus.CounterOpts{
+		deleteDuration_counter = prometheus.NewCounter(prometheus.CounterOpts{
 			Namespace: storageNodeNameSpace,
 			Subsystem: engineSubsystem,
 			Name:      "delete_duration",
-			Help:      "Accumulated duration of engine delete operations",
+			Help:      "Accumulated duration of engine delete operations [DEPRECATED]",
 		})
 
-		existsDuration = prometheus.NewCounter(prometheus.CounterOpts{
+		existsDuration_counter = prometheus.NewCounter(prometheus.CounterOpts{
 			Namespace: storageNodeNameSpace,
 			Subsystem: engineSubsystem,
 			Name:      "exists_duration",
-			Help:      "Accumulated duration of engine exists operations",
+			Help:      "Accumulated duration of engine exists operations [DEPRECATED]",
 		})
 
-		getDuration = prometheus.NewCounter(prometheus.CounterOpts{
+		getDuration_counter = prometheus.NewCounter(prometheus.CounterOpts{
 			Namespace: storageNodeNameSpace,
 			Subsystem: engineSubsystem,
 			Name:      "get_duration",
-			Help:      "Accumulated duration of engine get operations",
+			Help:      "Accumulated duration of engine get operations [DEPRECATED]",
 		})
 
-		headDuration = prometheus.NewCounter(prometheus.CounterOpts{
+		headDuration_counter = prometheus.NewCounter(prometheus.CounterOpts{
 			Namespace: storageNodeNameSpace,
 			Subsystem: engineSubsystem,
 			Name:      "head_duration",
-			Help:      "Accumulated duration of engine head operations",
+			Help:      "Accumulated duration of engine head operations [DEPRECATED]",
 		})
 
-		inhumeDuration = prometheus.NewCounter(prometheus.CounterOpts{
+		inhumeDuration_counter = prometheus.NewCounter(prometheus.CounterOpts{
 			Namespace: storageNodeNameSpace,
 			Subsystem: engineSubsystem,
 			Name:      "inhume_duration",
-			Help:      "Accumulated duration of engine inhume operations",
+			Help:      "Accumulated duration of engine inhume operations [DEPRECATED]",
 		})
 
-		putDuration = prometheus.NewCounter(prometheus.CounterOpts{
+		putDuration_counter = prometheus.NewCounter(prometheus.CounterOpts{
 			Namespace: storageNodeNameSpace,
 			Subsystem: engineSubsystem,
 			Name:      "put_duration",
-			Help:      "Accumulated duration of engine put operations",
+			Help:      "Accumulated duration of engine put operations [DEPRECATED]",
 		})
 
-		rangeDuration = prometheus.NewCounter(prometheus.CounterOpts{
+		rangeDuration_counter = prometheus.NewCounter(prometheus.CounterOpts{
 			Namespace: storageNodeNameSpace,
 			Subsystem: engineSubsystem,
 			Name:      "range_duration",
-			Help:      "Accumulated duration of engine range operations",
+			Help:      "Accumulated duration of engine range operations [DEPRECATED]",
 		})
 
-		searchDuration = prometheus.NewCounter(prometheus.CounterOpts{
+		searchDuration_counter = prometheus.NewCounter(prometheus.CounterOpts{
 			Namespace: storageNodeNameSpace,
 			Subsystem: engineSubsystem,
 			Name:      "search_duration",
-			Help:      "Accumulated duration of engine search operations",
+			Help:      "Accumulated duration of engine search operations [DEPRECATED]",
 		})
 
-		listObjectsDuration = prometheus.NewCounter(prometheus.CounterOpts{
+		listObjectsDuration_counter = prometheus.NewCounter(prometheus.CounterOpts{
 			Namespace: storageNodeNameSpace,
 			Subsystem: engineSubsystem,
 			Name:      "list_objects_duration",
-			Help:      "Accumulated duration of engine list objects operations",
+			Help:      "Accumulated duration of engine list objects operations [DEPRECATED]",
 		})
+	)
 
+	var (
 		containerSize = prometheus.NewGaugeVec(prometheus.GaugeOpts{
 			Namespace: storageNodeNameSpace,
 			Subsystem: engineSubsystem,
@@ -134,6 +228,18 @@ func newEngineMetrics() engineMetrics {
 		listObjectsDuration:           listObjectsDuration,
 		containerSize:                 *containerSize,
 		payloadSize:                   *payloadSize,
+
+		listContainersDuration_counter:        listContainersDuration_counter,
+		estimateContainerSizeDuration_counter: estimateContainerSizeDuration_counter,
+		deleteDuration_counter:                deleteDuration_counter,
+		existsDuration_counter:                existsDuration_counter,
+		getDuration_counter:                   getDuration_counter,
+		headDuration_counter:                  headDuration_counter,
+		inhumeDuration_counter:                inhumeDuration_counter,
+		putDuration_counter:                   putDuration_counter,
+		rangeDuration_counter:                 rangeDuration_counter,
+		searchDuration_counter:                searchDuration_counter,
+		listObjectsDuration_counter:           listObjectsDuration_counter,
 	}
 }
 
@@ -151,50 +257,73 @@ func (m engineMetrics) register() {
 	prometheus.MustRegister(m.listObjectsDuration)
 	prometheus.MustRegister(m.containerSize)
 	prometheus.MustRegister(m.payloadSize)
+
+	prometheus.MustRegister(m.listContainersDuration_counter)
+	prometheus.MustRegister(m.estimateContainerSizeDuration_counter)
+	prometheus.MustRegister(m.deleteDuration_counter)
+	prometheus.MustRegister(m.existsDuration_counter)
+	prometheus.MustRegister(m.getDuration_counter)
+	prometheus.MustRegister(m.headDuration_counter)
+	prometheus.MustRegister(m.inhumeDuration_counter)
+	prometheus.MustRegister(m.putDuration_counter)
+	prometheus.MustRegister(m.rangeDuration_counter)
+	prometheus.MustRegister(m.searchDuration_counter)
+	prometheus.MustRegister(m.listObjectsDuration_counter)
 }
 
 func (m engineMetrics) AddListContainersDuration(d time.Duration) {
-	m.listObjectsDuration.Add(float64(d))
+	m.listObjectsDuration_counter.Add(float64(d))
+	m.listObjectsDuration.Observe(d.Seconds())
 }
 
 func (m engineMetrics) AddEstimateContainerSizeDuration(d time.Duration) {
-	m.estimateContainerSizeDuration.Add(float64(d))
+	m.estimateContainerSizeDuration_counter.Add(float64(d))
+	m.estimateContainerSizeDuration.Observe(d.Seconds())
 }
 
 func (m engineMetrics) AddDeleteDuration(d time.Duration) {
-	m.deleteDuration.Add(float64(d))
+	m.deleteDuration_counter.Add(float64(d))
+	m.deleteDuration.Observe(d.Seconds())
 }
 
 func (m engineMetrics) AddExistsDuration(d time.Duration) {
-	m.existsDuration.Add(float64(d))
+	m.existsDuration_counter.Add(float64(d))
+	m.existsDuration.Observe(d.Seconds())
 }
 
 func (m engineMetrics) AddGetDuration(d time.Duration) {
-	m.getDuration.Add(float64(d))
+	m.getDuration_counter.Add(float64(d))
+	m.getDuration.Observe(d.Seconds())
 }
 
 func (m engineMetrics) AddHeadDuration(d time.Duration) {
-	m.headDuration.Add(float64(d))
+	m.headDuration_counter.Add(float64(d))
+	m.headDuration.Observe(d.Seconds())
 }
 
 func (m engineMetrics) AddInhumeDuration(d time.Duration) {
-	m.inhumeDuration.Add(float64(d))
+	m.inhumeDuration_counter.Add(float64(d))
+	m.inhumeDuration.Observe(d.Seconds())
 }
 
 func (m engineMetrics) AddPutDuration(d time.Duration) {
-	m.putDuration.Add(float64(d))
+	m.putDuration_counter.Add(float64(d))
+	m.putDuration.Observe(d.Seconds())
 }
 
 func (m engineMetrics) AddRangeDuration(d time.Duration) {
-	m.rangeDuration.Add(float64(d))
+	m.rangeDuration_counter.Add(float64(d))
+	m.rangeDuration.Observe(d.Seconds())
 }
 
 func (m engineMetrics) AddSearchDuration(d time.Duration) {
-	m.searchDuration.Add(float64(d))
+	m.searchDuration_counter.Add(float64(d))
+	m.searchDuration.Observe(d.Seconds())
 }
 
 func (m engineMetrics) AddListObjectsDuration(d time.Duration) {
-	m.listObjectsDuration.Add(float64(d))
+	m.listObjectsDuration_counter.Add(float64(d))
+	m.listObjectsDuration.Observe(d.Seconds())
 }
 
 func (m engineMetrics) AddToContainerSize(cnrID string, size int64) {