Merge pull request #5519 from oasisprotocol/kostko/feature/rhp-timeou…

…t-metric go/runtime/host: Add separate metric for RHP call timeouts
oasisprotocol · Jan 11, 2024 · 5b7def1 · 5b7def1
2 parents 2b26409 + e6a2244
commit 5b7def1
Show file tree

Hide file tree

Showing 3 changed files with 14 additions and 0 deletions.
diff --git a/.changelog/5519.feature.md b/.changelog/5519.feature.md
@@ -0,0 +1 @@
+go/runtime/host: Add separate metric for RHP call timeouts
diff --git a/docs/oasis-node/metrics.md b/docs/oasis-node/metrics.md
@@ -77,6 +77,7 @@ oasis_registry_runtimes | Gauge | Number of registry runtimes. |  | [registry](h
 oasis_rhp_failures | Counter | Number of failed Runtime Host calls. | call | [runtime/host/protocol](https://github.com/oasisprotocol/oasis-core/tree/master/go/runtime/host/protocol/connection.go)
 oasis_rhp_latency | Summary | Runtime Host call latency (seconds). | call | [runtime/host/protocol](https://github.com/oasisprotocol/oasis-core/tree/master/go/runtime/host/protocol/connection.go)
 oasis_rhp_successes | Counter | Number of successful Runtime Host calls. | call | [runtime/host/protocol](https://github.com/oasisprotocol/oasis-core/tree/master/go/runtime/host/protocol/connection.go)
+oasis_rhp_timeouts | Counter | Number of timed out Runtime Host calls. |  | [runtime/host/protocol](https://github.com/oasisprotocol/oasis-core/tree/master/go/runtime/host/protocol/connection.go)
 oasis_roothash_block_interval | Summary | Time between roothash blocks (seconds). | runtime | [roothash](https://github.com/oasisprotocol/oasis-core/tree/master/go/roothash/metrics.go)
 oasis_storage_failures | Counter | Number of storage failures. | call | [storage/api](https://github.com/oasisprotocol/oasis-core/tree/master/go/storage/api/metrics.go)
 oasis_storage_latency | Summary | Storage call latency (seconds). | call | [storage/api](https://github.com/oasisprotocol/oasis-core/tree/master/go/storage/api/metrics.go)

diff --git a/go/runtime/host/protocol/connection.go b/go/runtime/host/protocol/connection.go
@@ -49,11 +49,18 @@ var (
 		},
 		[]string{"call"},
 	)
+	rhpCallTimeouts = prometheus.NewCounter(
+		prometheus.CounterOpts{
+			Name: "oasis_rhp_timeouts",
+			Help: "Number of timed out Runtime Host calls.",
+		},
+	)
 
 	rhpCollectors = []prometheus.Collector{
 		rhpLatency,
 		rhpCallSuccesses,
 		rhpCallFailures,
+		rhpCallTimeouts,
 	}
 
 	metricsOnce sync.Once
@@ -287,6 +294,11 @@ func (c *connection) call(ctx context.Context, body *Body) (result *Body, err er
 			rhpLatency.With(prometheus.Labels{"call": body.Type()}).Observe(time.Since(start).Seconds())
 			if err != nil {
 				rhpCallFailures.With(prometheus.Labels{"call": body.Type()}).Inc()
+
+				// Specifically measure timeouts.
+				if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
+					rhpCallTimeouts.Inc()
+				}
 			} else {
 				rhpCallSuccesses.With(prometheus.Labels{"call": body.Type()}).Inc()
 			}