metrics, router: add more metrics (#416)

pingcap · Dec 22, 2023 · f64bde1 · f64bde1
1 parent 1a29ee5
commit f64bde1
Show file tree

Hide file tree

Showing 12 changed files with 573 additions and 93 deletions.
diff --git a/pkg/manager/router/backend_observer.go b/pkg/manager/router/backend_observer.go
@@ -15,6 +15,7 @@ import (
 	"github.com/pingcap/tiproxy/lib/config"
 	"github.com/pingcap/tiproxy/lib/util/errors"
 	"github.com/pingcap/tiproxy/lib/util/waitgroup"
+	"github.com/pingcap/tiproxy/pkg/metrics"
 	pnet "github.com/pingcap/tiproxy/pkg/proxy/net"
 	"go.uber.org/zap"
 )
@@ -156,6 +157,7 @@ func (bo *BackendObserver) Refresh() {
 
 func (bo *BackendObserver) observe(ctx context.Context) {
 	for ctx.Err() == nil {
+		startTime := time.Now()
 		backendInfo, err := bo.fetcher.GetBackendList(ctx)
 		if err != nil {
 			bo.logger.Error("fetching backends encounters error", zap.Error(err))
@@ -167,11 +169,17 @@ func (bo *BackendObserver) observe(ctx context.Context) {
 			}
 			bo.notifyIfChanged(bhMap)
 		}
-		select {
-		case <-time.After(bo.healthCheckConfig.Interval):
-		case <-bo.refreshChan:
-		case <-ctx.Done():
-			return
+
+		cost := time.Since(startTime)
+		metrics.HealthCheckCycleGauge.Set(cost.Seconds())
+		wait := bo.healthCheckConfig.Interval - cost
+		if wait > 0 {
+			select {
+			case <-time.After(wait):
+			case <-bo.refreshChan:
+			case <-ctx.Done():
+				return
+			}
 		}
 	}
 }

diff --git a/pkg/manager/router/backend_observer_test.go b/pkg/manager/router/backend_observer_test.go
@@ -172,6 +172,10 @@ func (ts *observerTestSuite) checkStatus(backend *backendServer, expectedStatus
 	require.True(ts.t, ok)
 	require.Equal(ts.t, expectedStatus, health.status)
 	require.True(ts.t, checkBackendStatusMetrics(backend.sqlAddr, health.status))
+	cycle, err := readHealthCheckCycle()
+	require.NoError(ts.t, err)
+	require.Greater(ts.t, cycle.Nanoseconds(), int64(0))
+	require.Less(ts.t, cycle.Nanoseconds(), 3*time.Second)
 }
 
 func (ts *observerTestSuite) getBackendsFromCh() map[string]*backendHealth {

diff --git a/pkg/manager/router/metrics.go b/pkg/manager/router/metrics.go
@@ -19,15 +19,16 @@ func checkBackendStatusMetrics(addr string, status BackendStatus) bool {
 	if err != nil {
 		return false
 	}
-	return val == 1
+	return int(val) == 1
 }
 
 func setBackendConnMetrics(addr string, conns int) {
 	metrics.BackendConnGauge.WithLabelValues(addr).Set(float64(conns))
 }
 
 func readBackendConnMetrics(addr string) (int, error) {
-	return metrics.ReadGauge(metrics.BackendConnGauge.WithLabelValues(addr))
+	val, err := metrics.ReadGauge(metrics.BackendConnGauge.WithLabelValues(addr))
+	return int(val), err
 }
 
 func succeedToLabel(succeed bool) string {
@@ -53,3 +54,8 @@ func setPingBackendMetrics(addr string, succeed bool, startTime time.Time) {
 	cost := time.Since(startTime)
 	metrics.PingBackendGauge.WithLabelValues(addr).Set(cost.Seconds())
 }
+
+func readHealthCheckCycle() (time.Duration, error) {
+	seconds, err := metrics.ReadGauge(metrics.HealthCheckCycleGauge)
+	return time.Duration(int(seconds * float64(time.Second))), err
+}
diff --git a/pkg/metrics/backend.go b/pkg/metrics/backend.go
@@ -43,4 +43,12 @@ var (
 			Name:      "ping_duration_seconds",
 			Help:      "Time (s) of pinging the SQL port of each backend.",
 		}, []string{LblBackend})
+
+	HealthCheckCycleGauge = prometheus.NewGauge(
+		prometheus.GaugeOpts{
+			Namespace: ModuleProxy,
+			Subsystem: LabelBackend,
+			Name:      "health_check_seconds",
+			Help:      "Time (s) of each health check cycle.",
+		})
 )