enhance: Add a counter monitoring for the rate-limit requests (#30109)

Add a counter monitoring metric for the ratelimited rpc requests with labels: proxy nodeID, rpc request type, and state. issue: #30052 Signed-off-by: bigsheeper <[email protected]>
milvus-io · Jan 21, 2024 · c8a7846 · c8a7846
1 parent 4f44942
commit c8a7846
Show file tree

Hide file tree

Showing 2 changed files with 17 additions and 0 deletions.
diff --git a/internal/proxy/rate_limit_interceptor.go b/internal/proxy/rate_limit_interceptor.go
@@ -20,14 +20,17 @@ import (
 	"context"
 	"fmt"
 	"reflect"
+	"strconv"
 
 	"github.com/golang/protobuf/proto"
 	"google.golang.org/grpc"
 
 	"github.com/milvus-io/milvus-proto/go-api/v2/milvuspb"
 	"github.com/milvus-io/milvus/internal/proto/internalpb"
 	"github.com/milvus-io/milvus/internal/types"
+	"github.com/milvus-io/milvus/pkg/metrics"
 	"github.com/milvus-io/milvus/pkg/util/merr"
+	"github.com/milvus-io/milvus/pkg/util/paramtable"
 )
 
 // RateLimitInterceptor returns a new unary server interceptors that performs request rate limiting.
@@ -39,12 +42,16 @@ func RateLimitInterceptor(limiter types.Limiter) grpc.UnaryServerInterceptor {
 		}
 
 		err = limiter.Check(collectionID, rt, n)
+		nodeID := strconv.FormatInt(paramtable.GetNodeID(), 10)
+		metrics.ProxyRateLimitReqCount.WithLabelValues(nodeID, rt.String(), metrics.TotalLabel).Inc()
 		if err != nil {
+			metrics.ProxyRateLimitReqCount.WithLabelValues(nodeID, rt.String(), metrics.FailLabel).Inc()
 			rsp := getFailedResponse(req, err)
 			if rsp != nil {
 				return rsp, nil
 			}
 		}
+		metrics.ProxyRateLimitReqCount.WithLabelValues(nodeID, rt.String(), metrics.SuccessLabel).Inc()
 		return handler(ctx, req)
 	}
 }

diff --git a/pkg/metrics/proxy_metrics.go b/pkg/metrics/proxy_metrics.go
@@ -297,6 +297,15 @@ var (
 		}, []string{
 			nodeIDLabelName,
 		})
+
+	// ProxyRateLimitReqCount integrates a counter monitoring metric for the rate-limit rpc requests.
+	ProxyRateLimitReqCount = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: milvusNamespace,
+			Subsystem: typeutil.ProxyRole,
+			Name:      "rate_limit_req_count",
+			Help:      "count of operation executed",
+		}, []string{nodeIDLabelName, msgTypeLabelName, statusLabelName})
 )
 
 // RegisterProxy registers Proxy metrics
@@ -341,6 +350,7 @@ func RegisterProxy(registry *prometheus.Registry) {
 
 	registry.MustRegister(ProxyWorkLoadScore)
 	registry.MustRegister(ProxyExecutingTotalNq)
+	registry.MustRegister(ProxyRateLimitReqCount)
 }
 
 func CleanupCollectionMetrics(nodeID int64, collection string) {