diff --git a/internal/proxy/rate_limit_interceptor.go b/internal/proxy/rate_limit_interceptor.go index 218ccea094ede..ad79ba58f72d8 100644 --- a/internal/proxy/rate_limit_interceptor.go +++ b/internal/proxy/rate_limit_interceptor.go @@ -20,6 +20,7 @@ import ( "context" "fmt" "reflect" + "strconv" "github.com/golang/protobuf/proto" "google.golang.org/grpc" @@ -27,7 +28,9 @@ import ( "github.com/milvus-io/milvus-proto/go-api/v2/milvuspb" "github.com/milvus-io/milvus/internal/proto/internalpb" "github.com/milvus-io/milvus/internal/types" + "github.com/milvus-io/milvus/pkg/metrics" "github.com/milvus-io/milvus/pkg/util/merr" + "github.com/milvus-io/milvus/pkg/util/paramtable" ) // RateLimitInterceptor returns a new unary server interceptors that performs request rate limiting. @@ -39,12 +42,16 @@ func RateLimitInterceptor(limiter types.Limiter) grpc.UnaryServerInterceptor { } err = limiter.Check(collectionID, rt, n) + nodeID := strconv.FormatInt(paramtable.GetNodeID(), 10) + metrics.ProxyRateLimitReqCount.WithLabelValues(nodeID, rt.String(), metrics.TotalLabel).Inc() if err != nil { + metrics.ProxyRateLimitReqCount.WithLabelValues(nodeID, rt.String(), metrics.FailLabel).Inc() rsp := getFailedResponse(req, err) if rsp != nil { return rsp, nil } } + metrics.ProxyRateLimitReqCount.WithLabelValues(nodeID, rt.String(), metrics.SuccessLabel).Inc() return handler(ctx, req) } } diff --git a/pkg/metrics/proxy_metrics.go b/pkg/metrics/proxy_metrics.go index 9e82173f9a0bf..e2349adff9316 100644 --- a/pkg/metrics/proxy_metrics.go +++ b/pkg/metrics/proxy_metrics.go @@ -297,6 +297,15 @@ var ( }, []string{ nodeIDLabelName, }) + + // ProxyRateLimitReqCount integrates a counter monitoring metric for the rate-limit rpc requests. + ProxyRateLimitReqCount = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: milvusNamespace, + Subsystem: typeutil.ProxyRole, + Name: "rate_limit_req_count", + Help: "count of operation executed", + }, []string{nodeIDLabelName, msgTypeLabelName, statusLabelName}) ) // RegisterProxy registers Proxy metrics @@ -341,6 +350,7 @@ func RegisterProxy(registry *prometheus.Registry) { registry.MustRegister(ProxyWorkLoadScore) registry.MustRegister(ProxyExecutingTotalNq) + registry.MustRegister(ProxyRateLimitReqCount) } func CleanupCollectionMetrics(nodeID int64, collection string) {