From ab58c812bdaf57377d7e9729ae5d4f066bde9bae Mon Sep 17 00:00:00 2001 From: Alexey Kuzbekov Date: Thu, 30 May 2024 00:42:09 +0100 Subject: [PATCH 1/2] New flag to disable detailed metrics for path Flag detailed-metrics added to provide a way to disable exposing all accessed paths to the metrics and prevent potential metrics cardinality explosion Signed-off-by: Alexey Kuzbekov --- .../controller/receiver_controller_test.go | 2 +- internal/server/event_server.go | 13 ++++++++--- internal/server/event_server_test.go | 2 +- internal/server/receiver_server.go | 22 ++++++++++++------- main.go | 6 +++-- 5 files changed, 30 insertions(+), 15 deletions(-) diff --git a/internal/controller/receiver_controller_test.go b/internal/controller/receiver_controller_test.go index 3b4da9cbd..7d80953b3 100644 --- a/internal/controller/receiver_controller_test.go +++ b/internal/controller/receiver_controller_test.go @@ -239,7 +239,7 @@ func TestReceiverReconciler_EventHandler(t *testing.T) { // Use the client from the manager as the server handler needs to list objects from the cache // which the "live" k8s client does not have access to. - receiverServer := server.NewReceiverServer("127.0.0.1:56788", logf.Log, testEnv.GetClient()) + receiverServer := server.NewReceiverServer("127.0.0.1:56788", logf.Log, testEnv.GetClient(), true) receiverMdlw := middleware.New(middleware.Config{ Recorder: prommetrics.NewRecorder(prommetrics.Config{ Prefix: "gotk_receiver", diff --git a/internal/server/event_server.go b/internal/server/event_server.go index 5ec4704e8..8cb75775b 100644 --- a/internal/server/event_server.go +++ b/internal/server/event_server.go @@ -52,17 +52,19 @@ type EventServer struct { logger logr.Logger kubeClient client.Client noCrossNamespaceRefs bool + detailedMetrics bool kuberecorder.EventRecorder } // NewEventServer returns an HTTP server that handles events -func NewEventServer(port string, logger logr.Logger, kubeClient client.Client, eventRecorder kuberecorder.EventRecorder, noCrossNamespaceRefs bool) *EventServer { +func NewEventServer(port string, logger logr.Logger, kubeClient client.Client, eventRecorder kuberecorder.EventRecorder, noCrossNamespaceRefs bool, detailedMetrics bool) *EventServer { return &EventServer{ port: port, logger: logger.WithName("event-server"), kubeClient: kubeClient, EventRecorder: eventRecorder, noCrossNamespaceRefs: noCrossNamespaceRefs, + detailedMetrics: detailedMetrics, } } @@ -82,8 +84,13 @@ func (s *EventServer) ListenAndServe(stopCh <-chan struct{}, mdlw middleware.Mid handler = middleware(handler) } mux := http.NewServeMux() - mux.Handle("/", handler) - h := std.Handler("", mdlw, mux) + path := "/" + mux.Handle(path, handler) + middlewarePath := "" + if !s.detailedMetrics { + middlewarePath = path + } + h := std.Handler(middlewarePath, mdlw, mux) srv := &http.Server{ Addr: s.port, Handler: h, diff --git a/internal/server/event_server_test.go b/internal/server/event_server_test.go index 9aec6ace0..1b47a3269 100644 --- a/internal/server/event_server_test.go +++ b/internal/server/event_server_test.go @@ -137,7 +137,7 @@ func TestEventServer(t *testing.T) { t.Fatalf("failed to create memory storage") } eventServer := NewEventServer("127.0.0.1:"+eventServerPort, - log.Log, kclient, record.NewFakeRecorder(32), true) + log.Log, kclient, record.NewFakeRecorder(32), true, true) stopCh := make(chan struct{}) go eventServer.ListenAndServe(stopCh, eventMdlw, store) defer close(stopCh) diff --git a/internal/server/receiver_server.go b/internal/server/receiver_server.go index 00f6a70f6..b0ef23c14 100644 --- a/internal/server/receiver_server.go +++ b/internal/server/receiver_server.go @@ -32,17 +32,19 @@ import ( // ReceiverServer handles webhook POST requests type ReceiverServer struct { - port string - logger logr.Logger - kubeClient client.Client + port string + logger logr.Logger + kubeClient client.Client + detailedMetrics bool } // NewReceiverServer returns an HTTP server that handles webhooks -func NewReceiverServer(port string, logger logr.Logger, kubeClient client.Client) *ReceiverServer { +func NewReceiverServer(port string, logger logr.Logger, kubeClient client.Client, detailedMetrics bool) *ReceiverServer { return &ReceiverServer{ - port: port, - logger: logger.WithName("receiver-server"), - kubeClient: kubeClient, + port: port, + logger: logger.WithName("receiver-server"), + kubeClient: kubeClient, + detailedMetrics: detailedMetrics, } } @@ -50,7 +52,11 @@ func NewReceiverServer(port string, logger logr.Logger, kubeClient client.Client func (s *ReceiverServer) ListenAndServe(stopCh <-chan struct{}, mdlw middleware.Middleware) { mux := http.NewServeMux() mux.Handle(apiv1.ReceiverWebhookPath, http.HandlerFunc(s.handlePayload())) - h := std.Handler("", mdlw, mux) + middlewarePath := "" + if !s.detailedMetrics { + middlewarePath = apiv1.ReceiverWebhookPath + } + h := std.Handler(middlewarePath, mdlw, mux) srv := &http.Server{ Addr: s.port, Handler: h, diff --git a/main.go b/main.go index 5530ea7ba..1e77b60da 100644 --- a/main.go +++ b/main.go @@ -87,6 +87,7 @@ func main() { aclOptions acl.Options rateLimiterOptions helper.RateLimiterOptions featureGates feathelper.FeatureGates + detailedMetrics bool ) flag.StringVar(&metricsAddr, "metrics-addr", ":8080", "The address the metric endpoint binds to.") @@ -97,6 +98,7 @@ func main() { flag.BoolVar(&watchAllNamespaces, "watch-all-namespaces", true, "Watch for custom resources in all namespaces, if set to false it will only watch the runtime namespace.") flag.DurationVar(&rateLimitInterval, "rate-limit-interval", 5*time.Minute, "Interval in which rate limit has effect.") + flag.BoolVar(&detailedMetrics, "detailed-metrics", true, "Count metrics for every requested path") clientOptions.BindFlags(flag.CommandLine) logOptions.BindFlags(flag.CommandLine) @@ -220,11 +222,11 @@ func main() { Registry: crtlmetrics.Registry, }), }) - eventServer := server.NewEventServer(eventsAddr, ctrl.Log, mgr.GetClient(), mgr.GetEventRecorderFor(controllerName), aclOptions.NoCrossNamespaceRefs) + eventServer := server.NewEventServer(eventsAddr, ctrl.Log, mgr.GetClient(), mgr.GetEventRecorderFor(controllerName), aclOptions.NoCrossNamespaceRefs, detailedMetrics) go eventServer.ListenAndServe(ctx.Done(), eventMdlw, store) setupLog.Info("starting webhook receiver server", "addr", receiverAddr) - receiverServer := server.NewReceiverServer(receiverAddr, ctrl.Log, mgr.GetClient()) + receiverServer := server.NewReceiverServer(receiverAddr, ctrl.Log, mgr.GetClient(), detailedMetrics) receiverMdlw := middleware.New(middleware.Config{ Recorder: prommetrics.NewRecorder(prommetrics.Config{ Prefix: "gotk_receiver", From c85b1eb391baea4bf6c8c5ee48335a69db5ff210 Mon Sep 17 00:00:00 2001 From: Alexey Kuzbekov Date: Thu, 30 May 2024 14:00:35 +0100 Subject: [PATCH 2/2] Change default behavior and naming Signed-off-by: Alexey Kuzbekov --- internal/server/event_server.go | 32 +++++++++++++++--------------- internal/server/receiver_server.go | 26 ++++++++++++------------ main.go | 8 ++++---- 3 files changed, 33 insertions(+), 33 deletions(-) diff --git a/internal/server/event_server.go b/internal/server/event_server.go index 8cb75775b..9aa5eb110 100644 --- a/internal/server/event_server.go +++ b/internal/server/event_server.go @@ -48,23 +48,23 @@ type eventContextKey struct{} // EventServer handles event POST requests type EventServer struct { - port string - logger logr.Logger - kubeClient client.Client - noCrossNamespaceRefs bool - detailedMetrics bool + port string + logger logr.Logger + kubeClient client.Client + noCrossNamespaceRefs bool + exportHTTPPathMetrics bool kuberecorder.EventRecorder } // NewEventServer returns an HTTP server that handles events -func NewEventServer(port string, logger logr.Logger, kubeClient client.Client, eventRecorder kuberecorder.EventRecorder, noCrossNamespaceRefs bool, detailedMetrics bool) *EventServer { +func NewEventServer(port string, logger logr.Logger, kubeClient client.Client, eventRecorder kuberecorder.EventRecorder, noCrossNamespaceRefs bool, exportHTTPPathMetrics bool) *EventServer { return &EventServer{ - port: port, - logger: logger.WithName("event-server"), - kubeClient: kubeClient, - EventRecorder: eventRecorder, - noCrossNamespaceRefs: noCrossNamespaceRefs, - detailedMetrics: detailedMetrics, + port: port, + logger: logger.WithName("event-server"), + kubeClient: kubeClient, + EventRecorder: eventRecorder, + noCrossNamespaceRefs: noCrossNamespaceRefs, + exportHTTPPathMetrics: exportHTTPPathMetrics, } } @@ -86,11 +86,11 @@ func (s *EventServer) ListenAndServe(stopCh <-chan struct{}, mdlw middleware.Mid mux := http.NewServeMux() path := "/" mux.Handle(path, handler) - middlewarePath := "" - if !s.detailedMetrics { - middlewarePath = path + handlerID := path + if s.exportHTTPPathMetrics { + handlerID = "" } - h := std.Handler(middlewarePath, mdlw, mux) + h := std.Handler(handlerID, mdlw, mux) srv := &http.Server{ Addr: s.port, Handler: h, diff --git a/internal/server/receiver_server.go b/internal/server/receiver_server.go index b0ef23c14..2eb9c5760 100644 --- a/internal/server/receiver_server.go +++ b/internal/server/receiver_server.go @@ -32,19 +32,19 @@ import ( // ReceiverServer handles webhook POST requests type ReceiverServer struct { - port string - logger logr.Logger - kubeClient client.Client - detailedMetrics bool + port string + logger logr.Logger + kubeClient client.Client + exportHTTPPathMetrics bool } // NewReceiverServer returns an HTTP server that handles webhooks -func NewReceiverServer(port string, logger logr.Logger, kubeClient client.Client, detailedMetrics bool) *ReceiverServer { +func NewReceiverServer(port string, logger logr.Logger, kubeClient client.Client, exportHTTPPathMetrics bool) *ReceiverServer { return &ReceiverServer{ - port: port, - logger: logger.WithName("receiver-server"), - kubeClient: kubeClient, - detailedMetrics: detailedMetrics, + port: port, + logger: logger.WithName("receiver-server"), + kubeClient: kubeClient, + exportHTTPPathMetrics: exportHTTPPathMetrics, } } @@ -52,11 +52,11 @@ func NewReceiverServer(port string, logger logr.Logger, kubeClient client.Client func (s *ReceiverServer) ListenAndServe(stopCh <-chan struct{}, mdlw middleware.Middleware) { mux := http.NewServeMux() mux.Handle(apiv1.ReceiverWebhookPath, http.HandlerFunc(s.handlePayload())) - middlewarePath := "" - if !s.detailedMetrics { - middlewarePath = apiv1.ReceiverWebhookPath + handlerID := apiv1.ReceiverWebhookPath + if s.exportHTTPPathMetrics { + handlerID = "" } - h := std.Handler(middlewarePath, mdlw, mux) + h := std.Handler(handlerID, mdlw, mux) srv := &http.Server{ Addr: s.port, Handler: h, diff --git a/main.go b/main.go index 1e77b60da..4cf29bcc3 100644 --- a/main.go +++ b/main.go @@ -87,7 +87,7 @@ func main() { aclOptions acl.Options rateLimiterOptions helper.RateLimiterOptions featureGates feathelper.FeatureGates - detailedMetrics bool + exportHTTPPathMetrics bool ) flag.StringVar(&metricsAddr, "metrics-addr", ":8080", "The address the metric endpoint binds to.") @@ -98,7 +98,7 @@ func main() { flag.BoolVar(&watchAllNamespaces, "watch-all-namespaces", true, "Watch for custom resources in all namespaces, if set to false it will only watch the runtime namespace.") flag.DurationVar(&rateLimitInterval, "rate-limit-interval", 5*time.Minute, "Interval in which rate limit has effect.") - flag.BoolVar(&detailedMetrics, "detailed-metrics", true, "Count metrics for every requested path") + flag.BoolVar(&exportHTTPPathMetrics, "export-http-path-metrics", false, "When enabled, the requests full path is included in the HTTP server metrics (risk as high cardinality") clientOptions.BindFlags(flag.CommandLine) logOptions.BindFlags(flag.CommandLine) @@ -222,11 +222,11 @@ func main() { Registry: crtlmetrics.Registry, }), }) - eventServer := server.NewEventServer(eventsAddr, ctrl.Log, mgr.GetClient(), mgr.GetEventRecorderFor(controllerName), aclOptions.NoCrossNamespaceRefs, detailedMetrics) + eventServer := server.NewEventServer(eventsAddr, ctrl.Log, mgr.GetClient(), mgr.GetEventRecorderFor(controllerName), aclOptions.NoCrossNamespaceRefs, exportHTTPPathMetrics) go eventServer.ListenAndServe(ctx.Done(), eventMdlw, store) setupLog.Info("starting webhook receiver server", "addr", receiverAddr) - receiverServer := server.NewReceiverServer(receiverAddr, ctrl.Log, mgr.GetClient(), detailedMetrics) + receiverServer := server.NewReceiverServer(receiverAddr, ctrl.Log, mgr.GetClient(), exportHTTPPathMetrics) receiverMdlw := middleware.New(middleware.Config{ Recorder: prommetrics.NewRecorder(prommetrics.Config{ Prefix: "gotk_receiver",