feat: add agent injector telemetry

agent-inject/handler.go

@@ -9,6 +9,7 @@ import (
 	"io"
 	"net/http"
 	"strings"
+	"time"
 
 	"github.com/hashicorp/go-hclog"
 	"github.com/hashicorp/vault-k8s/agent-inject/agent"
@@ -85,6 +86,14 @@ type Handler struct {
 func (h *Handler) Handle(w http.ResponseWriter, r *http.Request) {
 	h.Log.Info("Request received", "Method", r.Method, "URL", r.URL)
 
+	// Measure request processing duration and monitor request queue
+	requestQueue.Inc()
+	requestStart := time.Now()
+	defer func() {
+		requestProcessingTime.Observe(float64(time.Since(requestStart).Milliseconds()))
+		requestQueue.Dec()
+	}()
+
 	if ct := r.Header.Get("Content-Type"); ct != "application/json" {
 		msg := fmt.Sprintf("Invalid content-type: %q", ct)
 		http.Error(w, msg, http.StatusBadRequest)
@@ -109,7 +118,10 @@ func (h *Handler) Handle(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	var admResp admissionv1.AdmissionReview
+	var (
+		mutateResp MutateResponse
+		admResp    admissionv1.AdmissionReview
+	)
 
 	// Both v1 and v1beta1 AdmissionReview types are exactly the same, so the v1beta1 type can
 	// be decoded into the v1 type. However the runtime codec's decoder guesses which type to
@@ -126,7 +138,8 @@ func (h *Handler) Handle(w http.ResponseWriter, r *http.Request) {
 		h.Log.Error("error on request", "Error", msg, "Code", http.StatusInternalServerError)
 		return
 	} else {
-		admResp.Response = h.Mutate(admReq.Request)
+		mutateResp = h.Mutate(admReq.Request)
+		admResp.Response = mutateResp.Resp
 	}
 
 	// Default to a v1 AdmissionReview, otherwise the API server may not recognize the request
@@ -142,26 +155,43 @@ func (h *Handler) Handle(w http.ResponseWriter, r *http.Request) {
 		msg := fmt.Sprintf("error marshalling admission response: %s", err)
 		http.Error(w, msg, http.StatusInternalServerError)
 		h.Log.Error("error on request", "Error", msg, "Code", http.StatusInternalServerError)
+		incrementInjectionFailures(admReq.Request.Namespace)
 		return
 	}
 
 	if _, err := w.Write(resp); err != nil {
 		h.Log.Error("error writing response", "Error", err)
+		incrementInjectionFailures(admReq.Request.Namespace)
+		return
 	}
+
+	if admResp.Response.Allowed {
+		incrementInjections(admReq.Request.Namespace, mutateResp)
+	} else {
+		incrementInjectionFailures(admReq.Request.Namespace)
+	}
+}
+
+type MutateResponse struct {
+	Resp            *admissionv1.AdmissionResponse
+	InjectedInit    bool
+	InjectedSidecar bool
 }
 
 // Mutate takes an admission request and performs mutation if necessary,
 // returning the final API response.
-func (h *Handler) Mutate(req *admissionv1.AdmissionRequest) *admissionv1.AdmissionResponse {
+func (h *Handler) Mutate(req *admissionv1.AdmissionRequest) MutateResponse {
 	// Decode the pod from the request
 	var pod corev1.Pod
 	if err := json.Unmarshal(req.Object.Raw, &pod); err != nil {
 		h.Log.Error("could not unmarshal request to pod: %s", err)
 		h.Log.Debug("%s", req.Object.Raw)
-		return &admissionv1.AdmissionResponse{
-			UID: req.UID,
-			Result: &metav1.Status{
-				Message: err.Error(),
+		return MutateResponse{
+			Resp: &admissionv1.AdmissionResponse{
+				UID: req.UID,
+				Result: &metav1.Status{
+					Message: err.Error(),
+				},
 			},
 		}
 	}
@@ -178,7 +208,9 @@ func (h *Handler) Mutate(req *admissionv1.AdmissionRequest) *admissionv1.Admissi
 		err := fmt.Errorf("error checking if should inject agent: %s", err)
 		return admissionError(req.UID, err)
 	} else if !inject {
-		return resp
+		return MutateResponse{
+			Resp: resp,
+		}
 	}
 
 	h.Log.Debug("checking namespaces..")
@@ -249,14 +281,20 @@ func (h *Handler) Mutate(req *admissionv1.AdmissionRequest) *admissionv1.Admissi
 	patchType := admissionv1.PatchTypeJSONPatch
 	resp.PatchType = &patchType
 
-	return resp
+	return MutateResponse{
+		Resp:            resp,
+		InjectedInit:    agentSidecar.PrePopulate,
+		InjectedSidecar: !agentSidecar.PrePopulateOnly,
+	}
 }
 
-func admissionError(UID types.UID, err error) *admissionv1.AdmissionResponse {
-	return &admissionv1.AdmissionResponse{
-		UID: UID,
-		Result: &metav1.Status{
-			Message: err.Error(),
+func admissionError(UID types.UID, err error) MutateResponse {
+	return MutateResponse{
+		Resp: &admissionv1.AdmissionResponse{
+			UID: UID,
+			Result: &metav1.Status{
+				Message: err.Error(),
+			},
 		},
 	}
 }

agent-inject/handler_test.go

@@ -422,7 +422,7 @@ func TestHandlerHandle(t *testing.T) {
 	for _, tt := range cases {
 		t.Run(tt.Name, func(t *testing.T) {
 			req := require.New(t)
-			resp := tt.Handler.Mutate(&tt.Req)
+			resp := (tt.Handler.Mutate(&tt.Req)).Resp
 			if (tt.Err == "") != resp.Allowed {
 				t.Fatalf("allowed: %v, expected err: %v", resp.Allowed, tt.Err)
 			}

agent-inject/metrics.go

@@ -0,0 +1,76 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: MPL-2.0
+
+package agent_inject
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+const (
+	metricsNamespace            = "vault"
+	metricsSubsystem            = "agent_injector"
+	metricsLabelNamespace       = "namespace"
+	metricsLabelType            = "injection_type"
+	metricsLabelTypeBoth        = "init_and_sidecar"
+	metricsLabelTypeInitOnly    = "init_only"
+	metricsLabelTypeSidecarOnly = "sidecar_only"
+)
+
+var (
+	requestQueue = prometheus.NewGauge(prometheus.GaugeOpts{
+		Namespace: metricsNamespace,
+		Subsystem: metricsSubsystem,
+		Name:      "request_queue_length",
+		Help:      "Count of webhook requests in the injector's queue",
+	})
+
+	requestProcessingTime = prometheus.NewHistogram(prometheus.HistogramOpts{
+		Namespace: metricsNamespace,
+		Subsystem: metricsSubsystem,
+		Name:      "request_processing_duration_ms",
+		Buckets:   []float64{5, 10, 25, 50, 75, 100, 250, 500, 1000, 2500, 5000, 7500, 10000},
+	})
+
+	injectionsByNamespace = prometheus.NewCounterVec(prometheus.CounterOpts{
+		Namespace: metricsNamespace,
+		Subsystem: metricsSubsystem,
+		Name:      "injections_by_namespace_total",
+		Help:      "Total count of Agent Sidecar injections by namespace",
+	}, []string{metricsLabelNamespace, metricsLabelType})
+
+	failedInjectionsByNamespace = prometheus.NewCounterVec(prometheus.CounterOpts{
+		Namespace: metricsNamespace,
+		Subsystem: metricsSubsystem,
+		Name:      "failed_injections_by_namespace_total",
+		Help:      "Total count of failed Agent Sidecar injections by namespace",
+	}, []string{metricsLabelNamespace})
+)
+
+func incrementInjections(namespace string, res MutateResponse) {
+	// Injection type can be one of: init_and_sidecar (default); init_only; or sidecar_only
+	typeLabel := metricsLabelTypeBoth
+	if res.InjectedInit && !res.InjectedSidecar {
+		typeLabel = metricsLabelTypeInitOnly
+	} else if res.InjectedSidecar && !res.InjectedInit {
+		typeLabel = metricsLabelTypeSidecarOnly
+	}
+
+	injectionsByNamespace.With(prometheus.Labels{
+		metricsLabelNamespace: namespace,
+		metricsLabelType:      typeLabel,
+	}).Inc()
+}
+
+func incrementInjectionFailures(namespace string) {
+	failedInjectionsByNamespace.With(prometheus.Labels{metricsLabelNamespace: namespace}).Inc()
+}
+
+func MustRegisterInjectorMetrics(registry prometheus.Registerer) {
+	registry.MustRegister(
+		requestQueue,
+		requestProcessingTime,
+		injectionsByNamespace,
+		failedInjectionsByNamespace,
+	)
+}

subcommand/injector/command.go

@@ -27,6 +27,7 @@ import (
 	"github.com/hashicorp/vault-k8s/leader"
 	"github.com/hashicorp/vault-k8s/version"
 	"github.com/mitchellh/cli"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	adminv1 "k8s.io/api/admissionregistration/v1"
 	adminv1beta "k8s.io/api/admissionregistration/v1beta1"
@@ -231,6 +232,7 @@ func (c *Command) Run(args []string) int {
 
 	// Registering path to expose metrics
 	if c.flagTelemetryPath != "" {
+		agentInject.MustRegisterInjectorMetrics(prometheus.DefaultRegisterer)
 		c.UI.Info(fmt.Sprintf("Registering telemetry path on %q", c.flagTelemetryPath))
 		mux.Handle(c.flagTelemetryPath, promhttp.Handler())
 	}