feat: add counter for admissions

Signed-off-by: David Weber <[email protected]>

CHANGELOG.md

@@ -159,6 +159,9 @@ Adding a new version? You'll need three changes:
   [#5965](https://github.com/Kong/kubernetes-ingress-controller/pull/5965)
 - Fallback configuration no longer omits licenses and vaults.
   [#6048](https://github.com/Kong/kubernetes-ingress-controller/pull/6048)
+- Added new metric for Prometheus called `ingress_controller_admission_count`. It's a counter and has two labels
+  `allowed` to indicate if the resource was allowed and `resource` to indicate the resource under admission.
+  [#6084](https://github.com/Kong/kubernetes-ingress-controller/issues/6084)
 
 ### Fixed
 

internal/admission/handler.go

@@ -4,13 +4,13 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"net/http"
-
 	"github.com/go-logr/logr"
+	"github.com/kong/kubernetes-ingress-controller/v3/internal/metrics"
 	admissionv1 "k8s.io/api/admission/v1"
 	corev1 "k8s.io/api/core/v1"
 	netv1 "k8s.io/api/networking/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"net/http"
 
 	"github.com/kong/kubernetes-ingress-controller/v3/internal/annotations"
 	ctrlref "github.com/kong/kubernetes-ingress-controller/v3/internal/controllers/reference"
@@ -37,6 +37,8 @@ type RequestHandler struct {
 	// referring the validated resource (Secret) to check the changes on
 	// referred Secret will produce invalid configuration of the plugins.
 	ReferenceIndexers ctrlref.CacheIndexers
+	// PromMetrics provides the Prometheus registry to record metrics
+	PromMetrics *metrics.CtrlFuncMetrics
 
 	Logger logr.Logger
 }
@@ -63,6 +65,14 @@ func (h RequestHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		http.Error(w, err.Error(), http.StatusInternalServerError)
 		return
 	}
+
+	h.PromMetrics.RecordAdmissionCount(
+		response.Allowed,
+		fmt.Sprintf(
+			"%s.%s/%s",
+			review.Request.Resource.Resource, review.Request.Resource.Group, review.Request.Resource.Version,
+		),
+	)
 	review.Response = response
 
 	if err := json.NewEncoder(w).Encode(&review); err != nil {

internal/dataplane/kong_client.go

@@ -169,12 +169,13 @@ func NewKongClient(
 	kongConfigBuilder KongConfigBuilder,
 	cacheStores store.CacheStores,
 	fallbackConfigGenerator FallbackConfigGenerator,
+	prometheusMetrics *metrics.CtrlFuncMetrics,
 ) (*KongClient, error) {
 	c := &KongClient{
 		logger:                  logger,
 		requestTimeout:          timeout,
 		diagnostic:              diagnostic,
-		prometheusMetrics:       metrics.NewCtrlFuncMetrics(),
+		prometheusMetrics:       prometheusMetrics,
 		cache:                   &cacheStores,
 		kongConfig:              kongConfig,
 		eventRecorder:           eventRecorder,

internal/dataplane/kong_client_test.go

@@ -733,6 +733,7 @@ func setupTestKongClient(
 		configBuilder,
 		store.NewCacheStores(),
 		newMockFallbackConfigGenerator(),
+		metrics.NewCtrlFuncMetrics(),
 	)
 	require.NoError(t, err)
 	return kongClient
@@ -1002,6 +1003,7 @@ func TestKongClient_FallbackConfiguration_SuccessfulRecovery(t *testing.T) {
 		configBuilder,
 		originalCache,
 		fallbackConfigGenerator,
+		metrics.NewCtrlFuncMetrics(),
 	)
 	require.NoError(t, err)
 
@@ -1114,6 +1116,7 @@ func TestKongClient_FallbackConfiguration_FailedRecovery(t *testing.T) {
 		configBuilder,
 		originalCache,
 		fallbackConfigGenerator,
+		metrics.NewCtrlFuncMetrics(),
 	)
 	require.NoError(t, err)
 

internal/manager/run.go

@@ -5,6 +5,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"github.com/kong/kubernetes-ingress-controller/v3/internal/metrics"
 	"net/http"
 	"net/url"
 	"os"
@@ -185,7 +186,8 @@ func Run(
 	}
 
 	setupLog.Info("Starting Admission Server")
-	if err := setupAdmissionServer(ctx, c, clientsManager, referenceIndexers, mgr.GetClient(), logger, translatorFeatureFlags, storer); err != nil {
+	promMetrics := metrics.NewCtrlFuncMetrics()
+	if err := setupAdmissionServer(ctx, c, clientsManager, referenceIndexers, mgr.GetClient(), logger, translatorFeatureFlags, storer, promMetrics); err != nil {
 		return err
 	}
 
@@ -207,6 +209,7 @@ func Run(
 		configTranslator,
 		cache,
 		fallbackConfigGenerator,
+		promMetrics,
 	)
 	if err != nil {
 		return fmt.Errorf("failed to initialize kong data-plane client: %w", err)

internal/manager/setup.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"github.com/kong/kubernetes-ingress-controller/v3/internal/metrics"
 	"io"
 	"time"
 
@@ -195,6 +196,7 @@ func setupAdmissionServer(
 	logger logr.Logger,
 	translatorFeatures translator.FeatureFlags,
 	storer store.Storer,
+	promMetrics *metrics.CtrlFuncMetrics,
 ) error {
 	admissionLogger := logger.WithName("admission-server")
 
@@ -214,6 +216,7 @@ func setupAdmissionServer(
 			storer,
 		),
 		ReferenceIndexers: referenceIndexers,
+		PromMetrics:       promMetrics,
 		Logger:            admissionLogger,
 	}, admissionLogger)
 	if err != nil {

internal/metrics/prometheus.go

@@ -4,6 +4,7 @@ import (
 	"errors"
 	"fmt"
 	"net"
+	"strconv"
 	"sync"
 	"time"
 
@@ -27,6 +28,8 @@ type CtrlFuncMetrics struct {
 	ConfigPushDuration *prometheus.HistogramVec
 
 	ConfigPushSuccessTime *prometheus.GaugeVec
+
+	AdmissionCount *prometheus.CounterVec
 }
 
 const (
@@ -70,13 +73,24 @@ const (
 	DataplaneKey string = "dataplane"
 )
 
+const (
+	// AllowedKey defines the key of the metric label indicating admission was allowed.
+	AllowedKey string = "allowed"
+)
+
+const (
+	// AdmissionResourceKey defines the name of the metric label indicating which dataplane this time series is relevant for.
+	AdmissionResourceKey string = "resource"
+)
+
 const (
 	MetricNameConfigPushCount            = "ingress_controller_configuration_push_count"
 	MetricNameConfigPushBrokenResources  = "ingress_controller_configuration_push_broken_resource_count"
 	MetricNameConfigPushSuccessTime      = "ingress_controller_configuration_push_last_successful"
 	MetricNameTranslationCount           = "ingress_controller_translation_count"
 	MetricNameTranslationBrokenResources = "ingress_controller_translation_broken_resource_count"
 	MetricNameConfigPushDuration         = "ingress_controller_configuration_push_duration_milliseconds"
+	MetricNameAdmissionCount             = "ingress_controller_admission_count"
 )
 
 var _lock sync.Mutex
@@ -168,12 +182,27 @@ func NewCtrlFuncMetrics() *CtrlFuncMetrics {
 		[]string{DataplaneKey},
 	)
 
+	controllerMetrics.AdmissionCount = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: MetricNameAdmissionCount,
+			Help: fmt.Sprintf(
+				"Count of admissions processed by Kong. "+
+					"`%s` describes whether an admission was allowed. "+
+					"`%s` describes the resource under admission. ",
+				AllowedKey,
+				AdmissionResourceKey,
+			),
+		},
+		[]string{AllowedKey, AdmissionResourceKey},
+	)
+
 	metrics.Registry.Unregister(controllerMetrics.ConfigPushCount)
 	metrics.Registry.Unregister(controllerMetrics.ConfigPushBrokenResources)
 	metrics.Registry.Unregister(controllerMetrics.TranslationCount)
 	metrics.Registry.Unregister(controllerMetrics.TranslationBrokenResources)
 	metrics.Registry.Unregister(controllerMetrics.ConfigPushDuration)
 	metrics.Registry.Unregister(controllerMetrics.ConfigPushSuccessTime)
+	metrics.Registry.Unregister(controllerMetrics.AdmissionCount)
 
 	metrics.Registry.MustRegister(
 		controllerMetrics.ConfigPushCount,
@@ -182,6 +211,7 @@ func NewCtrlFuncMetrics() *CtrlFuncMetrics {
 		controllerMetrics.TranslationBrokenResources,
 		controllerMetrics.ConfigPushDuration,
 		controllerMetrics.ConfigPushSuccessTime,
+		controllerMetrics.AdmissionCount,
 	)
 
 	return controllerMetrics
@@ -223,6 +253,13 @@ func (c *CtrlFuncMetrics) RecordTranslationBrokenResources(count int) {
 	c.TranslationBrokenResources.Set(float64(count))
 }
 
+func (c *CtrlFuncMetrics) RecordAdmissionCount(allowed bool, resource string) {
+	c.ConfigPushCount.With(prometheus.Labels{
+		AllowedKey:           strconv.FormatBool(allowed),
+		AdmissionResourceKey: resource,
+	}).Inc()
+}
+
 type recordOption func(prometheus.Labels) prometheus.Labels
 
 func withError(err error) recordOption {

test/envtest/metrics_envtest_test.go

@@ -42,6 +42,7 @@ func TestMetricsAreServed(t *testing.T) {
 		metrics.MetricNameTranslationBrokenResources,
 		metrics.MetricNameConfigPushDuration,
 		metrics.MetricNameConfigPushSuccessTime,
+		metrics.MetricNameAdmissionCount,
 	}
 
 	metricsURL := fmt.Sprintf("http://%s/metrics", cfg.MetricsAddr)