feat: add counter for admissions

Signed-off-by: David Weber <[email protected]>
Signed-off-by: David Weber <[email protected]>

CHANGELOG.md

@@ -202,6 +202,9 @@ Adding a new version? You'll need three changes:
   [#6048](https://github.com/Kong/kubernetes-ingress-controller/pull/6048)
 - Add support for Gateway API GRPCRoute and pass related Gateway API conformance test.
   [#5776](https://github.com/Kong/kubernetes-ingress-controller/pull/5776)
+- Added new metric for Prometheus called `ingress_controller_admission_count`. It's a counter and has two labels
+  `allowed` to indicate if the resource was allowed and `resource` to indicate the resource under admission.
+  [#6084](https://github.com/Kong/kubernetes-ingress-controller/issues/6084)
 
 ### Fixed
 

internal/admission/handler.go

@@ -16,6 +16,7 @@ import (
 	ctrlref "github.com/kong/kubernetes-ingress-controller/v3/internal/controllers/reference"
 	"github.com/kong/kubernetes-ingress-controller/v3/internal/gatewayapi"
 	"github.com/kong/kubernetes-ingress-controller/v3/internal/labels"
+	"github.com/kong/kubernetes-ingress-controller/v3/internal/metrics"
 	"github.com/kong/kubernetes-ingress-controller/v3/internal/util"
 	kongv1 "github.com/kong/kubernetes-ingress-controller/v3/pkg/apis/configuration/v1"
 	kongv1alpha1 "github.com/kong/kubernetes-ingress-controller/v3/pkg/apis/configuration/v1alpha1"
@@ -37,6 +38,8 @@ type RequestHandler struct {
 	// referring the validated resource (Secret) to check the changes on
 	// referred Secret will produce invalid configuration of the plugins.
 	ReferenceIndexers ctrlref.CacheIndexers
+	// PromMetrics provides the Prometheus registry to record metrics
+	PromMetrics *metrics.CtrlFuncMetrics
 
 	Logger logr.Logger
 }
@@ -63,6 +66,14 @@ func (h RequestHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		http.Error(w, err.Error(), http.StatusInternalServerError)
 		return
 	}
+
+	h.PromMetrics.RecordAdmissionCount(
+		response.Allowed,
+		fmt.Sprintf(
+			"%s.%s/%s",
+			review.Request.Resource.Resource, review.Request.Resource.Group, review.Request.Resource.Version,
+		),
+	)
 	review.Response = response
 
 	if err := json.NewEncoder(w).Encode(&review); err != nil {

internal/dataplane/kong_client.go

@@ -197,12 +197,13 @@ func NewKongClient(
 	kongConfigBuilder KongConfigBuilder,
 	cacheStores store.CacheStores,
 	fallbackConfigGenerator FallbackConfigGenerator,
+	prometheusMetrics *metrics.CtrlFuncMetrics,
 ) (*KongClient, error) {
 	c := &KongClient{
 		logger:                  logger,
 		requestTimeout:          timeout,
 		diagnostic:              diagnostic,
-		prometheusMetrics:       metrics.NewCtrlFuncMetrics(),
+		prometheusMetrics:       prometheusMetrics,
 		cache:                   &cacheStores,
 		kongConfig:              kongConfig,
 		eventRecorder:           eventRecorder,

internal/dataplane/kong_client_test.go

@@ -939,6 +939,7 @@ func setupTestKongClient(
 		configBuilder,
 		store.NewCacheStores(),
 		newMockFallbackConfigGenerator(),
+		metrics.NewCtrlFuncMetrics(),
 	)
 	require.NoError(t, err)
 	return kongClient
@@ -1386,6 +1387,7 @@ func TestKongClient_FallbackConfiguration_SkipMakingRedundantSnapshot(t *testing
 		configBuilder,
 		originalCache,
 		fallbackConfigGenerator,
+		metrics.NewCtrlFuncMetrics(),
 	)
 	require.NoError(t, err)
 
@@ -1451,6 +1453,7 @@ func TestKongClient_FallbackConfiguration_FailedRecovery(t *testing.T) {
 		configBuilder,
 		originalCache,
 		fallbackConfigGenerator,
+		metrics.NewCtrlFuncMetrics(),
 	)
 	require.NoError(t, err)
 

internal/manager/run.go

@@ -37,6 +37,7 @@ import (
 	"github.com/kong/kubernetes-ingress-controller/v3/internal/manager/metadata"
 	"github.com/kong/kubernetes-ingress-controller/v3/internal/manager/telemetry"
 	"github.com/kong/kubernetes-ingress-controller/v3/internal/manager/utils/kongconfig"
+	"github.com/kong/kubernetes-ingress-controller/v3/internal/metrics"
 	"github.com/kong/kubernetes-ingress-controller/v3/internal/store"
 	"github.com/kong/kubernetes-ingress-controller/v3/internal/util"
 	"github.com/kong/kubernetes-ingress-controller/v3/internal/util/kubernetes/object/status"
@@ -185,7 +186,8 @@ func Run(
 	}
 
 	setupLog.Info("Starting Admission Server")
-	if err := setupAdmissionServer(ctx, c, clientsManager, referenceIndexers, mgr.GetClient(), logger, translatorFeatureFlags, storer); err != nil {
+	promMetrics := metrics.NewCtrlFuncMetrics()
+	if err := setupAdmissionServer(ctx, c, clientsManager, referenceIndexers, mgr.GetClient(), logger, translatorFeatureFlags, storer, promMetrics); err != nil {
 		return err
 	}
 
@@ -207,6 +209,7 @@ func Run(
 		configTranslator,
 		cache,
 		fallbackConfigGenerator,
+		promMetrics,
 	)
 	if err != nil {
 		return fmt.Errorf("failed to initialize kong data-plane client: %w", err)

internal/manager/setup.go

@@ -34,6 +34,7 @@ import (
 	konnectLicense "github.com/kong/kubernetes-ingress-controller/v3/internal/konnect/license"
 	"github.com/kong/kubernetes-ingress-controller/v3/internal/license"
 	"github.com/kong/kubernetes-ingress-controller/v3/internal/manager/scheme"
+	"github.com/kong/kubernetes-ingress-controller/v3/internal/metrics"
 	"github.com/kong/kubernetes-ingress-controller/v3/internal/store"
 	"github.com/kong/kubernetes-ingress-controller/v3/internal/util"
 	"github.com/kong/kubernetes-ingress-controller/v3/internal/util/kubernetes/object/status"
@@ -195,6 +196,7 @@ func setupAdmissionServer(
 	logger logr.Logger,
 	translatorFeatures translator.FeatureFlags,
 	storer store.Storer,
+	promMetrics *metrics.CtrlFuncMetrics,
 ) error {
 	admissionLogger := logger.WithName("admission-server")
 
@@ -214,6 +216,7 @@ func setupAdmissionServer(
 			storer,
 		),
 		ReferenceIndexers: referenceIndexers,
+		PromMetrics:       promMetrics,
 		Logger:            admissionLogger,
 	}, admissionLogger)
 	if err != nil {

internal/metrics/prometheus.go

@@ -4,6 +4,7 @@ import (
 	"errors"
 	"fmt"
 	"net"
+	"strconv"
 	"sync"
 	"time"
 
@@ -23,6 +24,7 @@ type CtrlFuncMetrics struct {
 	TranslationBrokenResources prometheus.Gauge
 	ConfigPushDuration         *prometheus.HistogramVec
 	ConfigPushSuccessTime      *prometheus.GaugeVec
+	AdmissionCount             *prometheus.CounterVec
 
 	// Fallback config push metrics.
 	FallbackTranslationCount           *prometheus.CounterVec
@@ -77,6 +79,16 @@ const (
 	DataplaneKey string = "dataplane"
 )
 
+const (
+	// AllowedKey defines the key of the metric label indicating admission was allowed.
+	AllowedKey string = "allowed"
+)
+
+const (
+	// AdmissionResourceKey defines the name of the metric label indicating which dataplane this time series is relevant for.
+	AdmissionResourceKey string = "resource"
+)
+
 // Regular config push metrics names.
 const (
 	MetricNameConfigPushCount            = "ingress_controller_configuration_push_count"
@@ -85,6 +97,7 @@ const (
 	MetricNameTranslationCount           = "ingress_controller_translation_count"
 	MetricNameTranslationBrokenResources = "ingress_controller_translation_broken_resource_count"
 	MetricNameConfigPushDuration         = "ingress_controller_configuration_push_duration_milliseconds"
+	MetricNameAdmissionCount             = "ingress_controller_admission_count"
 )
 
 // Fallback config push metrics names.
@@ -189,6 +202,20 @@ func NewCtrlFuncMetrics() *CtrlFuncMetrics {
 		[]string{DataplaneKey},
 	)
 
+	controllerMetrics.AdmissionCount = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: MetricNameAdmissionCount,
+			Help: fmt.Sprintf(
+				"Count of admissions processed by Kong. "+
+					"`%s` describes whether an admission was allowed. "+
+					"`%s` describes the resource under admission. ",
+				AllowedKey,
+				AdmissionResourceKey,
+			),
+		},
+		[]string{AllowedKey, AdmissionResourceKey},
+	)
+
 	controllerMetrics.FallbackTranslationCount = prometheus.NewCounterVec(
 		prometheus.CounterOpts{
 			Name: MetricNameFallbackTranslationCount,
@@ -302,6 +329,7 @@ func NewCtrlFuncMetrics() *CtrlFuncMetrics {
 		controllerMetrics.TranslationBrokenResources,
 		controllerMetrics.ConfigPushDuration,
 		controllerMetrics.ConfigPushSuccessTime,
+		controllerMetrics.AdmissionCount,
 		controllerMetrics.FallbackTranslationBrokenResources,
 		controllerMetrics.FallbackTranslationCount,
 		controllerMetrics.FallbackConfigPushCount,
@@ -356,6 +384,13 @@ func (c *CtrlFuncMetrics) RecordTranslationBrokenResources(count int) {
 	c.TranslationBrokenResources.Set(float64(count))
 }
 
+func (c *CtrlFuncMetrics) RecordAdmissionCount(allowed bool, resource string) {
+	c.ConfigPushCount.With(prometheus.Labels{
+		AllowedKey:           strconv.FormatBool(allowed),
+		AdmissionResourceKey: resource,
+	}).Inc()
+}
+
 // RecordFallbackTranslationFailure records a failed fallback configuration translation.
 func (c *CtrlFuncMetrics) RecordFallbackTranslationFailure() {
 	c.FallbackTranslationCount.With(prometheus.Labels{

test/envtest/metrics_envtest_test.go

@@ -70,6 +70,7 @@ func TestMetricsAreServed(t *testing.T) {
 				metrics.MetricNameTranslationBrokenResources,
 				metrics.MetricNameConfigPushDuration,
 				metrics.MetricNameConfigPushSuccessTime,
+				metrics.MetricNameAdmissionCount,
 			},
 		},
 	}