restore usage metrics module

cmd/kas/kasapp/configured_app.go

@@ -51,6 +51,8 @@ import (
 	observability_server "gitlab.com/gitlab-org/cluster-integration/gitlab-agent/v16/internal/module/observability/server"
 	reverse_tunnel_server "gitlab.com/gitlab-org/cluster-integration/gitlab-agent/v16/internal/module/reverse_tunnel/server"
 	"gitlab.com/gitlab-org/cluster-integration/gitlab-agent/v16/internal/module/reverse_tunnel/tunnel"
+	"gitlab.com/gitlab-org/cluster-integration/gitlab-agent/v16/internal/module/usage_metrics"
+	usage_metrics_server "gitlab.com/gitlab-org/cluster-integration/gitlab-agent/v16/internal/module/usage_metrics/server"
 	"gitlab.com/gitlab-org/cluster-integration/gitlab-agent/v16/internal/tool/cache"
 	"gitlab.com/gitlab-org/cluster-integration/gitlab-agent/v16/internal/tool/errz"
 	"gitlab.com/gitlab-org/cluster-integration/gitlab-agent/v16/internal/tool/grpctool"
@@ -215,6 +217,9 @@ func (a *ConfiguredApp) Run(ctx context.Context) (retErr error) {
 	// Agent tracker
 	agentTracker := a.constructAgentTracker(errRep, redisClient)
 
+	// Usage tracker
+	usageTracker := usage_metrics.NewUsageTracker()
+
 	// Module factories
 	factories := []modserver.Factory{
 		&observability_server.Factory{
@@ -223,6 +228,9 @@ func (a *ConfiguredApp) Run(ctx context.Context) (retErr error) {
 		&agent_configuration_server.Factory{
 			AgentRegisterer: agentTracker,
 		},
+		&usage_metrics_server.Factory{
+			UsageTracker: usageTracker,
+		},
 		&agent_registrar_server.Factory{
 			AgentRegisterer: agentTracker,
 		},
@@ -245,6 +253,7 @@ func (a *ConfiguredApp) Run(ctx context.Context) (retErr error) {
 			Api:              srvApi,
 			Config:           a.Configuration,
 			Registerer:       reg,
+			UsageTracker:     usageTracker,
 			AgentServer:      agentSrv.server,
 			ApiServer:        apiSrv.server,
 			RegisterAgentApi: kasToAgentRouter.RegisterAgentApi,

cmd/kas/kasapp/defaulting.go

@@ -7,6 +7,7 @@ import (
 	kubernetes_api_server "gitlab.com/gitlab-org/cluster-integration/gitlab-agent/v16/internal/module/kubernetes_api/server"
 	"gitlab.com/gitlab-org/cluster-integration/gitlab-agent/v16/internal/module/modserver"
 	observability_server "gitlab.com/gitlab-org/cluster-integration/gitlab-agent/v16/internal/module/observability/server"
+	usage_metrics_server "gitlab.com/gitlab-org/cluster-integration/gitlab-agent/v16/internal/module/usage_metrics/server"
 	"gitlab.com/gitlab-org/cluster-integration/gitlab-agent/v16/internal/tool/prototool"
 	"gitlab.com/gitlab-org/cluster-integration/gitlab-agent/v16/pkg/kascfg"
 )
@@ -52,6 +53,7 @@ var (
 	defaulters = []modserver.ApplyDefaults{
 		observability_server.ApplyDefaults,
 		agent_configuration_server.ApplyDefaults,
+		usage_metrics_server.ApplyDefaults,
 		kubernetes_api_server.ApplyDefaults,
 	}
 )

internal/gitlab/api/send_usage_ping.go

@@ -0,0 +1,29 @@
+package api
+
+import (
+	"context"
+	"net/http"
+
+	"gitlab.com/gitlab-org/cluster-integration/gitlab-agent/v16/internal/gitlab"
+)
+
+const (
+	UsagePingApiPath = "/api/v4/internal/kubernetes/usage_metrics"
+)
+
+type UsagePingData struct {
+	Counters       map[string]int64   `json:"counters,omitempty"`
+	UniqueCounters map[string][]int64 `json:"unique_counters,omitempty"`
+}
+
+func SendUsagePing(ctx context.Context, client gitlab.ClientInterface, data UsagePingData, opts ...gitlab.DoOption) error {
+	return client.Do(ctx,
+		joinOpts(opts,
+			gitlab.WithMethod(http.MethodPost),
+			gitlab.WithPath(UsagePingApiPath),
+			gitlab.WithJsonRequestBody(data),
+			gitlab.WithResponseHandler(gitlab.NoContentResponseHandler()),
+			gitlab.WithJWT(true),
+		)...,
+	)
+}

internal/module/kubernetes_api/server/factory.go

@@ -20,6 +20,23 @@ import (
 	"k8s.io/apimachinery/pkg/runtime/serializer"
 )
 
+const (
+	k8sApiRequestCountKnownMetric        = "k8s_api_proxy_request"
+	usersCiTunnelInteractionsCountMetric = "agent_users_using_ci_tunnel"
+	// `ci_access` metric names
+	k8sApiProxyRequestsViaCiAccessMetricName             = "k8s_api_proxy_requests_via_ci_access"
+	k8sApiProxyRequestsUniqueUsersViaCiAccessMetricName  = "k8s_api_proxy_requests_unique_users_via_ci_access"
+	k8sApiProxyRequestsUniqueAgentsViaCiAccessMetricName = "k8s_api_proxy_requests_unique_agents_via_ci_access"
+	// `user_access` metric names
+	k8sApiProxyRequestsViaUserAccessMetricName             = "k8s_api_proxy_requests_via_user_access"
+	k8sApiProxyRequestsUniqueUsersViaUserAccessMetricName  = "k8s_api_proxy_requests_unique_users_via_user_access"
+	k8sApiProxyRequestsUniqueAgentsViaUserAccessMetricName = "k8s_api_proxy_requests_unique_agents_via_user_access"
+	// PAT access metric names
+	k8sApiProxyRequestsViaPatAccessMetricName             = "k8s_api_proxy_requests_via_pat_access"
+	k8sApiProxyRequestsUniqueUsersViaPatAccessMetricName  = "k8s_api_proxy_requests_unique_users_via_pat_access"
+	k8sApiProxyRequestsUniqueAgentsViaPatAccessMetricName = "k8s_api_proxy_requests_unique_agents_via_pat_access"
+)
+
 type Factory struct {
 }
 
@@ -93,15 +110,26 @@ func (f *Factory) New(config *modserver.Config) (modserver.Module, error) {
 				tracer,
 				gapi.IsCacheableError,
 			),
-			responseSerializer:  serializer.NewCodecFactory(runtime.NewScheme()),
-			traceProvider:       config.TraceProvider,
-			tracePropagator:     config.TracePropagator,
-			meterProvider:       config.MeterProvider,
-			serverName:          serverName,
-			serverVia:           "gRPC/1.0 " + serverName,
-			urlPathPrefix:       k8sApi.UrlPathPrefix,
-			listenerGracePeriod: listenCfg.ListenGracePeriod.AsDuration(),
-			shutdownGracePeriod: listenCfg.ShutdownGracePeriod.AsDuration(),
+			requestCounter:           config.UsageTracker.RegisterCounter(k8sApiRequestCountKnownMetric),
+			ciTunnelUsersCounter:     config.UsageTracker.RegisterUniqueCounter(usersCiTunnelInteractionsCountMetric),
+			ciAccessRequestCounter:   config.UsageTracker.RegisterCounter(k8sApiProxyRequestsViaCiAccessMetricName),
+			ciAccessUsersCounter:     config.UsageTracker.RegisterUniqueCounter(k8sApiProxyRequestsUniqueUsersViaCiAccessMetricName),
+			ciAccessAgentsCounter:    config.UsageTracker.RegisterUniqueCounter(k8sApiProxyRequestsUniqueAgentsViaCiAccessMetricName),
+			userAccessRequestCounter: config.UsageTracker.RegisterCounter(k8sApiProxyRequestsViaUserAccessMetricName),
+			userAccessUsersCounter:   config.UsageTracker.RegisterUniqueCounter(k8sApiProxyRequestsUniqueUsersViaUserAccessMetricName),
+			userAccessAgentsCounter:  config.UsageTracker.RegisterUniqueCounter(k8sApiProxyRequestsUniqueAgentsViaUserAccessMetricName),
+			patAccessRequestCounter:  config.UsageTracker.RegisterCounter(k8sApiProxyRequestsViaPatAccessMetricName),
+			patAccessUsersCounter:    config.UsageTracker.RegisterUniqueCounter(k8sApiProxyRequestsUniqueUsersViaPatAccessMetricName),
+			patAccessAgentsCounter:   config.UsageTracker.RegisterUniqueCounter(k8sApiProxyRequestsUniqueAgentsViaPatAccessMetricName),
+			responseSerializer:       serializer.NewCodecFactory(runtime.NewScheme()),
+			traceProvider:            config.TraceProvider,
+			tracePropagator:          config.TracePropagator,
+			meterProvider:            config.MeterProvider,
+			serverName:               serverName,
+			serverVia:                "gRPC/1.0 " + serverName,
+			urlPathPrefix:            k8sApi.UrlPathPrefix,
+			listenerGracePeriod:      listenCfg.ListenGracePeriod.AsDuration(),
+			shutdownGracePeriod:      listenCfg.ShutdownGracePeriod.AsDuration(),
 		},
 		listener: listener,
 	}

internal/module/kubernetes_api/server/proxy.go

@@ -29,6 +29,7 @@ import (
 	"gitlab.com/gitlab-org/cluster-integration/gitlab-agent/v16/internal/module/kubernetes_api/rpc"
 	"gitlab.com/gitlab-org/cluster-integration/gitlab-agent/v16/internal/module/modserver"
 	"gitlab.com/gitlab-org/cluster-integration/gitlab-agent/v16/internal/module/modshared"
+	"gitlab.com/gitlab-org/cluster-integration/gitlab-agent/v16/internal/module/usage_metrics"
 	"gitlab.com/gitlab-org/cluster-integration/gitlab-agent/v16/internal/tool/cache"
 	"gitlab.com/gitlab-org/cluster-integration/gitlab-agent/v16/internal/tool/grpctool"
 	"gitlab.com/gitlab-org/cluster-integration/gitlab-agent/v16/internal/tool/httpz"
@@ -79,19 +80,30 @@ type proxyUserCacheKey struct {
 }
 
 type kubernetesApiProxy struct {
-	log                     *zap.Logger
-	api                     modserver.Api
-	kubernetesApiClient     rpc.KubernetesApiClient
-	gitLabClient            gitlab.ClientInterface
-	allowedOriginUrls       []string
-	allowedAgentsCache      *cache.CacheWithErr[string, *gapi.AllowedAgentsForJob]
-	authorizeProxyUserCache *cache.CacheWithErr[proxyUserCacheKey, *gapi.AuthorizeProxyUserResponse]
-	responseSerializer      runtime.NegotiatedSerializer
-	traceProvider           trace.TracerProvider
-	tracePropagator         propagation.TextMapPropagator
-	meterProvider           metric.MeterProvider
-	serverName              string
-	serverVia               string
+	log                      *zap.Logger
+	api                      modserver.Api
+	kubernetesApiClient      rpc.KubernetesApiClient
+	gitLabClient             gitlab.ClientInterface
+	allowedOriginUrls        []string
+	allowedAgentsCache       *cache.CacheWithErr[string, *gapi.AllowedAgentsForJob]
+	authorizeProxyUserCache  *cache.CacheWithErr[proxyUserCacheKey, *gapi.AuthorizeProxyUserResponse]
+	requestCounter           usage_metrics.Counter
+	ciTunnelUsersCounter     usage_metrics.UniqueCounter
+	ciAccessRequestCounter   usage_metrics.Counter
+	ciAccessUsersCounter     usage_metrics.UniqueCounter
+	ciAccessAgentsCounter    usage_metrics.UniqueCounter
+	userAccessRequestCounter usage_metrics.Counter
+	userAccessUsersCounter   usage_metrics.UniqueCounter
+	userAccessAgentsCounter  usage_metrics.UniqueCounter
+	patAccessRequestCounter  usage_metrics.Counter
+	patAccessUsersCounter    usage_metrics.UniqueCounter
+	patAccessAgentsCounter   usage_metrics.UniqueCounter
+	responseSerializer       runtime.NegotiatedSerializer
+	traceProvider            trace.TracerProvider
+	tracePropagator          propagation.TextMapPropagator
+	meterProvider            metric.MeterProvider
+	serverName               string
+	serverVia                string
 	// urlPathPrefix is guaranteed to end with / by defaulting.
 	urlPathPrefix       string
 	listenerGracePeriod time.Duration
@@ -176,7 +188,7 @@ func (p *kubernetesApiProxy) proxyInternal(w http.ResponseWriter, r *http.Reques
 		}
 	}
 
-	log, agentId, _, impConfig, eResp := p.authenticateAndImpersonateRequest(ctx, log, r)
+	log, agentId, userId, impConfig, eResp := p.authenticateAndImpersonateRequest(ctx, log, r)
 	if eResp != nil {
 		// If GitLab doesn't authorize the proxy user to make the call,
 		// we send an extra header to indicate that, so that the client
@@ -188,6 +200,9 @@ func (p *kubernetesApiProxy) proxyInternal(w http.ResponseWriter, r *http.Reques
 		return log, agentId, eResp
 	}
 
+	p.requestCounter.Inc() // Count only authenticated and authorized requests
+	p.ciTunnelUsersCounter.Add(userId)
+
 	md := metadata.Pairs(modserver.RoutingAgentIdMetadataKey, strconv.FormatInt(agentId, 10))
 	mkClient, err := p.kubernetesApiClient.MakeRequest(metadata.NewOutgoingContext(ctx, md))
 	if err != nil {
@@ -252,6 +267,10 @@ func (p *kubernetesApiProxy) authenticateAndImpersonateRequest(ctx context.Conte
 			}
 		}
 
+		// update usage metrics for `ci_access` requests using the CI tunnel
+		p.ciAccessRequestCounter.Inc()
+		p.ciAccessUsersCounter.Add(userId)
+		p.ciAccessAgentsCounter.Add(agentId)
 	case sessionCookieAuthn:
 		auth, eResp := p.authorizeProxyUser(ctx, log, agentId, "session_cookie", c.encryptedPublicSessionId, c.csrfToken)
 		if eResp != nil {
@@ -269,6 +288,10 @@ func (p *kubernetesApiProxy) authenticateAndImpersonateRequest(ctx context.Conte
 			}
 		}
 
+		// update usage metrics for `user_access` requests using the CI tunnel
+		p.userAccessRequestCounter.Inc()
+		p.userAccessUsersCounter.Add(userId)
+		p.userAccessAgentsCounter.Add(agentId)
 	case patAuthn:
 		// TODO: Figure out our authorization logic
 		//auth, eResp := p.authorizeProxyUser(ctx, log, agentId, "personal_access_token", c.token, "")
@@ -288,6 +311,10 @@ func (p *kubernetesApiProxy) authenticateAndImpersonateRequest(ctx context.Conte
 		//}
 		impConfig = nil
 
+		// update usage metrics for PAT requests using the CI tunnel
+		p.patAccessRequestCounter.Inc()
+		p.patAccessUsersCounter.Add(userId)
+		p.patAccessAgentsCounter.Add(agentId)
 	default: // This should never happen
 		msg := "Invalid authorization type"
 		p.api.HandleProcessingError(ctx, log, agentId, msg, err)

internal/module/modserver/api.go

@@ -7,6 +7,7 @@ import (
 	"github.com/redis/rueidis"
 	"gitlab.com/gitlab-org/cluster-integration/gitlab-agent/v16/internal/module/modshared"
 	"gitlab.com/gitlab-org/cluster-integration/gitlab-agent/v16/internal/module/observability"
+	"gitlab.com/gitlab-org/cluster-integration/gitlab-agent/v16/internal/module/usage_metrics"
 	"gitlab.com/gitlab-org/cluster-integration/gitlab-agent/v16/internal/tool/syncz"
 	"gitlab.com/gitlab-org/cluster-integration/gitlab-agent/v16/pkg/event"
 	"gitlab.com/gitlab-org/cluster-integration/gitlab-agent/v16/pkg/kascfg"
@@ -46,7 +47,8 @@ type Config struct {
 	Config *kascfg.ConfigurationFile
 	// Registerer allows to register metrics.
 	// Metrics should be registered in Run and unregistered before Run returns.
-	Registerer prometheus.Registerer
+	Registerer   prometheus.Registerer
+	UsageTracker usage_metrics.UsageTrackerRegisterer
 	// AgentServer is the gRPC server agentk is talking to.
 	// This can be used to add endpoints in Factory.New.
 	// Request handlers can obtain the per-request logger using grpctool.LoggerFromContext(requestContext).

internal/module/usage_metrics/BUILD.bazel

@@ -0,0 +1,20 @@
+load("//build:build.bzl", "go_custom_test")
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "usage_metrics",
+    srcs = ["api.go"],
+    importpath = "gitlab.com/gitlab-org/cluster-integration/gitlab-agent/v16/internal/module/usage_metrics",
+    visibility = ["//:__subpackages__"],
+)
+
+go_custom_test(
+    name = "usage_metrics_test",
+    srcs = ["api_test.go"],
+    embed = [":usage_metrics"],
+    deps = [
+        "@com_github_google_go_cmp//cmp",
+        "@com_github_google_go_cmp//cmp/cmpopts",
+        "@com_github_stretchr_testify//require",
+    ],
+)