Require Bearer token for /metrics endpoint (#3317)

Signed-off-by: João Vilaça <[email protected]>

Author: machadovilaca

cmd/hyperconverged-cluster-operator/main.go

@@ -57,6 +57,7 @@ import (
 	"github.com/kubevirt/hyperconverged-cluster-operator/controllers/nodes"
 	"github.com/kubevirt/hyperconverged-cluster-operator/controllers/observability"
 	"github.com/kubevirt/hyperconverged-cluster-operator/controllers/operands"
+	"github.com/kubevirt/hyperconverged-cluster-operator/pkg/authorization"
 	"github.com/kubevirt/hyperconverged-cluster-operator/pkg/monitoring/metrics"
 	hcoutil "github.com/kubevirt/hyperconverged-cluster-operator/pkg/util"
 )
@@ -341,7 +342,8 @@ func getCacheOption(operatorNamespace string, ci hcoutil.ClusterInfo) cache.Opti
 func getManagerOptions(operatorNamespace string, needLeaderElection bool, ci hcoutil.ClusterInfo, scheme *apiruntime.Scheme) manager.Options {
 	return manager.Options{
 		Metrics: server.Options{
-			BindAddress: fmt.Sprintf("%s:%d", hcoutil.MetricsHost, hcoutil.MetricsPort),
+			BindAddress:    fmt.Sprintf("%s:%d", hcoutil.MetricsHost, hcoutil.MetricsPort),
+			FilterProvider: authorization.HttpWithBearerToken,
 		},
 		HealthProbeBindAddress: fmt.Sprintf("%s:%d", hcoutil.HealthProbeHost, hcoutil.HealthProbePort),
 		ReadinessEndpointName:  hcoutil.ReadinessEndpointName,

cmd/hyperconverged-cluster-webhook/main.go

@@ -13,6 +13,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/webhook"
 
 	webhookscontrollers "github.com/kubevirt/hyperconverged-cluster-operator/controllers/webhooks"
+	"github.com/kubevirt/hyperconverged-cluster-operator/pkg/authorization"
 	"github.com/kubevirt/hyperconverged-cluster-operator/pkg/webhooks/validator"
 
 	csvv1alpha1 "github.com/operator-framework/api/pkg/operators/v1alpha1"
@@ -93,7 +94,8 @@ func main() {
 	// Create a new Cmd to provide shared dependencies and start components
 	mgr, err := manager.New(cfg, manager.Options{
 		Metrics: server.Options{
-			BindAddress: fmt.Sprintf("%s:%d", hcoutil.MetricsHost, hcoutil.MetricsPort),
+			BindAddress:    fmt.Sprintf("%s:%d", hcoutil.MetricsHost, hcoutil.MetricsPort),
+			FilterProvider: authorization.HttpWithBearerToken,
 		},
 		HealthProbeBindAddress: fmt.Sprintf("%s:%d", hcoutil.HealthProbeHost, hcoutil.HealthProbePort),
 		ReadinessEndpointName:  hcoutil.ReadinessEndpointName,

controllers/alerts/metrics_test.go

@@ -103,7 +103,7 @@ var _ = Describe("alert tests", func() {
 			req = commontestutils.NewReq(hco)
 			Expect(r.UpdateRelatedObjects(req)).To(Succeed())
 			Expect(req.StatusDirty).To(BeTrue())
-			Expect(hco.Status.RelatedObjects).To(HaveLen(5))
+			Expect(hco.Status.RelatedObjects).To(HaveLen(6))
 
 			Expect(ee.CheckEvents(expectedEvents)).To(BeTrue())
 		})

controllers/alerts/reconciler.go

@@ -69,6 +69,7 @@ func NewMonitoringReconciler(ci hcoutil.ClusterInfo, cl client.Client, ee hcouti
 			newRoleReconciler(namespace, owner),
 			newRoleBindingReconciler(namespace, owner, ci),
 			newMetricServiceReconciler(namespace, owner),
+			newSecretReconciler(namespace, owner),
 			newServiceMonitorReconciler(namespace, owner),
 		},
 		scheme:       scheme,

controllers/alerts/secret.go

@@ -0,0 +1,96 @@
+package alerts
+
+import (
+	"context"
+
+	"github.com/go-logr/logr"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	"github.com/kubevirt/hyperconverged-cluster-operator/pkg/authorization"
+	hcoutil "github.com/kubevirt/hyperconverged-cluster-operator/pkg/util"
+)
+
+const (
+	secretName = "hco-bearer-auth"
+)
+
+type secretReconciler struct {
+	theSecret *corev1.Secret
+}
+
+func newSecretReconciler(namespace string, owner metav1.OwnerReference) *secretReconciler {
+	return &secretReconciler{
+		theSecret: NewSecret(namespace, owner),
+	}
+}
+
+func (r *secretReconciler) Kind() string {
+	return "Secret"
+}
+
+func (r *secretReconciler) ResourceName() string {
+	return secretName
+}
+
+func (r *secretReconciler) GetFullResource() client.Object {
+	return r.theSecret.DeepCopy()
+}
+
+func (r *secretReconciler) EmptyObject() client.Object {
+	return &corev1.Secret{}
+}
+
+func (r *secretReconciler) UpdateExistingResource(ctx context.Context, cl client.Client, resource client.Object, logger logr.Logger) (client.Object, bool, error) {
+	found := resource.(*corev1.Secret)
+	modified := false
+
+	token, err := authorization.CreateToken()
+	if err != nil {
+		return nil, false, err
+	}
+
+	if found.Data["token"] == nil || string(found.Data["token"]) != token {
+		found.StringData = map[string]string{
+			"token": token,
+		}
+		modified = true
+	}
+
+	modified = updateCommonDetails(&r.theSecret.ObjectMeta, &found.ObjectMeta) || modified
+
+	if modified {
+		err := cl.Update(ctx, found)
+		if err != nil {
+			logger.Error(err, "failed to update the Secret")
+			return nil, false, err
+		}
+	}
+
+	return found, modified, nil
+}
+
+func NewSecret(namespace string, owner metav1.OwnerReference) *corev1.Secret {
+	token, err := authorization.CreateToken()
+	if err != nil {
+		logger.Error(err, "failed to create bearer token")
+		return nil
+	}
+
+	return &corev1.Secret{
+		TypeMeta: metav1.TypeMeta{
+			APIVersion: "v1",
+			Kind:       "Secret",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:            secretName,
+			Namespace:       namespace,
+			Labels:          hcoutil.GetLabels(hcoutil.HyperConvergedName, hcoutil.AppComponentMonitoring),
+			OwnerReferences: []metav1.OwnerReference{owner},
+		},
+		StringData: map[string]string{
+			"token": token,
+		},
+	}
+}

controllers/alerts/serviceMonitor.go

@@ -6,6 +6,7 @@ import (
 
 	"github.com/go-logr/logr"
 	monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
+	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
@@ -63,7 +64,17 @@ func NewServiceMonitor(namespace string, owner metav1.OwnerReference) *monitorin
 		Selector: metav1.LabelSelector{
 			MatchLabels: labels,
 		},
-		Endpoints: []monitoringv1.Endpoint{{Port: operatorPortName}},
+		Endpoints: []monitoringv1.Endpoint{{
+			Port: operatorPortName,
+			Authorization: &monitoringv1.SafeAuthorization{
+				Credentials: &corev1.SecretKeySelector{
+					LocalObjectReference: corev1.LocalObjectReference{
+						Name: secretName,
+					},
+					Key: "token",
+				},
+			},
+		}},
 	}
 
 	return &monitoringv1.ServiceMonitor{

controllers/hyperconverged/hyperconverged_controller_test.go

@@ -244,7 +244,7 @@ var _ = Describe("HyperconvergedController", func() {
 						foundResource),
 				).ToNot(HaveOccurred())
 				// Check conditions
-				Expect(foundResource.Status.RelatedObjects).To(HaveLen(26))
+				Expect(foundResource.Status.RelatedObjects).To(HaveLen(27))
 				expectedRef := corev1.ObjectReference{
 					Kind:            "PrometheusRule",
 					Namespace:       namespace,
@@ -309,7 +309,7 @@ var _ = Describe("HyperconvergedController", func() {
 
 				verifySystemHealthStatusError(foundResource)
 
-				Expect(foundResource.Status.RelatedObjects).To(HaveLen(21))
+				Expect(foundResource.Status.RelatedObjects).To(HaveLen(22))
 				expectedRef := corev1.ObjectReference{
 					Kind:            "PrometheusRule",
 					Namespace:       namespace,
@@ -820,7 +820,7 @@ var _ = Describe("HyperconvergedController", func() {
 				).To(Succeed())
 
 				Expect(foundResource.Status.RelatedObjects).ToNot(BeNil())
-				Expect(foundResource.Status.RelatedObjects).To(HaveLen(21))
+				Expect(foundResource.Status.RelatedObjects).To(HaveLen(22))
 				Expect(foundResource.ObjectMeta.Finalizers).To(Equal([]string{FinalizerName}))
 
 				// Now, delete HCO

deploy/cluster_role.yaml

@@ -858,6 +858,16 @@ rules:
   - get
   - list
   - watch
+- apiGroups:
+  - ""
+  resources:
+  - secrets
+  verbs:
+  - get
+  - list
+  - watch
+  - create
+  - update
 - apiGroups:
   - ""
   resources:

deploy/index-image/community-kubevirt-hyperconverged/1.14.0/manifests/kubevirt-hyperconverged-operator.v1.14.0.clusterserviceversion.yaml

@@ -329,6 +329,16 @@ spec:
           - get
           - list
           - watch
+        - apiGroups:
+          - ""
+          resources:
+          - secrets
+          verbs:
+          - get
+          - list
+          - watch
+          - create
+          - update
         - apiGroups:
           - ""
           resources:

deploy/olm-catalog/community-kubevirt-hyperconverged/1.14.0/manifests/kubevirt-hyperconverged-operator.v1.14.0.clusterserviceversion.yaml

@@ -9,7 +9,7 @@ metadata:
     certified: "false"
     console.openshift.io/disable-operand-delete: "true"
     containerImage: quay.io/kubevirt/hyperconverged-cluster-operator:1.14.0-unstable
-    createdAt: "2025-02-28 11:04:27"
+    createdAt: "2025-03-02 09:34:08"
     description: A unified operator deploying and controlling KubeVirt and its supporting
       operators with opinionated defaults
     features.operators.openshift.io/cnf: "false"
@@ -329,6 +329,16 @@ spec:
           - get
           - list
           - watch
+        - apiGroups:
+          - ""
+          resources:
+          - secrets
+          verbs:
+          - get
+          - list
+          - watch
+          - create
+          - update
         - apiGroups:
           - ""
           resources:

go.mod

@@ -11,6 +11,7 @@ require (
 	github.com/gertd/go-pluralize v0.2.1
 	github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32
 	github.com/go-logr/logr v1.4.2
+	github.com/golang-jwt/jwt/v5 v5.2.1
 	github.com/google/uuid v1.6.0
 	github.com/kubevirt/cluster-network-addons-operator v0.96.0
 	github.com/kubevirt/monitoring/pkg/metrics/parser v0.0.0-20240505100225-e29dee0bb12b

go.sum

@@ -80,6 +80,8 @@ github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6Wezm
 github.com/gobwas/ws v1.2.1/go.mod h1:hRKAFb8wOxFROYNsT1bqfWnhX+b5MFeJM9r2ZSwg/KY=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
+github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk=
+github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
 github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE=
 github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=

pkg/authorization/authorization_suite_test.go

@@ -0,0 +1,13 @@
+package authorization_test
+
+import (
+	"testing"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+func TestAuthorization(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Authorization Suite")
+}

pkg/authorization/bearer_token.go

@@ -0,0 +1,91 @@
+package authorization
+
+import (
+	"crypto/rand"
+	"fmt"
+	"os"
+
+	"github.com/golang-jwt/jwt/v5"
+)
+
+const (
+	TokenPathEnvVar = "KUBERNETES_SERVICE_TOKEN_PATH"
+
+	defaultTokenPath = "/var/run/secrets/kubernetes.io/serviceaccount/token"
+)
+
+var secretKey []byte
+
+func CreateToken() (string, error) {
+	token := jwt.New(jwt.SigningMethodHS256)
+
+	key, err := getSecretKey()
+	if err != nil {
+		return "", fmt.Errorf("error getting secret key: %v", err)
+	}
+
+	tokenString, err := token.SignedString(key)
+	if err != nil {
+		return "", fmt.Errorf("error signing token: %v", err)
+	}
+
+	return tokenString, nil
+}
+
+func ValidateToken(tokenString string) (bool, error) {
+	token, err := jwt.Parse(tokenString, func(token *jwt.Token) (any, error) {
+		return getSecretKey()
+	})
+	if err != nil {
+		return false, fmt.Errorf("error parsing token: %v", err)
+	}
+
+	return token.Valid, nil
+}
+
+func RefreshSecretKey() {
+	secretKey = nil
+}
+
+func getSecretKey() ([]byte, error) {
+	// if secretKey is already available, return it
+	if secretKey != nil {
+		return secretKey, nil
+	}
+
+	var err error
+
+	// get ServiceAccount token from file
+	secretKey, err = getServiceAccountToken()
+	if err != nil {
+		// if ServiceAccount token is not available, generate an in-memory token
+		secretKey, err = generateInMemorySecretKey()
+	}
+
+	return secretKey, err
+}
+
+func getServiceAccountToken() ([]byte, error) {
+	tokenPath := os.Getenv(TokenPathEnvVar)
+	if tokenPath == "" {
+		tokenPath = defaultTokenPath
+	}
+
+	saToken, err := os.ReadFile(tokenPath)
+	if err != nil {
+		return nil, fmt.Errorf("error reading ServiceAccount token: %v", err)
+	}
+
+	return saToken, nil
+}
+
+func generateInMemorySecretKey() ([]byte, error) {
+	tokenBytes := make([]byte, 32)
+
+	_, err := rand.Read(tokenBytes)
+	if err != nil {
+		return nil, fmt.Errorf("error generating in-memory token: %v", err)
+	}
+
+	return tokenBytes, nil
+}