Ta update configs to enable mtls

.chloggen/TA-update-configs-to-enable-mtls.yaml

@@ -0,0 +1,18 @@
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: enhancement
+
+# The name of the component, or a single word describing the area of concern, (e.g. collector, target allocator, auto-instrumentation, opamp, github action)
+component: target allocator collector
+
+# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: "Enable mTLS between the TA and collector for passing secrets in the scrape_config securely"
+
+# One or more tracking issues related to the change
+issues: [1669]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext: |
+  This change enables mTLS between the collector and the target allocator.
+  This is necessary for passing secrets securely from the TA to the collector for scraping endpoints that have authentication.

.github/workflows/e2e.yaml

@@ -34,13 +34,16 @@ jobs:
          - e2e-upgrade
          - e2e-multi-instrumentation
          - e2e-metadata-filters
+         - e2e-ta-collector-mtls
        include:
          - group: e2e-instrumentation
            setup: "add-instrumentation-params prepare-e2e"
          - group: e2e-multi-instrumentation
            setup: "add-multi-instrumentation-params prepare-e2e"
          - group: e2e-metadata-filters
            setup: "add-operator-arg OPERATOR_ARG='--annotations-filter=.*filter.out --annotations-filter=config.*.gke.io.* --labels=.*filter.out' prepare-e2e"
+         - group: e2e-ta-collector-mtls
+           setup: "add-operator-arg OPERATOR_ARG='--feature-gates=operator.targetallocator.mtls' add-certmanager-permissions prepare-e2e"
          - group: e2e-automatic-rbac
            setup: "add-rbac-permissions-to-operator prepare-e2e"
     steps:

.gitignore

@@ -1,4 +1,3 @@
-
 # Binaries for programs and plugins
 *.exe
 *.exe~
@@ -39,8 +38,9 @@ config/manager/kustomization.yaml
 kubeconfig
 tests/_build/
 config/rbac/extra-permissions-operator/
+config/rbac/certmanager-permissions/
 
 # autoinstrumentation artifacts
 build
 node_modules
-package-lock.json
+package-lock.json

Makefile

@@ -306,6 +306,18 @@ e2e-prometheuscr: chainsaw
 e2e-targetallocator: chainsaw
 	$(CHAINSAW) test --test-dir ./tests/e2e-targetallocator
 
+.PHONY: add-certmanager-permissions
+add-certmanager-permissions: 
+	# Kustomize only allows patches in the folder where the kustomization is located
+	# This folder is ignored by .gitignore
+	cp -r tests/e2e-ta-collector-mtls/certmanager-permissions config/rbac/certmanager-permissions
+	cd config/rbac && $(KUSTOMIZE) edit add patch --kind ClusterRole --name manager-role --path certmanager-permissions/certmanager.yaml
+
+# Target allocator collector mTLS end-to-tests
+.PHONY: e2e-ta-collector-mtls
+e2e-ta-collector-mtls: chainsaw
+	$(CHAINSAW) test --test-dir ./tests/e2e-ta-collector-mtls
+
 # end-to-end-test for Annotations/Labels Filters
 .PHONY: e2e-metadata-filters
 e2e-metadata-filters: chainsaw

cmd/otel-allocator/README.md

@@ -211,9 +211,42 @@ rules:
 
 ### Service / Pod monitor endpoint credentials
 
-If your service or pod monitor endpoints require credentials or other supported form of authentication (bearer token, basic auth, OAuth2 etc.), you need to ensure that the collector has access to this information. Due to some limitations in how the endpoints configuration is handled, target allocator currently does **not** support credentials provided via secrets. It is only possible to provide credentials in a file (for more details see issue https://github.com/open-telemetry/opentelemetry-operator/issues/1669).
+If your service or pod monitor endpoints require authentication (such as bearer tokens, basic auth, OAuth2, etc.), you must ensure that the collector has access to these credentials.
+
+To secure the connection between the target allocator and the collector so that the secrets can be retrieved, mTLS is used. This involves the use of cert-manager to manage the CA, server, and client certificates.
+
+Prerequisites:
+- Ensure cert-manager is installed in your Kubernetes cluster.
+- Grant RBAC Permissions:
+
+    - The target allocator needs the appropriate RBAC permissions to get the secrets referenced in the Service / Pod monitor.
+
+    -  The operator needs the appropriate RBAC permissions to manage cert-manager resources. The following clusterRole can be used to grant the necessary permissions:
+
+        ```yaml
+        apiVersion: rbac.authorization.k8s.io/v1
+        kind: ClusterRole
+        metadata:
+          name:  opentelemetry-operator-controller-manager-cert-manager-role
+        rules:
+        - apiGroups:
+          - cert-manager.io
+          resources:
+          - issuers
+          - certificaterequests
+          - certificates
+          verbs:
+          - create
+          - get
+          - list
+          - watch
+          - update
+          - patch
+          - delete
+        ```
+
+- Enable the `operator.targetallocator.mtls` feature gate in the operator's deployment. 
 
-In order to ensure your endpoints can be scraped, your collector instance needs to have the particular secret mounted as a file at the correct path.
 
 
 # Design

cmd/otel-allocator/config/config.go

@@ -115,29 +115,34 @@ func LoadFromCLI(target *Config, flagSet *pflag.FlagSet) error {
 		target.PrometheusCR.Enabled = prometheusCREnabled
 	}
 
-	target.HTTPS.Enabled, err = getHttpsEnabled(flagSet)
-	if err != nil {
+	if httpsEnabled, changed, err := getHttpsEnabled(flagSet); err != nil {
 		return err
+	} else if changed {
+		target.HTTPS.Enabled = httpsEnabled
 	}
 
-	target.HTTPS.ListenAddr, err = getHttpsListenAddr(flagSet)
-	if err != nil {
+	if listenAddrHttps, changed, err := getHttpsListenAddr(flagSet); err != nil {
 		return err
+	} else if changed {
+		target.HTTPS.ListenAddr = listenAddrHttps
 	}
 
-	target.HTTPS.CAFilePath, err = getHttpsCAFilePath(flagSet)
-	if err != nil {
+	if caFilePath, changed, err := getHttpsCAFilePath(flagSet); err != nil {
 		return err
+	} else if changed {
+		target.HTTPS.CAFilePath = caFilePath
 	}
 
-	target.HTTPS.TLSCertFilePath, err = getHttpsTLSCertFilePath(flagSet)
-	if err != nil {
+	if tlsCertFilePath, changed, err := getHttpsTLSCertFilePath(flagSet); err != nil {
 		return err
+	} else if changed {
+		target.HTTPS.TLSCertFilePath = tlsCertFilePath
 	}
 
-	target.HTTPS.TLSKeyFilePath, err = getHttpsTLSKeyFilePath(flagSet)
-	if err != nil {
+	if tlsKeyFilePath, changed, err := getHttpsTLSKeyFilePath(flagSet); err != nil {
 		return err
+	} else if changed {
+		target.HTTPS.TLSKeyFilePath = tlsKeyFilePath
 	}
 
 	return nil

cmd/otel-allocator/config/config_test.go

@@ -64,6 +64,7 @@ func TestLoad(t *testing.T) {
 				},
 				HTTPS: HTTPSServerConfig{
 					Enabled:         true,
+					ListenAddr:      ":8443",
 					CAFilePath:      "/path/to/ca.pem",
 					TLSCertFilePath: "/path/to/cert.pem",
 					TLSKeyFilePath:  "/path/to/key.pem",

cmd/otel-allocator/config/flags.go

@@ -78,22 +78,47 @@ func getPrometheusCREnabled(flagSet *pflag.FlagSet) (value bool, changed bool, e
 	return
 }
 
-func getHttpsListenAddr(flagSet *pflag.FlagSet) (string, error) {
-	return flagSet.GetString(listenAddrHttpsFlagName)
+func getHttpsListenAddr(flagSet *pflag.FlagSet) (value string, changed bool, err error) {
+	if changed = flagSet.Changed(listenAddrHttpsFlagName); !changed {
+		value, err = ":8443", nil
+		return
+	}
+	value, err = flagSet.GetString(listenAddrHttpsFlagName)
+	return
 }
 
-func getHttpsEnabled(flagSet *pflag.FlagSet) (bool, error) {
-	return flagSet.GetBool(httpsEnabledFlagName)
+func getHttpsEnabled(flagSet *pflag.FlagSet) (value bool, changed bool, err error) {
+	if changed = flagSet.Changed(httpsEnabledFlagName); !changed {
+		value, err = false, nil
+		return
+	}
+	value, err = flagSet.GetBool(httpsEnabledFlagName)
+	return
 }
 
-func getHttpsCAFilePath(flagSet *pflag.FlagSet) (string, error) {
-	return flagSet.GetString(httpsCAFilePathFlagName)
+func getHttpsCAFilePath(flagSet *pflag.FlagSet) (value string, changed bool, err error) {
+	if changed = flagSet.Changed(httpsCAFilePathFlagName); !changed {
+		value, err = "", nil
+		return
+	}
+	value, err = flagSet.GetString(httpsCAFilePathFlagName)
+	return
 }
 
-func getHttpsTLSCertFilePath(flagSet *pflag.FlagSet) (string, error) {
-	return flagSet.GetString(httpsTLSCertFilePathFlagName)
+func getHttpsTLSCertFilePath(flagSet *pflag.FlagSet) (value string, changed bool, err error) {
+	if changed = flagSet.Changed(httpsTLSCertFilePathFlagName); !changed {
+		value, err = "", nil
+		return
+	}
+	value, err = flagSet.GetString(httpsTLSCertFilePathFlagName)
+	return
 }
 
-func getHttpsTLSKeyFilePath(flagSet *pflag.FlagSet) (string, error) {
-	return flagSet.GetString(httpsTLSKeyFilePathFlagName)
+func getHttpsTLSKeyFilePath(flagSet *pflag.FlagSet) (value string, changed bool, err error) {
+	if changed = flagSet.Changed(httpsTLSKeyFilePathFlagName); !changed {
+		value, err = "", nil
+		return
+	}
+	value, err = flagSet.GetString(httpsTLSKeyFilePathFlagName)
+	return
 }

cmd/otel-allocator/config/flags_test.go

@@ -77,13 +77,19 @@ func TestFlagGetters(t *testing.T) {
 			name:          "HttpsServer",
 			flagArgs:      []string{"--" + httpsEnabledFlagName, "true"},
 			expectedValue: true,
-			getterFunc:    func(fs *pflag.FlagSet) (interface{}, error) { return getHttpsEnabled(fs) },
+			getterFunc: func(fs *pflag.FlagSet) (interface{}, error) {
+				value, _, err := getHttpsEnabled(fs)
+				return value, err
+			},
 		},
 		{
 			name:          "HttpsServerKey",
 			flagArgs:      []string{"--" + httpsTLSKeyFilePathFlagName, "/path/to/tls.key"},
 			expectedValue: "/path/to/tls.key",
-			getterFunc:    func(fs *pflag.FlagSet) (interface{}, error) { return getHttpsTLSKeyFilePath(fs) },
+			getterFunc: func(fs *pflag.FlagSet) (interface{}, error) {
+				value, _, err := getHttpsTLSKeyFilePath(fs)
+				return value, err
+			},
 		},
 	}
 

cmd/otel-allocator/config/testdata/config_test.yaml

@@ -7,6 +7,7 @@ prometheus_cr:
   scrape_interval: 60s
 https:
   enabled: true
+  listen_addr: :8443
   ca_file_path: /path/to/ca.pem
   tls_cert_file_path: /path/to/cert.pem
   tls_key_file_path: /path/to/key.pem