clustermesh: switch to the "local" user to access kvstoremesh data

Now that the support for the "local" etcd user has been available for
one version [1], and Cilium agents are mounting the corresponding
certificate [2], let's switch the agents to actually use it, rather than
the remote one, to access cached data. Additionally, let's strip down
the permissions of the remote user, as remote clusters don't need to
access cached data.

[1]: cb6a58b ("clustermesh: granular etcd permissions for kvstoremesh cached data")
[2]: c464e66 ("helm: mount kvstoremesh-specific certificate into cilium agents")

Signed-off-by: Marco Iorio <[email protected]>

Documentation/operations/troubleshooting_clustermesh.rst

@@ -99,9 +99,9 @@ you may perform the following steps to troubleshoot ClusterMesh issues.
       containers in the clustermesh-apiserver deployment, to authenticate against the
       sidecar etcd instance. Not applicable if an external etcd cluster is used.
 
-    * ``clustermesh-apiserver-remote-cert``, which is used by Cilium agents, and
-      optionally the kvstoremesh container in the clustermesh-apiserver deployment,
-      to authenticate against remote etcd instances (either internal or external).
+    * ``clustermesh-apiserver-remote-cert``, which is used by Cilium agents, or
+      the kvstoremesh container in the clustermesh-apiserver deployment when
+      KVStoreMesh is enabled, to authenticate against remote etcd instances.
 
     * ``clustermesh-apiserver-local-cert``, which is used by Cilium agents to
       authenticate against the local etcd instance. Only applicable if KVStoreMesh

install/kubernetes/cilium/templates/clustermesh-apiserver/users-configmap.yaml

@@ -17,10 +17,6 @@ metadata:
 data:
   users.yaml: |
     users:
-    {{- if .Values.clustermesh.apiserver.kvstoremesh.enabled }}
-    - name: remote-{{ .Values.cluster.name }}
-      role: remote
-    {{- end }}
     {{- range .Values.clustermesh.config.clusters }}
     - name: remote-{{ .name }}
       role: remote

install/kubernetes/cilium/templates/clustermesh-config/_helpers.tpl

@@ -3,7 +3,13 @@
 {{- $domain := index . 1 -}}
 {{- $override := index . 2 -}}
 {{- /* The parenthesis around $cluster.tls are required, since it can be null: https://stackoverflow.com/a/68807258 */}}
-{{- $prefix := ternary "common-" (printf "%s." $cluster.name) (or (ne $override "") (empty ($cluster.tls).cert) (empty ($cluster.tls).key)) -}}
+{{- $prefix := ternary "common-" (printf "%s." $cluster.name) (or (empty ($cluster.tls).cert) (empty ($cluster.tls).key)) -}}
+{{- /* KVStoreMesh is enabled, and we are generating the secret used by Cilium agents. */}}
+{{- /* In other words, we want to connect to KVStoreMesh, opposed to the etcd instance */}}
+{{- /* in the remote cluster; hence we need to use the dedicated certificate and key.  */}}
+{{- if ne $override "" -}}
+{{- $prefix = "local-" -}}
+{{- end -}}
 
 endpoints:
 {{- if ne $override "" }}
@@ -13,8 +19,11 @@ endpoints:
 {{- else }}
 - https://{{ $cluster.address | required "missing clustermesh.apiserver.config.clusters.address" }}:{{ $cluster.port }}
 {{- end }}
-{{- if not (empty ($cluster.tls).caCert) }}
-{{- /* The custom CA configuration takes effect only if a custom certificate and key are also set */}}
+{{- if or (ne $override "") (not (empty ($cluster.tls).caCert)) }}
+{{- /* The custom CA configuration takes effect only if a custom certificate and key are also set, */}}
+{{- /* otherwise we may enter this branch, but the prefix is still set to common-.                 */}}
+{{- /* Additionally, when KVStoreMesh is enabled, and we are generating the secret for the agents, */}}
+{{- /* we want to always use the corresponding CA certificate, that is the one with local- prefix. */}}
 trusted-ca-file: /var/lib/cilium/clustermesh/{{ $prefix }}etcd-client-ca.crt
 {{- else }}
 trusted-ca-file: /var/lib/cilium/clustermesh/common-etcd-client-ca.crt

pkg/kvstore/etcdinit/init.go

@@ -311,10 +311,5 @@ func rangesForRemoteRole(clusterName string) []keyRange {
 		rangeForPrefix(kvstore.StatePrefix),
 		rangeForKey(path.Join(kvstore.ClusterConfigPrefix, clusterName)),
 		rangeForPrefix(path.Join(kvstore.SyncedPrefix, clusterName)),
-
-		// kvstoremesh-specific prefixes still allowed for backward compatibility
-		rangeForPrefix(kvstore.CachePrefix),
-		rangeForPrefix(kvstore.ClusterConfigPrefix),
-		rangeForPrefix(kvstore.SyncedPrefix),
 	}
 }