New Feature: Multi Cluster TargetGroupBindings (kubernetes-sigs#3853)

* implement multicluster support

* correct documentation

apis/elbv2/v1alpha1/targetgroupbinding_types.go

@@ -109,6 +109,10 @@ type TargetGroupBindingSpec struct {
 	// targetGroupARN is the Amazon Resource Name (ARN) for the TargetGroup.
 	TargetGroupARN string `json:"targetGroupARN"`
 
+	// MultiClusterTargetGroup Denotes if the TargetGroup is shared among multiple clusters
+	// +optional
+	MultiClusterTargetGroup bool `json:"multiClusterTargetGroup,omitempty"`
+
 	// targetType is the TargetType of TargetGroup. If unspecified, it will be automatically inferred.
 	// +optional
 	TargetType *TargetType `json:"targetType,omitempty"`

apis/elbv2/v1beta1/targetgroupbinding_types.go

@@ -127,6 +127,10 @@ type TargetGroupBindingSpec struct {
 	// +kubebuilder:validation:MinLength=1
 	TargetGroupARN string `json:"targetGroupARN"`
 
+	// MultiClusterTargetGroup Denotes if the TargetGroup is shared among multiple clusters
+	// +optional
+	MultiClusterTargetGroup bool `json:"multiClusterTargetGroup,omitempty"`
+
 	// targetType is the TargetType of TargetGroup. If unspecified, it will be automatically inferred.
 	// +optional
 	TargetType *TargetType `json:"targetType,omitempty"`

config/crd/bases/elbv2.k8s.aws_targetgroupbindings.yaml

@@ -60,6 +60,10 @@ spec:
           spec:
             description: TargetGroupBindingSpec defines the desired state of TargetGroupBinding
             properties:
+              multiClusterTargetGroup:
+                description: MultiClusterTargetGroup Denotes if the TargetGroup is
+                  shared among multiple clusters
+                type: boolean
               networking:
                 description: networking provides the networking setup for ELBV2 LoadBalancer
                   to access targets in TargetGroup.
@@ -233,6 +237,10 @@ spec:
                 - ipv4
                 - ipv6
                 type: string
+              multiClusterTargetGroup:
+                description: MultiClusterTargetGroup Denotes if the TargetGroup is
+                  shared among multiple clusters
+                type: boolean
               networking:
                 description: networking defines the networking rules to allow ELBV2
                   LoadBalancer to access targets in TargetGroup.

config/rbac/role.yaml

@@ -4,6 +4,15 @@ kind: ClusterRole
 metadata:
   name: controller-role
 rules:
+- apiGroups:
+  - ""
+  resources:
+  - configmaps
+  verbs:
+  - create
+  - delete
+  - get
+  - update
 - apiGroups:
   - ""
   resources:

controllers/elbv2/targetgroupbinding_controller.go

@@ -87,6 +87,7 @@ type targetGroupBindingReconciler struct {
 // +kubebuilder:rbac:groups="",resources=nodes,verbs=get;list;watch
 // +kubebuilder:rbac:groups="",resources=endpoints,verbs=get;list;watch
 // +kubebuilder:rbac:groups="",resources=namespaces,verbs=get;list;watch
+// +kubebuilder:rbac:groups="",resources=configmaps,verbs=get;delete;create;update
 // +kubebuilder:rbac:groups="",resources=events,verbs=create;patch
 // +kubebuilder:rbac:groups="discovery.k8s.io",resources=endpointslices,verbs=get;list;watch
 

docs/guide/service/annotations.md

@@ -52,7 +52,8 @@
 | [service.beta.kubernetes.io/aws-load-balancer-security-groups](#security-groups)                 | stringList              |                           |                                                        | 
 | [service.beta.kubernetes.io/aws-load-balancer-manage-backend-security-group-rules](#manage-backend-sg-rules)  | boolean    | true                      | If `service.beta.kubernetes.io/aws-load-balancer-security-groups` is specified, this must also be explicitly specified otherwise it defaults to `false`. |
 | [service.beta.kubernetes.io/aws-load-balancer-inbound-sg-rules-on-private-link-traffic](#update-security-settings)         | string                  |                           |                                                                                   
-| [service.beta.kubernetes.io/aws-load-balancer-listener-attributes.${Protocol}-${Port}](#listener-attributes)         | stringMap                  |                           |   
+| [service.beta.kubernetes.io/aws-load-balancer-listener-attributes.${Protocol}-${Port}](#listener-attributes)         | stringMap                  |                           |
+| [service.beta.kubernetes.io/aws-load-balancer-multi-cluster-target-group](#multi-cluster-target-group)             | boolean                | false                    | If specified, the controller will only operate on targets that exist within the cluster, ignoring targets from other sources.                            |
 
 ## Traffic Routing
 Traffic Routing can be controlled with following annotations:
@@ -289,6 +290,20 @@ for proxy protocol v2 configuration.
         service.beta.kubernetes.io/aws-load-balancer-cross-zone-load-balancing-enabled 
         ```
 
+- <a name="multi-cluster-target-group">`service.beta.kubernetes.io/aws-load-balancer-multi-cluster-target-group`</a> Allows you to share the created Target Group ARN with other Load Balancer Controller managed clusters.
+
+    !!!warning ""
+    This feature does not offer any Deletion Protection. Deleting the service will still delete the Target Group. If you need to support
+    Target Groups shared with multiple clusters, it's recommended to use an out-of-band Target Group that is not managed by a Load Balancer Controller.
+
+    !!!note ""
+    - It is not recommended to change this value frequently, if ever. The recommended way to set this value is on creation of the service.
+
+    !!!example
+    ```
+    service.beta.kubernetes.io/aws-load-balancer-multi-cluster-target-group: "true"
+    ```
+
 
 ## AWS Resource Tags
 The AWS Load Balancer Controller automatically applies following tags to the AWS resources it creates (NLB/TargetGroups/Listener/ListenerRule):

docs/guide/targetgroupbinding/targetgroupbinding.md

@@ -92,6 +92,37 @@ spec:
   ...
 ```
 
+## MultiCluster Target Group
+TargetGroupBinding CRD supports sharing the same target group ARN among multiple clusters. Setting this flag will ensure the controller only operates on targets within the cluster.
+
+!!!tip ""
+    The default value is false, meaning that the controller assumes full control over the target group ARN and will deregister any targets that are not found within the cluster.
+    To set this flag for TGBs managed by the controller use either:
+    ALB: alb.ingress.kubernetes.io/multi-cluster-target-group: "true"
+    NLB: service.beta.kubernetes.io/aws-load-balancer-multi-cluster-target-group: "true"
+
+
+!!!warning ""
+    It is not recommended to change this value after TGB creation. Changing between shared / not shared might lead to leaked targets.
+
+!!!warning ""
+    Only use this flag if you intend to share the target group ARN in multiple clusters. This flag will slow down reconciles and put a small additonal load on the kubernetes control plane.
+
+
+## Sample YAML
+```yaml
+apiVersion: elbv2.k8s.aws/v1beta1
+kind: TargetGroupBinding
+metadata:
+  name: my-tgb
+spec:
+  serviceRef:
+    name: awesome-service # route traffic to the awesome-service
+    port: 80
+  targetGroupARN: <arn-to-targetGroup>
+  multiClusterTargetGroup: "true"
+```
+
 
 ## Reference
 See the [reference](./spec.md) for TargetGroupBinding CR

docs/guide/use_cases/blue_green/index.md

@@ -11,7 +11,7 @@ Multiple target groups can be attached to the same forward action of a listener
 In addition to the weighted target group, AWS announced the advanced request routing feature in 2019. Advanced request routing gives developers the ability to write rules (and route traffic) based on standard and custom HTTP headers and methods, the request path, the query string, and the source IP address. This new feature simplifies the application architecture by eliminating the need for a proxy fleet for routing, blocks unwanted traffic at the load balancer, and enables the implementation of A/B testing.
 
 ## Overview
-The ALB is configured to split traffic using annotations on the ingress resrouces. More specifically, the [ingress annotation](../../../guide/ingress/annotations.md#actions) `alb.ingress.kubernetes.io/actions.${service-name}` configures custom actions on the listener. 
+The ALB is configured to split traffic using annotations on the ingress resources. More specifically, the [ingress annotation](../../../guide/ingress/annotations.md#actions) `alb.ingress.kubernetes.io/actions.${service-name}` configures custom actions on the listener. 
 
 The body of the annotation is a JSON document that identifies an action type, and configures it. The supported [actions](https://docs.aws.amazon.com/elasticloadbalancing/latest/application/load-balancer-listeners.html#rule-action-types) are `redirect`, `forward`, and `fixed-response`. 
 

docs/guide/use_cases/multi_cluster/index.md

@@ -0,0 +1,121 @@
+# MultiCluster Target Groups
+
+The load balancer controller assumes full control over the configured target groups. When a target group is registered with the controller it de registers any targets not currently in the cluster. Target groups that have MultiCluster support enabled can be associated to multiple Kubernetes clusters or support arbitrary targets from other sources.
+
+
+## Overview
+
+When enabled, MultiCluster mode supports multiple methods, and every cluster associated with a target group has one of these methods. It's recommended to use new resources when configuring MutliCluster mode. There is a period of time when MultiCluster must take a snapshot of the cluster state in order to support the selected mode. This data is stored into ConfigMap, which resides in the same namespace as your load balancer resources. ConfigMap stores snapshots of managed targets at `aws-lbc-targets-$TARGET_GROUP_BINDING_NAME`
+
+When using an ALB, you must specify this annotation in the ingress or service:
+
+`alb.ingress.kubernetes.io/multi-cluster-target-group: "true"`
+
+When using an NLB, you specify this annotation in your service:
+
+`service.beta.kubernetes.io/aws-load-balancer-multi-cluster-target-group: "true"`
+
+When using any out-of-band TargetGroupBindings, you must specify this field in the spec:
+
+`multiClusterTargetGroup: true`
+
+
+### Example
+
+We will be setting up an echoserver in two clusters in order to demonstrate MultiCluster mode. See the full echoserver example in the 'Examples' tab.
+
+The following ingress configures the Target Group Binding as MultiCluster. We will take the created Target Group and share it in a second cluster.
+
+```
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: echoserver
+  namespace: echoserver
+  annotations:
+    alb.ingress.kubernetes.io/multi-cluster-target-group: "true"    
+    alb.ingress.kubernetes.io/scheme: internet-facing
+    alb.ingress.kubernetes.io/tags: Environment=dev,Team=test
+spec:
+  ingressClassName: alb
+  rules:
+    - http:
+        paths:
+          - path: /
+            pathType: Exact
+            backend:
+              service:
+                name: echoserver
+                port:
+                  number: 80
+```
+
+Verify that MultiCluster is enabled by verifying that the created Target Group Binding is marked as MultiCluster.
+
+```
+kubectl -n echoserver get targetgroupbinding k8s-echoserv-echoserv-cc0122e143 -o yaml
+apiVersion: elbv2.k8s.aws/v1beta1
+kind: TargetGroupBinding
+metadata:
+  annotations:
+    elbv2.k8s.aws/checkpoint: cKay81gadoTtBSg6uVVginqtmCVG-1ApTvYN4YLD37U/_4kBy3Yg64qrXzjvIb2LlC3O__ex1qjozynsqHXmPgo
+    elbv2.k8s.aws/checkpoint-timestamp: "1729021572"
+  creationTimestamp: "2024-10-15T19:46:06Z"
+  finalizers:
+  - elbv2.k8s.aws/resources
+  generation: 1
+  labels:
+    ingress.k8s.aws/stack-name: echoserver
+    ingress.k8s.aws/stack-namespace: echoserver
+  name: k8s-echoserv-echoserv-cc0122e143
+  namespace: echoserver
+  resourceVersion: "79121011"
+  uid: 9ceaa2ea-14bb-44a5-abb0-69c7d2aac52c
+spec:
+  ipAddressType: ipv4
+  multiClusterTargetGroup: true <<< HERE
+  networking:
+    ingress:
+    - from:
+      - securityGroup:
+          groupID: sg-06a2bd7d790ac1d2e
+      ports:
+      - port: 32197
+        protocol: TCP
+  serviceRef:
+    name: echoserver
+    port: 80
+  targetGroupARN: arn:aws:elasticloadbalancing:us-east-1:565768096483:targetgroup/k8s-echoserv-echoserv-cc0122e143/6816b87346280ee7
+  targetType: instance
+  vpcID: vpc-0a7ef5bd8943067a8
+```
+
+In another cluster, you can now register that Target Group ARN in a Target Group Binding.
+
+```
+apiVersion: elbv2.k8s.aws/v1beta1
+kind: TargetGroupBinding
+metadata:
+  name: MyTargetGroupBinding
+  namespace: echoserver
+spec:
+  serviceRef:
+    name: echoserver
+    port: 80
+  multiClusterTargetGroup: true
+  targetType: instance
+  ipAddressType: ipv4
+  networking:
+    ingress:
+    - from:
+      - securityGroup:
+          groupID: $SG_FROM_ABOVE
+      ports:
+      - port: 32197
+        protocol: TCP
+  targetGroupARN: $TG_FROM_ABOVE
+```
+
+The configured TargetGroup should have targets from both clusters available to service traffic.
+
+

helm/aws-load-balancer-controller/crds/crds.yaml

@@ -392,6 +392,10 @@ spec:
                 - name
                 - port
                 type: object
+              multiClusterTargetGroup:
+                description: multiClusterTargetGroup Denotes if the TargetGroup is shared
+                  among multiple clusters
+                type: boolean
               targetGroupARN:
                 description: targetGroupARN is the Amazon Resource Name (ARN) for
                   the TargetGroup.
@@ -616,6 +620,10 @@ spec:
                 - name
                 - port
                 type: object
+              multiClusterTargetGroup:
+                description: multiClusterTargetGroup Denotes if the TargetGroup is shared
+                  among multiple clusters
+                type: boolean
               targetGroupARN:
                 description: targetGroupARN is the Amazon Resource Name (ARN) for
                   the TargetGroup.

helm/aws-load-balancer-controller/templates/rbac.yaml

@@ -75,6 +75,9 @@ rules:
 - apiGroups: [""]
   resources: [nodes, namespaces, endpoints]
   verbs: [get, list, watch]
+- apiGroups: [""]
+  resources: [configmaps]
+  verbs: [get, delete, create, update]
 {{- if .Values.clusterSecretsPermissions.allowAllSecrets }}
 - apiGroups: [""]
   resources: [secrets]

main.go

@@ -111,8 +111,9 @@ func main() {
 	azInfoProvider := networking.NewDefaultAZInfoProvider(cloud.EC2(), ctrl.Log.WithName("az-info-provider"))
 	vpcInfoProvider := networking.NewDefaultVPCInfoProvider(cloud.EC2(), ctrl.Log.WithName("vpc-info-provider"))
 	subnetResolver := networking.NewDefaultSubnetsResolver(azInfoProvider, cloud.EC2(), cloud.VpcID(), controllerCFG.ClusterName, ctrl.Log.WithName("subnets-resolver"))
+	multiClusterManager := targetgroupbinding.NewMultiClusterManager(mgr.GetClient(), mgr.GetAPIReader(), ctrl.Log)
 	tgbResManager := targetgroupbinding.NewDefaultResourceManager(mgr.GetClient(), cloud.ELBV2(), cloud.EC2(),
-		podInfoRepo, sgManager, sgReconciler, vpcInfoProvider,
+		podInfoRepo, sgManager, sgReconciler, vpcInfoProvider, multiClusterManager,
 		cloud.VpcID(), controllerCFG.ClusterName, controllerCFG.FeatureGates.Enabled(config.EndpointsFailOpen), controllerCFG.EnableEndpointSlices, controllerCFG.DisableRestrictedSGRules,
 		controllerCFG.ServiceTargetENISGTags, mgr.GetEventRecorderFor("targetGroupBinding"), ctrl.Log)
 	backendSGProvider := networking.NewBackendSGProvider(controllerCFG.ClusterName, controllerCFG.BackendSecurityGroup,

mkdocs.yml

@@ -36,6 +36,7 @@ nav:
         - Externally Managed Load Balancer: guide/use_cases/self_managed_lb/index.md
         - Frontend Security Groups: guide/use_cases/frontend_sg/index.md
         - Blue/Green: guide/use_cases/blue_green/index.md
+        - MultiCluster Target Groups: guide/use_cases/multi_cluster/index.md
   - Examples:
     - EchoServer: examples/echo_server.md
     - gRPCServer: examples/grpc_server.md

pkg/algorithm/maps.go

@@ -1,5 +1,10 @@
 package algorithm
 
+import (
+	"k8s.io/apimachinery/pkg/util/sets"
+	"strings"
+)
+
 // MapFindFirst get from list of maps until first found.
 func MapFindFirst(key string, maps ...map[string]string) (string, bool) {
 	for _, m := range maps {
@@ -47,3 +52,25 @@ func DiffStringMap(desired map[string]string, current map[string]string) (map[st
 
 	return modify, remove
 }
+
+func CSVToStringSet(csv string) sets.Set[string] {
+	s := sets.Set[string]{}
+
+	if len(csv) == 0 {
+		return s
+	}
+
+	for _, v := range strings.Split(csv, ",") {
+		s.Insert(v)
+	}
+
+	return s
+}
+
+func StringSetToCSV(s sets.Set[string]) string {
+	keyList := make([]string, 0, len(s))
+	for k := range s {
+		keyList = append(keyList, k)
+	}
+	return strings.Join(keyList, ",")
+}