diff --git a/.gitignore b/.gitignore index 0f2180c1af..d9917959af 100644 --- a/.gitignore +++ b/.gitignore @@ -24,3 +24,4 @@ site *.swo *~ *.bak +scripts/aws_sdk_model_override/* diff --git a/apis/elbv2/v1alpha1/targetgroupbinding_types.go b/apis/elbv2/v1alpha1/targetgroupbinding_types.go index 78c4f5b8ea..4af605afa7 100644 --- a/apis/elbv2/v1alpha1/targetgroupbinding_types.go +++ b/apis/elbv2/v1alpha1/targetgroupbinding_types.go @@ -107,7 +107,12 @@ type TargetGroupBindingNetworking struct { // TargetGroupBindingSpec defines the desired state of TargetGroupBinding type TargetGroupBindingSpec struct { // targetGroupARN is the Amazon Resource Name (ARN) for the TargetGroup. - TargetGroupARN string `json:"targetGroupARN"` + // +optional + TargetGroupARN string `json:"targetGroupARN,omitempty"` + + // targetGroupName is the Name of the TargetGroup. + // +optional + TargetGroupName string `json:"targetGroupName,omitempty"` // MultiClusterTargetGroup Denotes if the TargetGroup is shared among multiple clusters // +optional @@ -138,6 +143,7 @@ type TargetGroupBindingStatus struct { // +kubebuilder:printcolumn:name="SERVICE-PORT",type="string",JSONPath=".spec.serviceRef.port",description="The Kubernetes Service's port" // +kubebuilder:printcolumn:name="TARGET-TYPE",type="string",JSONPath=".spec.targetType",description="The AWS TargetGroup's TargetType" // +kubebuilder:printcolumn:name="ARN",type="string",JSONPath=".spec.targetGroupARN",description="The AWS TargetGroup's Amazon Resource Name",priority=1 +// +kubebuilder:printcolumn:name="NAME",type="string",JSONPath=".spec.targetGroupName",description="The AWS TargetGroup's Name",priority=2 // +kubebuilder:printcolumn:name="AGE",type="date",JSONPath=".metadata.creationTimestamp" // TargetGroupBinding is the Schema for the TargetGroupBinding API type TargetGroupBinding struct { diff --git a/apis/elbv2/v1beta1/ingressclassparams_types.go b/apis/elbv2/v1beta1/ingressclassparams_types.go index 23479ff8f1..4fc9216946 100644 --- a/apis/elbv2/v1beta1/ingressclassparams_types.go +++ b/apis/elbv2/v1beta1/ingressclassparams_types.go @@ -101,6 +101,12 @@ type Listener struct { ListenerAttributes []Attribute `json:"listenerAttributes,omitempty"` } +// Information about a load balancer capacity reservation. +type MinimumLoadBalancerCapacity struct { + // The Capacity Units Value. + CapacityUnits int32 `json:"capacityUnits"` +} + // IngressClassParamsSpec defines the desired state of IngressClassParams type IngressClassParamsSpec struct { // CertificateArn specifies the ARN of the certificates for all Ingresses that belong to IngressClass with this IngressClassParams. @@ -146,6 +152,10 @@ type IngressClassParamsSpec struct { // Listeners define a list of listeners with their protocol, port and attributes. // +optional Listeners []Listener `json:"listeners,omitempty"` + + // MinimumLoadBalancerCapacity define the capacity reservation for LoadBalancers for all Ingress that belong to IngressClass with this IngressClassParams. + // +optional + MinimumLoadBalancerCapacity *MinimumLoadBalancerCapacity `json:"minimumLoadBalancerCapacity,omitempty"` } // +kubebuilder:object:root=true diff --git a/apis/elbv2/v1beta1/targetgroupbinding_types.go b/apis/elbv2/v1beta1/targetgroupbinding_types.go index 437f2d3a62..337e892024 100644 --- a/apis/elbv2/v1beta1/targetgroupbinding_types.go +++ b/apis/elbv2/v1beta1/targetgroupbinding_types.go @@ -124,8 +124,12 @@ type TargetGroupBindingNetworking struct { // TargetGroupBindingSpec defines the desired state of TargetGroupBinding type TargetGroupBindingSpec struct { // targetGroupARN is the Amazon Resource Name (ARN) for the TargetGroup. - // +kubebuilder:validation:MinLength=1 - TargetGroupARN string `json:"targetGroupARN"` + // +optional + TargetGroupARN string `json:"targetGroupARN,omitempty"` + + // targetGroupName is the Name of the TargetGroup. + // +optional + TargetGroupName string `json:"targetGroupName,omitempty"` // MultiClusterTargetGroup Denotes if the TargetGroup is shared among multiple clusters // +optional @@ -169,6 +173,7 @@ type TargetGroupBindingStatus struct { // +kubebuilder:printcolumn:name="SERVICE-PORT",type="string",JSONPath=".spec.serviceRef.port",description="The Kubernetes Service's port" // +kubebuilder:printcolumn:name="TARGET-TYPE",type="string",JSONPath=".spec.targetType",description="The AWS TargetGroup's TargetType" // +kubebuilder:printcolumn:name="ARN",type="string",JSONPath=".spec.targetGroupARN",description="The AWS TargetGroup's Amazon Resource Name",priority=1 +// +kubebuilder:printcolumn:name="NAME",type="string",JSONPath=".spec.targetGroupName",description="The AWS TargetGroup's Name",priority=2 // +kubebuilder:printcolumn:name="AGE",type="date",JSONPath=".metadata.creationTimestamp" // TargetGroupBinding is the Schema for the TargetGroupBinding API type TargetGroupBinding struct { diff --git a/apis/elbv2/v1beta1/zz_generated.deepcopy.go b/apis/elbv2/v1beta1/zz_generated.deepcopy.go index d4630526ab..108289a522 100644 --- a/apis/elbv2/v1beta1/zz_generated.deepcopy.go +++ b/apis/elbv2/v1beta1/zz_generated.deepcopy.go @@ -169,6 +169,11 @@ func (in *IngressClassParamsSpec) DeepCopyInto(out *IngressClassParamsSpec) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.MinimumLoadBalancerCapacity != nil { + in, out := &in.MinimumLoadBalancerCapacity, &out.MinimumLoadBalancerCapacity + *out = new(MinimumLoadBalancerCapacity) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new IngressClassParamsSpec. @@ -216,6 +221,21 @@ func (in *Listener) DeepCopy() *Listener { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MinimumLoadBalancerCapacity) DeepCopyInto(out *MinimumLoadBalancerCapacity) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MinimumLoadBalancerCapacity. +func (in *MinimumLoadBalancerCapacity) DeepCopy() *MinimumLoadBalancerCapacity { + if in == nil { + return nil + } + out := new(MinimumLoadBalancerCapacity) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *NetworkingIngressRule) DeepCopyInto(out *NetworkingIngressRule) { *out = *in diff --git a/config/crd/bases/elbv2.k8s.aws_ingressclassparams.yaml b/config/crd/bases/elbv2.k8s.aws_ingressclassparams.yaml index 26b39f0e8c..7f65f28ad7 100644 --- a/config/crd/bases/elbv2.k8s.aws_ingressclassparams.yaml +++ b/config/crd/bases/elbv2.k8s.aws_ingressclassparams.yaml @@ -133,6 +133,18 @@ spec: - value type: object type: array + minimumLoadBalancerCapacity: + description: MinimumLoadBalancerCapacity define the capacity reservation + for LoadBalancers for all Ingress that belong to IngressClass with + this IngressClassParams. + properties: + capacityUnits: + description: The Capacity Units Value. + format: int32 + type: integer + required: + - capacityUnits + type: object namespaceSelector: description: |- NamespaceSelector restrict the namespaces of Ingresses that are allowed to specify the IngressClass with this IngressClassParams. diff --git a/config/crd/bases/elbv2.k8s.aws_targetgroupbindings.yaml b/config/crd/bases/elbv2.k8s.aws_targetgroupbindings.yaml index 8f2dd41d00..f9af43e6ff 100644 --- a/config/crd/bases/elbv2.k8s.aws_targetgroupbindings.yaml +++ b/config/crd/bases/elbv2.k8s.aws_targetgroupbindings.yaml @@ -32,6 +32,11 @@ spec: name: ARN priority: 1 type: string + - description: The AWS TargetGroup's Name + jsonPath: .spec.targetGroupName + name: NAME + priority: 2 + type: string - jsonPath: .metadata.creationTimestamp name: AGE type: date @@ -160,6 +165,9 @@ spec: description: targetGroupARN is the Amazon Resource Name (ARN) for the TargetGroup. type: string + targetGroupName: + description: targetGroupName is the Name of the TargetGroup. + type: string targetType: description: targetType is the TargetType of TargetGroup. If unspecified, it will be automatically inferred. @@ -169,7 +177,6 @@ spec: type: string required: - serviceRef - - targetGroupARN type: object status: description: TargetGroupBindingStatus defines the observed state of TargetGroupBinding @@ -202,6 +209,11 @@ spec: name: ARN priority: 1 type: string + - description: The AWS TargetGroup's Name + jsonPath: .spec.targetGroupName + name: NAME + priority: 2 + type: string - jsonPath: .metadata.creationTimestamp name: AGE type: date @@ -387,7 +399,9 @@ spec: targetGroupARN: description: targetGroupARN is the Amazon Resource Name (ARN) for the TargetGroup. - minLength: 1 + type: string + targetGroupName: + description: targetGroupName is the Name of the TargetGroup. type: string targetType: description: targetType is the TargetType of TargetGroup. If unspecified, @@ -402,7 +416,6 @@ spec: type: string required: - serviceRef - - targetGroupARN type: object status: description: TargetGroupBindingStatus defines the observed state of TargetGroupBinding diff --git a/controllers/ingress/group_controller.go b/controllers/ingress/group_controller.go index 175bbb6906..185faa1246 100644 --- a/controllers/ingress/group_controller.go +++ b/controllers/ingress/group_controller.go @@ -59,7 +59,7 @@ func NewGroupReconciler(cloud aws.Cloud, k8sClient client.Client, eventRecorder annotationParser, subnetsResolver, authConfigBuilder, enhancedBackendBuilder, trackingProvider, elbv2TaggingManager, controllerConfig.FeatureGates, cloud.VpcID(), controllerConfig.ClusterName, controllerConfig.DefaultTags, controllerConfig.ExternalManagedTags, - controllerConfig.DefaultSSLPolicy, controllerConfig.DefaultTargetType, backendSGProvider, sgResolver, + controllerConfig.DefaultSSLPolicy, controllerConfig.DefaultTargetType, controllerConfig.DefaultLoadBalancerScheme, backendSGProvider, sgResolver, controllerConfig.EnableBackendSecurityGroup, controllerConfig.DisableRestrictedSGRules, controllerConfig.IngressConfig.AllowedCertificateAuthorityARNs, controllerConfig.FeatureGates.Enabled(config.EnableIPTargetType), logger) stackMarshaller := deploy.NewDefaultStackMarshaller() stackDeployer := deploy.NewDefaultStackDeployer(cloud, k8sClient, networkingSGManager, networkingSGReconciler, elbv2TaggingManager, @@ -170,6 +170,10 @@ func (r *groupReconciler) buildAndDeployModel(ctx context.Context, ingGroup ingr r.logger.Info("successfully built model", "model", stackJSON) if err := r.stackDeployer.Deploy(ctx, stack); err != nil { + var requeueNeededAfter *runtime.RequeueNeededAfter + if errors.As(err, &requeueNeededAfter) { + return nil, nil, err + } r.recordIngressGroupEvent(ctx, ingGroup, corev1.EventTypeWarning, k8s.IngressEventReasonFailedDeployModel, fmt.Sprintf("Failed deploy model due to %v", err)) return nil, nil, err } diff --git a/controllers/service/service_controller.go b/controllers/service/service_controller.go index 2ed7612b01..18dd36de3d 100644 --- a/controllers/service/service_controller.go +++ b/controllers/service/service_controller.go @@ -45,7 +45,7 @@ func NewServiceReconciler(cloud aws.Cloud, k8sClient client.Client, eventRecorde serviceUtils := service.NewServiceUtils(annotationParser, serviceFinalizer, controllerConfig.ServiceConfig.LoadBalancerClass, controllerConfig.FeatureGates) modelBuilder := service.NewDefaultModelBuilder(annotationParser, subnetsResolver, vpcInfoProvider, cloud.VpcID(), trackingProvider, elbv2TaggingManager, cloud.EC2(), controllerConfig.FeatureGates, controllerConfig.ClusterName, controllerConfig.DefaultTags, controllerConfig.ExternalManagedTags, - controllerConfig.DefaultSSLPolicy, controllerConfig.DefaultTargetType, controllerConfig.FeatureGates.Enabled(config.EnableIPTargetType), serviceUtils, + controllerConfig.DefaultSSLPolicy, controllerConfig.DefaultTargetType, controllerConfig.DefaultLoadBalancerScheme, controllerConfig.FeatureGates.Enabled(config.EnableIPTargetType), serviceUtils, backendSGProvider, sgResolver, controllerConfig.EnableBackendSecurityGroup, controllerConfig.DisableRestrictedSGRules, logger) stackMarshaller := deploy.NewDefaultStackMarshaller() stackDeployer := deploy.NewDefaultStackDeployer(cloud, k8sClient, networkingSGManager, networkingSGReconciler, elbv2TaggingManager, controllerConfig, serviceTagPrefix, logger) @@ -124,6 +124,10 @@ func (r *serviceReconciler) buildModel(ctx context.Context, svc *corev1.Service) func (r *serviceReconciler) deployModel(ctx context.Context, svc *corev1.Service, stack core.Stack) error { if err := r.stackDeployer.Deploy(ctx, stack); err != nil { + var requeueNeededAfter *runtime.RequeueNeededAfter + if errors.As(err, &requeueNeededAfter) { + return err + } r.eventRecorder.Event(svc, corev1.EventTypeWarning, k8s.ServiceEventReasonFailedDeployModel, fmt.Sprintf("Failed deploy model due to %v", err)) return err } diff --git a/docs/deploy/configurations.md b/docs/deploy/configurations.md index 44751238bc..7c63741930 100644 --- a/docs/deploy/configurations.md +++ b/docs/deploy/configurations.md @@ -79,6 +79,7 @@ Currently, you can set only 1 namespace to watch in this flag. See [this Kuberne | default-ssl-policy | string | ELBSecurityPolicy-2016-08 | Default SSL Policy that will be applied to all Ingresses or Services that do not have the SSL Policy annotation | | default-tags | stringMap | | AWS Tags that will be applied to all AWS resources managed by this controller. Specified Tags takes highest priority | | default-target-type | string | instance | Default target type for Ingresses and Services - ip, instance | +| default-load-balancer-scheme | string | internal | Default scheme for ELBs - internal, internet-facing | | [disable-ingress-class-annotation](#disable-ingress-class-annotation) | boolean | false | Disable new usage of the `kubernetes.io/ingress.class` annotation | | [disable-ingress-group-name-annotation](#disable-ingress-group-name-annotation) | boolean | false | Disallow new use of the `alb.ingress.kubernetes.io/group.name` annotation | | disable-restricted-sg-rules | boolean | false | Disable the usage of restricted security group rules | @@ -104,6 +105,7 @@ Currently, you can set only 1 namespace to watch in this flag. See [this Kuberne | [sync-period](#sync-period) | duration | 10h0m0s | Period at which the controller forces the repopulation of its local object stores | | targetgroupbinding-max-concurrent-reconciles | int | 3 | Maximum number of concurrently running reconcile loops for targetGroupBinding | | targetgroupbinding-max-exponential-backoff-delay | duration | 16m40s | Maximum duration of exponential backoff for targetGroupBinding reconcile failures | +| [lb-stabilization-monitor-interval](#lb-stabilization-monitor-interval) | duration | 2m | Interval at which the controller monitors the state of load balancer after creation | tolerate-non-existent-backend-service | boolean | true | Whether to allow rules which refer to backend services that do not exist (When enabled, it will return 503 error if backend service not exist) | | tolerate-non-existent-backend-action | boolean | true | Whether to allow rules which refer to backend actions that do not exist (When enabled, it will return 503 error if backend action not exist) | | watch-namespace | string | | Namespace the controller watches for updates to Kubernetes objects, If empty, all namespaces are watched. | @@ -137,6 +139,9 @@ Once disabled: As best practice, we do not recommend users to manually modify the resources managed by the controller. And users should not depend on the controller auto-reconciliation to revert the manual modification, or to mitigate any security risks. +### lb-stabilization-monitor-interval +`--lb-stabilization-monitor-interval` defines a fixed interval for the controller to monitor the state of load balancer after the creation for stabilization, default to 2m. It monitors the load balancer state so that once it becomes active it can make the required updates like capacity reservation for the active load balancer. It calls DescribeLoadBalancer API at a fixed interval to monitor the state. Please be mindful that lower value will result into frequent calls which may incur unnecessary AWS API usage. + ### waf-addons By default, the controller assumes sole ownership of the WAF addons associated to the provisioned ALBs, via the flag `--enable-waf` and `--enable-wafv2`. And the users should disable them accordingly if they want a third party like AWS Firewall Manager to associate or remove the WAF-ACL of the ALBs. @@ -177,3 +182,4 @@ There are a set of key=value pairs that describe AWS load balancer controller fe | NLBHealthCheckAdvancedConfiguration | string | true | Enable or disable advanced health check configuration for NLB, for example health check timeout | | ALBSingleSubnet | string | false | If enabled, controller will allow using only 1 subnet for provisioning ALB, which need to get whitelisted by ELB in advance | | NLBSecurityGroup | string | true | Enable or disable all NLB security groups actions including frontend sg creation, backend sg creation, and backend sg modifications | +| LBCapacityReservation | string | true | Enable or disable the capacity reservation feature on ALB and NLB \ No newline at end of file diff --git a/docs/guide/ingress/annotations.md b/docs/guide/ingress/annotations.md index 2dc2947ede..7035dc80f8 100644 --- a/docs/guide/ingress/annotations.md +++ b/docs/guide/ingress/annotations.md @@ -61,6 +61,7 @@ You can add annotations to kubernetes Ingress and Service objects to customize t | [alb.ingress.kubernetes.io/mutual-authentication](#mutual-authentication) | json |N/A| Ingress | Exclusive | | [alb.ingress.kubernetes.io/multi-cluster-target-group](#multi-cluster-target-group) | boolean |N/A| Ingress, Service | N/A | | [alb.ingress.kubernetes.io/listener-attributes.${Protocol}-${Port}](#listener-attributes) | stringMap |N/A| Ingress |Merge| +| [alb.ingress.kubernetes.io/minimum-load-balancer-capacity](#load-balancer-capacity-reservation) | stringMap |N/A| Ingress | Exclusive | ## IngressGroup IngressGroup feature enables you to group multiple Ingress resources together. @@ -931,6 +932,26 @@ In addition, you can use annotations to specify additional tags alb.ingress.kubernetes.io/tags: Environment=dev,Team=test ``` +## Capacity Unit Reservation +Load balancer capacity unit reservation can be configured via following annotations: + +- `alb.ingress.kubernetes.io/minimum-load-balancer-capacity` specifies the + [Capacity Unit Reservation](https://docs.aws.amazon.com/elasticloadbalancing/latest/application/capacity-unit-reservation.html) to be configured. + + !!!example + - set the capacity unit reservation to 1000 + ``` + alb.ingress.kubernetes.io/minimum-load-balancer-capacity: CapacityUnits=1000 + ``` + - reset the capacity unit reservation + ``` + alb.ingress.kubernetes.io/minimum-load-balancer-capacity: CapacityUnits=0 + ``` + + !!!note "Notes" + - If you specify this annotation, but remove it later, the capacity unit reservation is not reset. You need to reset the capacity by setting the capacity units to zero as show in the example above. + - If users do not want the controller to manage the capacity unit reservation on load balancer, they can disable the feature by setting controller command line feature gate flag ```--feature-gates=LBCapacityReservation=true``` + ## Addons - `alb.ingress.kubernetes.io/waf-acl-id` specifies the identifier for the Amazon WAF Classic web ACL. diff --git a/docs/guide/ingress/ingress_class.md b/docs/guide/ingress/ingress_class.md index 3322cc7cfb..fd5d6ee954 100644 --- a/docs/guide/ingress/ingress_class.md +++ b/docs/guide/ingress/ingress_class.md @@ -140,6 +140,16 @@ You can use IngressClassParams to enforce settings for a set of Ingresses. spec: certificateArn: ['arn:aws:acm:us-east-1:123456789:certificate/test-arn-1','arn:aws:acm:us-east-1:123456789:certificate/test-arn-2'] ``` + - with minimumLoadBalancerCapacity.capacityUnits + ``` + apiVersion: elbv2.k8s.aws/v1beta1 + kind: IngressClassParams + metadata: + name: class2048-config + spec: + minimumLoadBalancerCapacity: + capacityUnits: 1000 + ``` ### IngressClassParams specification @@ -233,3 +243,12 @@ Cluster administrators can use `loadBalancerAttributes` field to specify the [Lo 1. If `loadBalancerAttributes` is set, the attributes defined will be applied to the load balancer that belong to this IngressClass. If you specify invalid keys or values for the load balancer attributes, the controller will fail to reconcile ingresses belonging to the particular ingress class. 2. If `loadBalancerAttributes` un-specified, Ingresses with this IngressClass can continue to use `alb.ingress.kubernetes.io/load-balancer-attributes` annotation to specify the load balancer attributes. + +#### spec.minimumLoadBalancerCapacity + +Cluster administrators can use the optional `minimumLoadBalancerCapacity` field to specify the capacity reservation for the load balancers that belong to this IngressClass. +They may specify `capacityUnits`. If the field is specified, LBC will ignore the `alb.ingress.kubernetes.io/minimum-load-balancer-capacity annotation` annotation. + +##### spec.minimumLoadBalancerCapacity.capacityUnits + +If `capacityUnits` is specified, it must be to valid positive value greater than 0. If set to 0, the LBC will reset the capacity reservation for the load balancer. \ No newline at end of file diff --git a/docs/guide/service/annotations.md b/docs/guide/service/annotations.md index e9c494288c..a27c8c7813 100644 --- a/docs/guide/service/annotations.md +++ b/docs/guide/service/annotations.md @@ -56,6 +56,7 @@ | [service.beta.kubernetes.io/aws-load-balancer-multi-cluster-target-group](#multi-cluster-target-group) | boolean | false | If specified, the controller will only operate on targets that exist within the cluster, ignoring targets from other sources. | | [service.beta.kubernetes.io/aws-load-balancer-enable-prefix-for-ipv6-source-nat](#enable-prefix-for-ipv6-source-nat) | string | off | Optional annotation. dualstack lb only. Allowed values - on and off | | [service.beta.kubernetes.io/aws-load-balancer-source-nat-ipv6-prefixes](#source-nat-ipv6-prefixes) | stringList | | Optional annotation. dualstack lb only. This annotation is only applicable when user has to set the service.beta.kubernetes.io/aws-load-balancer-enable-prefix-for-ipv6-source-nat to "on". Length must match the number of subnets | +| [service.beta.kubernetes.io/aws-load-balancer-minimum-load-balancer-capacity](#load-balancer-capacity-reservation) | stringMap | | ## Traffic Routing Traffic Routing can be controlled with following annotations: @@ -579,6 +580,25 @@ Load balancer access can be controlled via following annotations: service.beta.kubernetes.io/aws-load-balancer-inbound-sg-rules-on-private-link-traffic: "off" ``` +## Capacity Unit Reservation +Load balancer capacity unit reservation can be configured via following annotations: + +- `service.beta.kubernetes.io/aws-load-balancer-minimum-load-balancer-capacity` specifies the + [Capacity Unit Reservation](https://docs.aws.amazon.com/elasticloadbalancing/latest/network/capacity-unit-reservation.html) to be configured. + + !!!example + - set the capacity unit reservation to 1000 + ``` + service.beta.kubernetes.io/aws-load-balancer-minimum-load-balancer-capacity: CapacityUnits=3000 + ``` + - reset the capacity unit reservation + ``` + service.beta.kubernetes.io/aws-load-balancer-minimum-load-balancer-capacity: CapacityUnits=0 + ``` + + !!!note "Notes" + - If you specify this annotation, but remove it later, the capacity unit reservation is not reset. You need to reset the capacity by setting the capacity units to zero as show in the example above. + - If users do not want the controller to manage the capacity unit reservation on load balancer, they can disable the feature by setting controller command line feature gate flag ```--feature-gates=LBCapacityReservation=true``` ## Legacy Cloud Provider The AWS Load Balancer Controller manages Kubernetes Services in a compatible way with the AWS cloud provider's legacy service controller. diff --git a/docs/guide/targetgroupbinding/targetgroupbinding.md b/docs/guide/targetgroupbinding/targetgroupbinding.md index 8da8ff89ee..cec6b6f028 100644 --- a/docs/guide/targetgroupbinding/targetgroupbinding.md +++ b/docs/guide/targetgroupbinding/targetgroupbinding.md @@ -16,8 +16,25 @@ TargetGroupBinding CR supports TargetGroups of either `instance` or `ip` TargetT !!!tip "" If TargetType is not explicitly specified, a mutating webhook will automatically call AWS API to find the TargetType for your TargetGroup and set it to correct value. +## Choosing the Target Group +One can either use ``targetGroupARN`` of ``targetGroupName`` to identify a Target Group. Although both are unique and immutable in an AWS region, one only has control of the ``targetGroupName``, for ``targetGroupARN`` is generated by AWS and contain random characters. + +If you provide both ``targetGroupARN`` and ``targetGroupName``, beware that ``targetGroupARN`` prevails. + + +## Sample YAMLs +```yaml +apiVersion: elbv2.k8s.aws/v1beta1 +kind: TargetGroupBinding +metadata: + name: my-tgb +spec: + serviceRef: + name: awesome-service # route traffic to the awesome-service + port: 80 + targetGroupName: +``` -## Sample YAML ```yaml apiVersion: elbv2.k8s.aws/v1beta1 kind: TargetGroupBinding diff --git a/docs/install/iam_policy.json b/docs/install/iam_policy.json index 603c5b280b..1a5b4d614b 100644 --- a/docs/install/iam_policy.json +++ b/docs/install/iam_policy.json @@ -41,7 +41,8 @@ "elasticloadbalancing:DescribeTargetHealth", "elasticloadbalancing:DescribeTags", "elasticloadbalancing:DescribeTrustStores", - "elasticloadbalancing:DescribeListenerAttributes" + "elasticloadbalancing:DescribeListenerAttributes", + "elasticloadbalancing:DescribeCapacityReservation" ], "Resource": "*" }, @@ -191,7 +192,8 @@ "elasticloadbalancing:ModifyTargetGroup", "elasticloadbalancing:ModifyTargetGroupAttributes", "elasticloadbalancing:DeleteTargetGroup", - "elasticloadbalancing:ModifyListenerAttributes" + "elasticloadbalancing:ModifyListenerAttributes", + "elasticloadbalancing:ModifyCapacityReservation" ], "Resource": "*", "Condition": { diff --git a/docs/install/iam_policy_cn.json b/docs/install/iam_policy_cn.json index ae1ad1793f..ba8a39fa79 100644 --- a/docs/install/iam_policy_cn.json +++ b/docs/install/iam_policy_cn.json @@ -41,7 +41,8 @@ "elasticloadbalancing:DescribeTargetHealth", "elasticloadbalancing:DescribeTags", "elasticloadbalancing:DescribeTrustStores", - "elasticloadbalancing:DescribeListenerAttributes" + "elasticloadbalancing:DescribeListenerAttributes", + "elasticloadbalancing:DescribeCapacityReservation" ], "Resource": "*" }, @@ -213,7 +214,8 @@ "elasticloadbalancing:ModifyTargetGroup", "elasticloadbalancing:ModifyTargetGroupAttributes", "elasticloadbalancing:DeleteTargetGroup", - "elasticloadbalancing:ModifyListenerAttributes" + "elasticloadbalancing:ModifyListenerAttributes", + "elasticloadbalancing:ModifyCapacityReservation" ], "Resource": "*", "Condition": { diff --git a/docs/install/iam_policy_us-gov.json b/docs/install/iam_policy_us-gov.json index f2d3d1d293..828f77f4d8 100644 --- a/docs/install/iam_policy_us-gov.json +++ b/docs/install/iam_policy_us-gov.json @@ -41,7 +41,8 @@ "elasticloadbalancing:DescribeTargetHealth", "elasticloadbalancing:DescribeTags", "elasticloadbalancing:DescribeTrustStores", - "elasticloadbalancing:DescribeListenerAttributes" + "elasticloadbalancing:DescribeListenerAttributes", + "elasticloadbalancing:DescribeCapacityReservation" ], "Resource": "*" }, @@ -213,7 +214,8 @@ "elasticloadbalancing:ModifyTargetGroup", "elasticloadbalancing:ModifyTargetGroupAttributes", "elasticloadbalancing:DeleteTargetGroup", - "elasticloadbalancing:ModifyListenerAttributes" + "elasticloadbalancing:ModifyListenerAttributes", + "elasticloadbalancing:ModifyCapacityReservation" ], "Resource": "*", "Condition": { diff --git a/go.mod b/go.mod index a042291590..1e05dbf7d8 100644 --- a/go.mod +++ b/go.mod @@ -4,19 +4,19 @@ go 1.22.8 require ( github.com/aws/aws-sdk-go v1.55.5 - github.com/aws/aws-sdk-go-v2 v1.32.3 + github.com/aws/aws-sdk-go-v2 v1.32.5 github.com/aws/aws-sdk-go-v2/config v1.27.27 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.11 github.com/aws/aws-sdk-go-v2/service/acm v1.28.4 github.com/aws/aws-sdk-go-v2/service/appmesh v1.27.7 github.com/aws/aws-sdk-go-v2/service/ec2 v1.173.0 - github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2 v1.41.0 + github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2 v1.42.0 github.com/aws/aws-sdk-go-v2/service/resourcegroupstaggingapi v1.23.3 github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.31.7 github.com/aws/aws-sdk-go-v2/service/shield v1.27.3 github.com/aws/aws-sdk-go-v2/service/wafregional v1.23.3 github.com/aws/aws-sdk-go-v2/service/wafv2 v1.51.4 - github.com/aws/smithy-go v1.22.0 + github.com/aws/smithy-go v1.22.1 github.com/evanphx/json-patch v5.7.0+incompatible github.com/gavv/httpexpect/v2 v2.9.0 github.com/go-logr/logr v1.4.1 @@ -57,8 +57,8 @@ require ( github.com/andybalholm/brotli v1.0.4 // indirect github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535 // indirect github.com/aws/aws-sdk-go-v2/credentials v1.17.27 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.22 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.22 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.24 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.24 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.3 // indirect github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.17 // indirect diff --git a/go.sum b/go.sum index c8386e14df..b13e212624 100644 --- a/go.sum +++ b/go.sum @@ -40,6 +40,8 @@ github.com/aws/aws-sdk-go v1.55.5 h1:KKUZBfBoyqy5d3swXyiC7Q76ic40rYcbqH7qjh59kzU github.com/aws/aws-sdk-go v1.55.5/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU= github.com/aws/aws-sdk-go-v2 v1.32.3 h1:T0dRlFBKcdaUPGNtkBSwHZxrtis8CQU17UpNBZYd0wk= github.com/aws/aws-sdk-go-v2 v1.32.3/go.mod h1:2SK5n0a2karNTv5tbP1SjsX0uhttou00v/HpXKM1ZUo= +github.com/aws/aws-sdk-go-v2 v1.32.5 h1:U8vdWJuY7ruAkzaOdD7guwJjD06YSKmnKCJs7s3IkIo= +github.com/aws/aws-sdk-go-v2 v1.32.5/go.mod h1:P5WJBrYqqbWVaOxgH0X/FYYD47/nooaPOZPlQdmiN2U= github.com/aws/aws-sdk-go-v2/config v1.27.27 h1:HdqgGt1OAP0HkEDDShEl0oSYa9ZZBSOmKpdpsDMdO90= github.com/aws/aws-sdk-go-v2/config v1.27.27/go.mod h1:MVYamCg76dFNINkZFu4n4RjDixhVr51HLj4ErWzrVwg= github.com/aws/aws-sdk-go-v2/credentials v1.17.27 h1:2raNba6gr2IfA0eqqiP2XiQ0UVOpGPgDSi0I9iAP+UI= @@ -48,8 +50,12 @@ github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.11 h1:KreluoV8FZDEtI6Co2xuNk github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.11/go.mod h1:SeSUYBLsMYFoRvHE0Tjvn7kbxaUhl75CJi1sbfhMxkU= github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.22 h1:Jw50LwEkVjuVzE1NzkhNKkBf9cRN7MtE1F/b2cOKTUM= github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.22/go.mod h1:Y/SmAyPcOTmpeVaWSzSKiILfXTVJwrGmYZhcRbhWuEY= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.24 h1:4usbeaes3yJnCFC7kfeyhkdkPtoRYPa/hTmCqMpKpLI= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.24/go.mod h1:5CI1JemjVwde8m2WG3cz23qHKPOxbpkq0HaoreEgLIY= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.22 h1:981MHwBaRZM7+9QSR6XamDzF/o7ouUGxFzr+nVSIhrs= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.22/go.mod h1:1RA1+aBEfn+CAB/Mh0MB6LsdCYCnjZm7tKXtnk499ZQ= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.24 h1:N1zsICrQglfzaBnrfM0Ys00860C+QFwu6u/5+LomP+o= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.24/go.mod h1:dCn9HbJ8+K31i8IQ8EWmWj0EiIk0+vKiHNMxTTYveAg= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 h1:hT8rVHwugYE2lEfdFE0QWVo81lF7jMrYJVDWI+f+VxU= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0/go.mod h1:8tu/lYfQfFe6IGnaOdrpVgEL2IrrDOf6/m9RQum4NkY= github.com/aws/aws-sdk-go-v2/service/acm v1.28.4 h1:wiW1Y6/1lysA0eJZRq0I53YYKuV9MNAzL15z2eZRlEE= @@ -58,8 +64,8 @@ github.com/aws/aws-sdk-go-v2/service/appmesh v1.27.7 h1:q44a6kysAfej9zZwRnraOg9s github.com/aws/aws-sdk-go-v2/service/appmesh v1.27.7/go.mod h1:ZYSmrgAMp0rTCHH+SGsoxZo+PPbgsDqBzewTp3tSJ60= github.com/aws/aws-sdk-go-v2/service/ec2 v1.173.0 h1:ta62lid9JkIpKZtZZXSj6rP2AqY5x1qYGq53ffxqD9Q= github.com/aws/aws-sdk-go-v2/service/ec2 v1.173.0/go.mod h1:o6QDjdVKpP5EF0dp/VlvqckzuSDATr1rLdHt3A5m0YY= -github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2 v1.41.0 h1:W+xNfPS8dQ8YoszdkHqTDYIgCrWJvyUU/ZgdJ0frCRE= -github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2 v1.41.0/go.mod h1:6WuvTcPjB9gff93p/2LNBg09d8xK99jpVO6+fRSCKEU= +github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2 v1.42.0 h1:C4/D90/j3EF/SokpC4HO1aPMkZV1dgqUbmejdpxQiAE= +github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2 v1.42.0/go.mod h1:pZP3I+Ts+XuhJJtZE49+ABVjfxm7u9/hxcNUYSpY3OE= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.3 h1:dT3MqvGhSoaIhRseqw2I0yH81l7wiR2vjs57O51EAm8= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.3/go.mod h1:GlAeCkHwugxdHaueRr4nhPuY+WW+gR8UjlcqzPr1SPI= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.17 h1:HGErhhrxZlQ044RiM+WdoZxp0p+EGM62y3L6pwA4olE= @@ -82,6 +88,8 @@ github.com/aws/aws-sdk-go-v2/service/wafv2 v1.51.4 h1:1khBA5uryBRJoCb4G2iR5RT06B github.com/aws/aws-sdk-go-v2/service/wafv2 v1.51.4/go.mod h1:QpFImaPGKNwa+MiZ+oo6LbV1PVQBapc0CnrAMRScoxM= github.com/aws/smithy-go v1.22.0 h1:uunKnWlcoL3zO7q+gG2Pk53joueEOsnNB28QdMsmiMM= github.com/aws/smithy-go v1.22.0/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= +github.com/aws/smithy-go v1.22.1 h1:/HPHZQ0g7f4eUeK6HKglFz8uwVfZKgoI25rb/J+dnro= +github.com/aws/smithy-go v1.22.1/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= diff --git a/helm/aws-load-balancer-controller/crds/crds.yaml b/helm/aws-load-balancer-controller/crds/crds.yaml index 048d51e841..b72e687892 100644 --- a/helm/aws-load-balancer-controller/crds/crds.yaml +++ b/helm/aws-load-balancer-controller/crds/crds.yaml @@ -132,6 +132,18 @@ spec: - value type: object type: array + minimumLoadBalancerCapacity: + description: MinimumLoadBalancerCapacity define the capacity reservation + for LoadBalancers for all Ingress that belong to IngressClass with + this IngressClassParams. + properties: + capacityUnits: + description: The Capacity Units Value. + format: int32 + type: integer + required: + - capacityUnits + type: object namespaceSelector: description: |- NamespaceSelector restrict the namespaces of Ingresses that are allowed to specify the IngressClass with this IngressClassParams. @@ -272,6 +284,11 @@ spec: name: ARN priority: 1 type: string + - description: The AWS TargetGroup's Name + jsonPath: .spec.targetGroupName + name: NAME + priority: 2 + type: string - jsonPath: .metadata.creationTimestamp name: AGE type: date @@ -400,6 +417,9 @@ spec: description: targetGroupARN is the Amazon Resource Name (ARN) for the TargetGroup. type: string + targetGroupName: + description: targetGroupName is the Name of the TargetGroup. + type: string targetType: description: targetType is the TargetType of TargetGroup. If unspecified, it will be automatically inferred. @@ -409,7 +429,6 @@ spec: type: string required: - serviceRef - - targetGroupARN type: object status: description: TargetGroupBindingStatus defines the observed state of TargetGroupBinding @@ -442,6 +461,11 @@ spec: name: ARN priority: 1 type: string + - description: The AWS TargetGroup's Name + jsonPath: .spec.targetGroupName + name: NAME + priority: 2 + type: string - jsonPath: .metadata.creationTimestamp name: AGE type: date @@ -627,7 +651,9 @@ spec: targetGroupARN: description: targetGroupARN is the Amazon Resource Name (ARN) for the TargetGroup. - minLength: 1 + type: string + targetGroupName: + description: targetGroupName is the Name of the TargetGroup. type: string targetType: description: targetType is the TargetType of TargetGroup. If unspecified, @@ -642,7 +668,6 @@ spec: type: string required: - serviceRef - - targetGroupARN type: object status: description: TargetGroupBindingStatus defines the observed state of TargetGroupBinding diff --git a/helm/aws-load-balancer-controller/templates/deployment.yaml b/helm/aws-load-balancer-controller/templates/deployment.yaml index da672ab34d..4506d489e8 100644 --- a/helm/aws-load-balancer-controller/templates/deployment.yaml +++ b/helm/aws-load-balancer-controller/templates/deployment.yaml @@ -110,6 +110,9 @@ spec: {{- if .Values.targetgroupbindingMaxExponentialBackoffDelay }} - --targetgroupbinding-max-exponential-backoff-delay={{ .Values.targetgroupbindingMaxExponentialBackoffDelay }} {{- end }} + {{- if .Values.lbStabilizationMonitorInterval }} + - --lb-stabilization-monitor-interval={{ .Values.lbStabilizationMonitorInterval }} + {{- end }} {{- if .Values.logLevel }} - --log-level={{ .Values.logLevel }} {{- end }} diff --git a/helm/aws-load-balancer-controller/values.yaml b/helm/aws-load-balancer-controller/values.yaml index a33542b8b5..d7fcb02fab 100644 --- a/helm/aws-load-balancer-controller/values.yaml +++ b/helm/aws-load-balancer-controller/values.yaml @@ -233,6 +233,9 @@ targetgroupbindingMaxConcurrentReconciles: # Maximum duration of exponential backoff for targetGroupBinding reconcile failures targetgroupbindingMaxExponentialBackoffDelay: +# Interval at which the controller monitors the state of load balancer after creation for stabilization +lbStabilizationMonitorInterval: + # Period at which the controller forces the repopulation of its local object stores. (default 10h0m0s) syncPeriod: @@ -358,6 +361,7 @@ controllerConfig: # SubnetsClusterTagCheck: true # NLBHealthCheckAdvancedConfig: true # ALBSingleSubnet: false + # LBCapacityReservation: true certDiscovery: allowedCertificateAuthorityARNs: "" # empty means all CAs are in scope diff --git a/pkg/annotations/constants.go b/pkg/annotations/constants.go index 3bad8a76b5..b2bc9aad1e 100644 --- a/pkg/annotations/constants.go +++ b/pkg/annotations/constants.go @@ -13,50 +13,51 @@ const ( AnnotationPrefixIngress = "alb.ingress.kubernetes.io" // Ingress annotation suffixes - IngressSuffixLoadBalancerName = "load-balancer-name" - IngressSuffixGroupName = "group.name" - IngressSuffixGroupOrder = "group.order" - IngressSuffixTags = "tags" - IngressSuffixIPAddressType = "ip-address-type" - IngressSuffixScheme = "scheme" - IngressSuffixSubnets = "subnets" - IngressSuffixCustomerOwnedIPv4Pool = "customer-owned-ipv4-pool" - IngressSuffixLoadBalancerAttributes = "load-balancer-attributes" - IngressSuffixWAFv2ACLARN = "wafv2-acl-arn" - IngressSuffixWAFACLID = "waf-acl-id" - IngressSuffixWebACLID = "web-acl-id" // deprecated, use "waf-acl-id" instead. - IngressSuffixShieldAdvancedProtection = "shield-advanced-protection" - IngressSuffixSecurityGroups = "security-groups" - IngressSuffixListenPorts = "listen-ports" - IngressSuffixSSLRedirect = "ssl-redirect" - IngressSuffixInboundCIDRs = "inbound-cidrs" - IngressSuffixCertificateARN = "certificate-arn" - IngressSuffixSSLPolicy = "ssl-policy" - IngressSuffixTargetType = "target-type" - IngressSuffixBackendProtocol = "backend-protocol" - IngressSuffixBackendProtocolVersion = "backend-protocol-version" - IngressSuffixTargetGroupAttributes = "target-group-attributes" - IngressSuffixHealthCheckPort = "healthcheck-port" - IngressSuffixHealthCheckProtocol = "healthcheck-protocol" - IngressSuffixHealthCheckPath = "healthcheck-path" - IngressSuffixHealthCheckIntervalSeconds = "healthcheck-interval-seconds" - IngressSuffixHealthCheckTimeoutSeconds = "healthcheck-timeout-seconds" - IngressSuffixHealthyThresholdCount = "healthy-threshold-count" - IngressSuffixUnhealthyThresholdCount = "unhealthy-threshold-count" - IngressSuffixSuccessCodes = "success-codes" - IngressSuffixAuthType = "auth-type" - IngressSuffixAuthIDPCognito = "auth-idp-cognito" - IngressSuffixAuthIDPOIDC = "auth-idp-oidc" - IngressSuffixAuthOnUnauthenticatedRequest = "auth-on-unauthenticated-request" - IngressSuffixAuthScope = "auth-scope" - IngressSuffixAuthSessionCookie = "auth-session-cookie" - IngressSuffixAuthSessionTimeout = "auth-session-timeout" - IngressSuffixTargetNodeLabels = "target-node-labels" - IngressSuffixManageSecurityGroupRules = "manage-backend-security-group-rules" - IngressSuffixMutualAuthentication = "mutual-authentication" - IngressSuffixSecurityGroupPrefixLists = "security-group-prefix-lists" - IngressSuffixlsAttsAnnotationPrefix = "listener-attributes" - IngressLBSuffixMultiClusterTargetGroup = "multi-cluster-target-group" + IngressSuffixLoadBalancerName = "load-balancer-name" + IngressSuffixGroupName = "group.name" + IngressSuffixGroupOrder = "group.order" + IngressSuffixTags = "tags" + IngressSuffixIPAddressType = "ip-address-type" + IngressSuffixScheme = "scheme" + IngressSuffixSubnets = "subnets" + IngressSuffixCustomerOwnedIPv4Pool = "customer-owned-ipv4-pool" + IngressSuffixLoadBalancerAttributes = "load-balancer-attributes" + IngressSuffixWAFv2ACLARN = "wafv2-acl-arn" + IngressSuffixWAFACLID = "waf-acl-id" + IngressSuffixWebACLID = "web-acl-id" // deprecated, use "waf-acl-id" instead. + IngressSuffixShieldAdvancedProtection = "shield-advanced-protection" + IngressSuffixSecurityGroups = "security-groups" + IngressSuffixListenPorts = "listen-ports" + IngressSuffixSSLRedirect = "ssl-redirect" + IngressSuffixInboundCIDRs = "inbound-cidrs" + IngressSuffixCertificateARN = "certificate-arn" + IngressSuffixSSLPolicy = "ssl-policy" + IngressSuffixTargetType = "target-type" + IngressSuffixBackendProtocol = "backend-protocol" + IngressSuffixBackendProtocolVersion = "backend-protocol-version" + IngressSuffixTargetGroupAttributes = "target-group-attributes" + IngressSuffixHealthCheckPort = "healthcheck-port" + IngressSuffixHealthCheckProtocol = "healthcheck-protocol" + IngressSuffixHealthCheckPath = "healthcheck-path" + IngressSuffixHealthCheckIntervalSeconds = "healthcheck-interval-seconds" + IngressSuffixHealthCheckTimeoutSeconds = "healthcheck-timeout-seconds" + IngressSuffixHealthyThresholdCount = "healthy-threshold-count" + IngressSuffixUnhealthyThresholdCount = "unhealthy-threshold-count" + IngressSuffixSuccessCodes = "success-codes" + IngressSuffixAuthType = "auth-type" + IngressSuffixAuthIDPCognito = "auth-idp-cognito" + IngressSuffixAuthIDPOIDC = "auth-idp-oidc" + IngressSuffixAuthOnUnauthenticatedRequest = "auth-on-unauthenticated-request" + IngressSuffixAuthScope = "auth-scope" + IngressSuffixAuthSessionCookie = "auth-session-cookie" + IngressSuffixAuthSessionTimeout = "auth-session-timeout" + IngressSuffixTargetNodeLabels = "target-node-labels" + IngressSuffixManageSecurityGroupRules = "manage-backend-security-group-rules" + IngressSuffixMutualAuthentication = "mutual-authentication" + IngressSuffixSecurityGroupPrefixLists = "security-group-prefix-lists" + IngressSuffixlsAttsAnnotationPrefix = "listener-attributes" + IngressLBSuffixMultiClusterTargetGroup = "multi-cluster-target-group" + IngressSuffixLoadBalancerCapacityReservation = "minimum-load-balancer-capacity" // NLB annotation suffixes // prefixes service.beta.kubernetes.io, service.kubernetes.io @@ -101,4 +102,5 @@ const ( SvcLBSuffixMultiClusterTargetGroup = "aws-load-balancer-multi-cluster-target-group" ScvLBSuffixEnablePrefixForIpv6SourceNat = "aws-load-balancer-enable-prefix-for-ipv6-source-nat" ScvLBSuffixSourceNatIpv6Prefixes = "aws-load-balancer-source-nat-ipv6-prefixes" + SvcLBSuffixLoadBalancerCapacityReservation = "aws-load-balancer-minimum-load-balancer-capacity" ) diff --git a/pkg/aws/services/elbv2.go b/pkg/aws/services/elbv2.go index 4b49990656..b89b83d00c 100644 --- a/pkg/aws/services/elbv2.go +++ b/pkg/aws/services/elbv2.go @@ -59,6 +59,8 @@ type ELBV2 interface { AddListenerCertificatesWithContext(ctx context.Context, input *elasticloadbalancingv2.AddListenerCertificatesInput) (*elasticloadbalancingv2.AddListenerCertificatesOutput, error) DescribeListenerAttributesWithContext(ctx context.Context, input *elasticloadbalancingv2.DescribeListenerAttributesInput) (*elasticloadbalancingv2.DescribeListenerAttributesOutput, error) ModifyListenerAttributesWithContext(ctx context.Context, input *elasticloadbalancingv2.ModifyListenerAttributesInput) (*elasticloadbalancingv2.ModifyListenerAttributesOutput, error) + ModifyCapacityReservationWithContext(ctx context.Context, input *elasticloadbalancingv2.ModifyCapacityReservationInput) (*elasticloadbalancingv2.ModifyCapacityReservationOutput, error) + DescribeCapacityReservationWithContext(ctx context.Context, input *elasticloadbalancingv2.DescribeCapacityReservationInput) (*elasticloadbalancingv2.DescribeCapacityReservationOutput, error) } func NewELBV2(awsClientsProvider provider.AWSClientsProvider) ELBV2 { @@ -440,3 +442,19 @@ func (c *elbv2Client) ModifyListenerAttributesWithContext(ctx context.Context, i } return client.ModifyListenerAttributes(ctx, input) } + +func (c *elbv2Client) ModifyCapacityReservationWithContext(ctx context.Context, input *elasticloadbalancingv2.ModifyCapacityReservationInput) (*elasticloadbalancingv2.ModifyCapacityReservationOutput, error) { + client, err := c.awsClientsProvider.GetELBv2Client(ctx, "ModifyCapacityReservation") + if err != nil { + return nil, err + } + return client.ModifyCapacityReservation(ctx, input) +} + +func (c *elbv2Client) DescribeCapacityReservationWithContext(ctx context.Context, input *elasticloadbalancingv2.DescribeCapacityReservationInput) (*elasticloadbalancingv2.DescribeCapacityReservationOutput, error) { + client, err := c.awsClientsProvider.GetELBv2Client(ctx, "DescribeCapacityReservation") + if err != nil { + return nil, err + } + return client.DescribeCapacityReservation(ctx, input) +} diff --git a/pkg/aws/services/elbv2_mocks.go b/pkg/aws/services/elbv2_mocks.go index 42c82464c6..805c0e8ac4 100644 --- a/pkg/aws/services/elbv2_mocks.go +++ b/pkg/aws/services/elbv2_mocks.go @@ -201,6 +201,21 @@ func (mr *MockELBV2MockRecorder) DeregisterTargetsWithContext(arg0, arg1 interfa return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeregisterTargetsWithContext", reflect.TypeOf((*MockELBV2)(nil).DeregisterTargetsWithContext), arg0, arg1) } +// DescribeCapacityReservationWithContext mocks base method. +func (m *MockELBV2) DescribeCapacityReservationWithContext(arg0 context.Context, arg1 *elasticloadbalancingv2.DescribeCapacityReservationInput) (*elasticloadbalancingv2.DescribeCapacityReservationOutput, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DescribeCapacityReservationWithContext", arg0, arg1) + ret0, _ := ret[0].(*elasticloadbalancingv2.DescribeCapacityReservationOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// DescribeCapacityReservationWithContext indicates an expected call of DescribeCapacityReservationWithContext. +func (mr *MockELBV2MockRecorder) DescribeCapacityReservationWithContext(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DescribeCapacityReservationWithContext", reflect.TypeOf((*MockELBV2)(nil).DescribeCapacityReservationWithContext), arg0, arg1) +} + // DescribeListenerAttributesWithContext mocks base method. func (m *MockELBV2) DescribeListenerAttributesWithContext(arg0 context.Context, arg1 *elasticloadbalancingv2.DescribeListenerAttributesInput) (*elasticloadbalancingv2.DescribeListenerAttributesOutput, error) { m.ctrl.T.Helper() @@ -426,6 +441,21 @@ func (mr *MockELBV2MockRecorder) DescribeTrustStoresWithContext(arg0, arg1 inter return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DescribeTrustStoresWithContext", reflect.TypeOf((*MockELBV2)(nil).DescribeTrustStoresWithContext), arg0, arg1) } +// ModifyCapacityReservationWithContext mocks base method. +func (m *MockELBV2) ModifyCapacityReservationWithContext(arg0 context.Context, arg1 *elasticloadbalancingv2.ModifyCapacityReservationInput) (*elasticloadbalancingv2.ModifyCapacityReservationOutput, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ModifyCapacityReservationWithContext", arg0, arg1) + ret0, _ := ret[0].(*elasticloadbalancingv2.ModifyCapacityReservationOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ModifyCapacityReservationWithContext indicates an expected call of ModifyCapacityReservationWithContext. +func (mr *MockELBV2MockRecorder) ModifyCapacityReservationWithContext(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ModifyCapacityReservationWithContext", reflect.TypeOf((*MockELBV2)(nil).ModifyCapacityReservationWithContext), arg0, arg1) +} + // ModifyListenerAttributesWithContext mocks base method. func (m *MockELBV2) ModifyListenerAttributesWithContext(arg0 context.Context, arg1 *elasticloadbalancingv2.ModifyListenerAttributesInput) (*elasticloadbalancingv2.ModifyListenerAttributesOutput, error) { m.ctrl.T.Helper() diff --git a/pkg/config/controller_config.go b/pkg/config/controller_config.go index 7d6b42e44d..c1d775b6f2 100644 --- a/pkg/config/controller_config.go +++ b/pkg/config/controller_config.go @@ -17,11 +17,13 @@ const ( flagK8sClusterName = "cluster-name" flagDefaultTags = "default-tags" flagDefaultTargetType = "default-target-type" + flagDefaultLoadBalancerScheme = "default-load-balancer-scheme" flagExternalManagedTags = "external-managed-tags" flagServiceTargetENISGTags = "service-target-eni-security-group-tags" flagServiceMaxConcurrentReconciles = "service-max-concurrent-reconciles" flagTargetGroupBindingMaxConcurrentReconciles = "targetgroupbinding-max-concurrent-reconciles" flagTargetGroupBindingMaxExponentialBackoffDelay = "targetgroupbinding-max-exponential-backoff-delay" + flagLbStabilizationMonitorInterval = "lb-stabilization-monitor-interval" flagDefaultSSLPolicy = "default-ssl-policy" flagEnableBackendSG = "enable-backend-security-group" flagBackendSecurityGroup = "backend-security-group" @@ -34,6 +36,7 @@ const ( defaultEnableBackendSG = true defaultEnableEndpointSlices = false defaultDisableRestrictedSGRules = false + defaultLbStabilizationMonitorInterval = time.Second * 120 ) var ( @@ -72,6 +75,9 @@ type ControllerConfig struct { // Default target type for Ingress and Service objects DefaultTargetType string + // Default scheme for ELB + DefaultLoadBalancerScheme string + // List of Tag keys on AWS resources that will be managed externally. ExternalManagedTags []string @@ -102,6 +108,9 @@ type ControllerConfig struct { // DisableRestrictedSGRules specifies whether to use restricted security group rules DisableRestrictedSGRules bool + // LBStabilizationMonitorInterval specifies the duration of interval to monitor the load balancer state for stabilization + LBStabilizationMonitorInterval time.Duration + FeatureGates FeatureGates } @@ -114,6 +123,8 @@ func (cfg *ControllerConfig) BindFlags(fs *pflag.FlagSet) { "Default AWS Tags that will be applied to all AWS resources managed by this controller") fs.StringVar(&cfg.DefaultTargetType, flagDefaultTargetType, string(elbv2.TargetTypeInstance), "Default target type for Ingresses and Services - ip, instance") + fs.StringVar(&cfg.DefaultLoadBalancerScheme, flagDefaultLoadBalancerScheme, string(elbv2.LoadBalancerSchemeInternal), + "Default scheme for ELBs") fs.StringSliceVar(&cfg.ExternalManagedTags, flagExternalManagedTags, nil, "List of Tag keys on AWS resources that will be managed externally") fs.IntVar(&cfg.ServiceMaxConcurrentReconciles, flagServiceMaxConcurrentReconciles, defaultMaxConcurrentReconciles, @@ -122,6 +133,8 @@ func (cfg *ControllerConfig) BindFlags(fs *pflag.FlagSet) { "Maximum number of concurrently running reconcile loops for targetGroupBinding") fs.DurationVar(&cfg.TargetGroupBindingMaxExponentialBackoffDelay, flagTargetGroupBindingMaxExponentialBackoffDelay, defaultMaxExponentialBackoffDelay, "Maximum duration of exponential backoff for targetGroupBinding reconcile failures") + fs.DurationVar(&cfg.LBStabilizationMonitorInterval, flagLbStabilizationMonitorInterval, defaultLbStabilizationMonitorInterval, + "Duration of interval to monitor the load balancer state for stabilization") fs.StringVar(&cfg.DefaultSSLPolicy, flagDefaultSSLPolicy, defaultSSLPolicy, "Default SSL policy for load balancers listeners") fs.BoolVar(&cfg.EnableBackendSecurityGroup, flagEnableBackendSG, defaultEnableBackendSG, @@ -162,6 +175,9 @@ func (cfg *ControllerConfig) Validate() error { if err := cfg.validateDefaultTargetType(); err != nil { return err } + if err := cfg.validateDefaultLoadBalancerScheme(); err != nil { + return err + } if err := cfg.validateBackendSecurityGroupConfiguration(); err != nil { return err } @@ -205,6 +221,15 @@ func (cfg *ControllerConfig) validateDefaultTargetType() error { } } +func (cfg *ControllerConfig) validateDefaultLoadBalancerScheme() error { + switch cfg.DefaultLoadBalancerScheme { + case string(elbv2.LoadBalancerSchemeInternal), string(elbv2.LoadBalancerSchemeInternetFacing): + return nil + default: + return errors.Errorf("invalid value %v for default scheme", cfg.DefaultLoadBalancerScheme) + } +} + func (cfg *ControllerConfig) validateBackendSecurityGroupConfiguration() error { if len(cfg.BackendSecurityGroup) == 0 { return nil diff --git a/pkg/config/feature_gates.go b/pkg/config/feature_gates.go index 6835f907f6..e220faff7a 100644 --- a/pkg/config/feature_gates.go +++ b/pkg/config/feature_gates.go @@ -22,6 +22,7 @@ const ( NLBHealthCheckAdvancedConfig Feature = "NLBHealthCheckAdvancedConfig" NLBSecurityGroup Feature = "NLBSecurityGroup" ALBSingleSubnet Feature = "ALBSingleSubnet" + LBCapacityReservation Feature = "LBCapacityReservation" ) type FeatureGates interface { @@ -60,6 +61,7 @@ func NewFeatureGates() FeatureGates { NLBHealthCheckAdvancedConfig: true, NLBSecurityGroup: true, ALBSingleSubnet: false, + LBCapacityReservation: true, }, } } diff --git a/pkg/deploy/elbv2/load_balancer_capacity_reservation_reconciler.go b/pkg/deploy/elbv2/load_balancer_capacity_reservation_reconciler.go new file mode 100644 index 0000000000..0a42db648e --- /dev/null +++ b/pkg/deploy/elbv2/load_balancer_capacity_reservation_reconciler.go @@ -0,0 +1,112 @@ +package elbv2 + +import ( + "context" + awssdk "github.com/aws/aws-sdk-go-v2/aws" + elbv2sdk "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2" + elbv2types "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2/types" + "github.com/go-logr/logr" + "reflect" + "sigs.k8s.io/aws-load-balancer-controller/pkg/aws/services" + "sigs.k8s.io/aws-load-balancer-controller/pkg/config" + elbv2model "sigs.k8s.io/aws-load-balancer-controller/pkg/model/elbv2" +) + +// reconciler for LoadBalancer Capacity Reservation +type LoadBalancerCapacityReservationReconciler interface { + // Reconcile loadBalancer capacity reservation + Reconcile(ctx context.Context, resLB *elbv2model.LoadBalancer, sdkLB LoadBalancerWithTags) error +} + +// NewDefaultLoadBalancerCapacityReservationReconciler constructs new defaultLoadBalancerCapacityReservationReconciler. +func NewDefaultLoadBalancerCapacityReservationReconciler(elbv2Client services.ELBV2, featureGates config.FeatureGates, logger logr.Logger) *defaultLoadBalancerCapacityReservationReconciler { + return &defaultLoadBalancerCapacityReservationReconciler{ + elbv2Client: elbv2Client, + logger: logger, + featureGates: featureGates, + } +} + +var _ LoadBalancerCapacityReservationReconciler = &defaultLoadBalancerCapacityReservationReconciler{} + +// default implementation for LoadBalancerCapacityReservationReconciler +type defaultLoadBalancerCapacityReservationReconciler struct { + elbv2Client services.ELBV2 + logger logr.Logger + featureGates config.FeatureGates +} + +func (r *defaultLoadBalancerCapacityReservationReconciler) Reconcile(ctx context.Context, resLB *elbv2model.LoadBalancer, sdkLB LoadBalancerWithTags) error { + desiredCapacityReservation := resLB.Spec.MinimumLoadBalancerCapacity + // If the annotation is missing or not set, skip the capacity reservation + if desiredCapacityReservation == nil { + return nil + } + //If the value of desired capacityUnits is zero then set desiredCapacityReservation to nil to reset the capacity + if desiredCapacityReservation.CapacityUnits == 0 { + desiredCapacityReservation = nil + } + currentCapacityReservation, err := r.getCurrentCapacityReservation(ctx, sdkLB) + if err != nil { + return err + } + isLBCapacityReservationDrifted := !reflect.DeepEqual(desiredCapacityReservation, currentCapacityReservation) + if !isLBCapacityReservationDrifted { + return nil + } + if desiredCapacityReservation == nil { + //If the value of desired capacityUnits is nil then reset capacity + req := &elbv2sdk.ModifyCapacityReservationInput{ + LoadBalancerArn: sdkLB.LoadBalancer.LoadBalancerArn, + ResetCapacityReservation: awssdk.Bool(true), + } + + r.logger.Info("resetting loadBalancer capacity reservation", + "stackID", resLB.Stack().StackID(), + "resourceID", resLB.ID(), + "arn", awssdk.ToString(sdkLB.LoadBalancer.LoadBalancerArn)) + if _, err := r.elbv2Client.ModifyCapacityReservationWithContext(ctx, req); err != nil { + return err + } + r.logger.Info("reset successful for loadBalancer capacity reservation", + "stackID", resLB.Stack().StackID(), + "resourceID", resLB.ID(), + "arn", awssdk.ToString(sdkLB.LoadBalancer.LoadBalancerArn)) + } else { + req := &elbv2sdk.ModifyCapacityReservationInput{ + LoadBalancerArn: sdkLB.LoadBalancer.LoadBalancerArn, + MinimumLoadBalancerCapacity: &elbv2types.MinimumLoadBalancerCapacity{ + CapacityUnits: awssdk.Int32(desiredCapacityReservation.CapacityUnits), + }, + } + r.logger.Info("modifying loadBalancer capacity reservation", + "stackID", resLB.Stack().StackID(), + "resourceID", resLB.ID(), + "arn", awssdk.ToString(sdkLB.LoadBalancer.LoadBalancerArn), + "change", desiredCapacityReservation) + if _, err := r.elbv2Client.ModifyCapacityReservationWithContext(ctx, req); err != nil { + return err + } + r.logger.Info("modified loadBalancer capacity reservation", + "stackID", resLB.Stack().StackID(), + "resourceID", resLB.ID(), + "arn", awssdk.ToString(sdkLB.LoadBalancer.LoadBalancerArn)) + } + return nil +} + +func (r *defaultLoadBalancerCapacityReservationReconciler) getCurrentCapacityReservation(ctx context.Context, sdkLB LoadBalancerWithTags) (*elbv2model.MinimumLoadBalancerCapacity, error) { + req := &elbv2sdk.DescribeCapacityReservationInput{ + LoadBalancerArn: sdkLB.LoadBalancer.LoadBalancerArn, + } + resp, err := r.elbv2Client.DescribeCapacityReservationWithContext(ctx, req) + if err != nil { + return nil, err + } + var sdkLBMinimumCapacity = &elbv2model.MinimumLoadBalancerCapacity{} + if (resp.CapacityReservationState == nil || len(resp.CapacityReservationState) == 0) && resp.MinimumLoadBalancerCapacity == nil { + return nil, nil + } + sdkLBMinimumCapacity.CapacityUnits = awssdk.ToInt32(resp.MinimumLoadBalancerCapacity.CapacityUnits) + return sdkLBMinimumCapacity, nil +} diff --git a/pkg/deploy/elbv2/load_balancer_capacity_reservation_reconciler_test.go b/pkg/deploy/elbv2/load_balancer_capacity_reservation_reconciler_test.go new file mode 100644 index 0000000000..4e258cf087 --- /dev/null +++ b/pkg/deploy/elbv2/load_balancer_capacity_reservation_reconciler_test.go @@ -0,0 +1,328 @@ +package elbv2 + +import ( + "context" + awssdk "github.com/aws/aws-sdk-go-v2/aws" + elbv2sdk "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2" + elbv2types "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2/types" + "github.com/go-logr/logr" + "github.com/golang/mock/gomock" + "github.com/pkg/errors" + "github.com/stretchr/testify/assert" + "sigs.k8s.io/aws-load-balancer-controller/pkg/aws/services" + "sigs.k8s.io/aws-load-balancer-controller/pkg/config" + coremodel "sigs.k8s.io/aws-load-balancer-controller/pkg/model/core" + elbv2model "sigs.k8s.io/aws-load-balancer-controller/pkg/model/elbv2" + "sigs.k8s.io/controller-runtime/pkg/log" + "testing" +) + +func Test_defaultLoadBalancerCapacityReservationReconciler_updateSDKLoadBalancerWithCapacityReservation(t *testing.T) { + type describeCapacityReservationWithContextCall struct { + req *elbv2sdk.DescribeCapacityReservationInput + resp *elbv2sdk.DescribeCapacityReservationOutput + err error + } + + type modifyCapacityReservationWithContextCall struct { + req *elbv2sdk.ModifyCapacityReservationInput + resp *elbv2sdk.ModifyCapacityReservationOutput + err error + } + + type fields struct { + describeCapacityReservationWithContextCalls []describeCapacityReservationWithContextCall + modifyCapacityReservationWithContextCalls []modifyCapacityReservationWithContextCall + } + type args struct { + sdkLB LoadBalancerWithTags + resLB *elbv2model.LoadBalancer + } + + stack := coremodel.NewDefaultStack(coremodel.StackID{Namespace: "namespace", Name: "name"}) + tests := []struct { + name string + featureGates map[config.Feature]bool + fields fields + args args + wantErr error + }{ + { + name: "capacity reservation should be updated", + featureGates: map[config.Feature]bool{ + config.LBCapacityReservation: true, + }, + fields: fields{ + describeCapacityReservationWithContextCalls: []describeCapacityReservationWithContextCall{ + { + req: &elbv2sdk.DescribeCapacityReservationInput{ + LoadBalancerArn: awssdk.String("my-arn"), + }, + resp: &elbv2sdk.DescribeCapacityReservationOutput{ + CapacityReservationState: nil, + MinimumLoadBalancerCapacity: nil, + }, + }, + }, + modifyCapacityReservationWithContextCalls: []modifyCapacityReservationWithContextCall{ + { + req: &elbv2sdk.ModifyCapacityReservationInput{ + LoadBalancerArn: awssdk.String("my-arn"), + MinimumLoadBalancerCapacity: &elbv2types.MinimumLoadBalancerCapacity{CapacityUnits: awssdk.Int32(1200)}, + }, + }, + }, + }, + args: args{ + sdkLB: LoadBalancerWithTags{ + LoadBalancer: &elbv2types.LoadBalancer{ + LoadBalancerArn: awssdk.String("my-arn"), + }, + }, + resLB: &elbv2model.LoadBalancer{ + ResourceMeta: coremodel.NewResourceMeta(stack, "AWS::ElasticLoadBalancingV2::LoadBalancer", "id-1"), + Spec: elbv2model.LoadBalancerSpec{ + MinimumLoadBalancerCapacity: &elbv2model.MinimumLoadBalancerCapacity{CapacityUnits: 1200}, + }, + }, + }, + }, + { + name: "reset capacity with zero value", + featureGates: map[config.Feature]bool{ + config.LBCapacityReservation: true, + }, + fields: fields{ + describeCapacityReservationWithContextCalls: []describeCapacityReservationWithContextCall{ + { + req: &elbv2sdk.DescribeCapacityReservationInput{ + LoadBalancerArn: awssdk.String("my-arn"), + }, + resp: &elbv2sdk.DescribeCapacityReservationOutput{ + CapacityReservationState: []elbv2types.ZonalCapacityReservationState{}, + MinimumLoadBalancerCapacity: &elbv2types.MinimumLoadBalancerCapacity{CapacityUnits: awssdk.Int32(1200)}, + }, + }, + }, + modifyCapacityReservationWithContextCalls: []modifyCapacityReservationWithContextCall{ + { + req: &elbv2sdk.ModifyCapacityReservationInput{ + LoadBalancerArn: awssdk.String("my-arn"), + ResetCapacityReservation: awssdk.Bool(true), + }, + }, + }, + }, + args: args{ + sdkLB: LoadBalancerWithTags{ + LoadBalancer: &elbv2types.LoadBalancer{ + LoadBalancerArn: awssdk.String("my-arn"), + }, + }, + resLB: &elbv2model.LoadBalancer{ + ResourceMeta: coremodel.NewResourceMeta(stack, "AWS::ElasticLoadBalancingV2::LoadBalancer", "id-1"), + Spec: elbv2model.LoadBalancerSpec{ + MinimumLoadBalancerCapacity: &elbv2model.MinimumLoadBalancerCapacity{CapacityUnits: 0}, + }, + }, + }, + }, + { + name: "no capacity reservation should be updated as their is no change", + featureGates: map[config.Feature]bool{ + config.LBCapacityReservation: true, + }, + fields: fields{ + describeCapacityReservationWithContextCalls: []describeCapacityReservationWithContextCall{ + { + req: &elbv2sdk.DescribeCapacityReservationInput{ + LoadBalancerArn: awssdk.String("my-arn"), + }, + resp: &elbv2sdk.DescribeCapacityReservationOutput{ + CapacityReservationState: []elbv2types.ZonalCapacityReservationState{}, + MinimumLoadBalancerCapacity: &elbv2types.MinimumLoadBalancerCapacity{CapacityUnits: awssdk.Int32(1200)}, + }, + }, + }, + modifyCapacityReservationWithContextCalls: nil, + }, + args: args{ + sdkLB: LoadBalancerWithTags{ + LoadBalancer: &elbv2types.LoadBalancer{ + LoadBalancerArn: awssdk.String("my-arn"), + }, + }, + resLB: &elbv2model.LoadBalancer{ + ResourceMeta: coremodel.NewResourceMeta(stack, "AWS::ElasticLoadBalancingV2::LoadBalancer", "id-1"), + Spec: elbv2model.LoadBalancerSpec{ + MinimumLoadBalancerCapacity: &elbv2model.MinimumLoadBalancerCapacity{CapacityUnits: 1200}, + }, + }, + }, + }, + { + name: "no capacity reservation should be updated as their is no specification on resource", + featureGates: map[config.Feature]bool{ + config.LBCapacityReservation: true, + }, + fields: fields{ + describeCapacityReservationWithContextCalls: nil, + modifyCapacityReservationWithContextCalls: nil, + }, + args: args{ + sdkLB: LoadBalancerWithTags{ + LoadBalancer: &elbv2types.LoadBalancer{ + LoadBalancerArn: awssdk.String("my-arn"), + }, + }, + resLB: &elbv2model.LoadBalancer{ + ResourceMeta: coremodel.NewResourceMeta(stack, "AWS::ElasticLoadBalancingV2::LoadBalancer", "id-1"), + Spec: elbv2model.LoadBalancerSpec{ + MinimumLoadBalancerCapacity: nil, + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + elbv2Client := services.NewMockELBV2(ctrl) + + for _, call := range tt.fields.describeCapacityReservationWithContextCalls { + elbv2Client.EXPECT().DescribeCapacityReservationWithContext(gomock.Any(), call.req).Return(call.resp, call.err) + } + for _, call := range tt.fields.modifyCapacityReservationWithContextCalls { + elbv2Client.EXPECT().ModifyCapacityReservationWithContext(gomock.Any(), call.req).Return(call.resp, call.err) + } + r := &defaultLoadBalancerCapacityReservationReconciler{ + elbv2Client: elbv2Client, + logger: logr.New(&log.NullLogSink{}), + } + err := r.Reconcile(context.Background(), tt.args.resLB, tt.args.sdkLB) + if tt.wantErr != nil { + assert.EqualError(t, err, tt.wantErr.Error()) + } else { + assert.NoError(t, err) + } + }) + } +} + +func Test_defaultLoadBalancerCapacityReservationReconciler_getCurrentLoadBalancerCapacityReservation(t *testing.T) { + type describeCapacityReservationWithContextCall struct { + req *elbv2sdk.DescribeCapacityReservationInput + resp *elbv2sdk.DescribeCapacityReservationOutput + err error + } + type fields struct { + describeCapacityReservationWithContextCalls []describeCapacityReservationWithContextCall + } + type args struct { + sdkLB LoadBalancerWithTags + } + tests := []struct { + name string + fields fields + args args + want *elbv2model.MinimumLoadBalancerCapacity + wantErr error + }{ + { + name: "no capacity reservation case", + fields: fields{ + describeCapacityReservationWithContextCalls: []describeCapacityReservationWithContextCall{ + { + req: &elbv2sdk.DescribeCapacityReservationInput{ + LoadBalancerArn: awssdk.String("my-arn"), + }, + resp: &elbv2sdk.DescribeCapacityReservationOutput{ + CapacityReservationState: nil, + MinimumLoadBalancerCapacity: nil, + }, + }, + }, + }, + args: args{ + sdkLB: LoadBalancerWithTags{ + LoadBalancer: &elbv2types.LoadBalancer{ + LoadBalancerArn: awssdk.String("my-arn"), + }, + Tags: nil, + }, + }, + want: nil, + }, + { + name: "standard case", + fields: fields{ + describeCapacityReservationWithContextCalls: []describeCapacityReservationWithContextCall{ + { + req: &elbv2sdk.DescribeCapacityReservationInput{ + LoadBalancerArn: awssdk.String("my-arn"), + }, + resp: &elbv2sdk.DescribeCapacityReservationOutput{ + CapacityReservationState: []elbv2types.ZonalCapacityReservationState{}, + MinimumLoadBalancerCapacity: &elbv2types.MinimumLoadBalancerCapacity{CapacityUnits: awssdk.Int32(3000)}, + }, + }, + }, + }, + args: args{ + sdkLB: LoadBalancerWithTags{ + LoadBalancer: &elbv2types.LoadBalancer{ + LoadBalancerArn: awssdk.String("my-arn"), + }, + Tags: nil, + }, + }, + want: &elbv2model.MinimumLoadBalancerCapacity{CapacityUnits: 3000}, + }, + { + name: "error case", + fields: fields{ + describeCapacityReservationWithContextCalls: []describeCapacityReservationWithContextCall{ + { + req: &elbv2sdk.DescribeCapacityReservationInput{ + LoadBalancerArn: awssdk.String("my-arn"), + }, + err: errors.New("some error"), + }, + }, + }, + args: args{ + sdkLB: LoadBalancerWithTags{ + LoadBalancer: &elbv2types.LoadBalancer{ + LoadBalancerArn: awssdk.String("my-arn"), + }, + Tags: nil, + }, + }, + wantErr: errors.New("some error"), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + elbv2Client := services.NewMockELBV2(ctrl) + featureGates := config.NewFeatureGates() + for _, call := range tt.fields.describeCapacityReservationWithContextCalls { + elbv2Client.EXPECT().DescribeCapacityReservationWithContext(gomock.Any(), call.req).Return(call.resp, call.err) + } + + r := &defaultLoadBalancerCapacityReservationReconciler{ + elbv2Client: elbv2Client, + featureGates: featureGates, + } + got, err := r.getCurrentCapacityReservation(context.Background(), tt.args.sdkLB) + if tt.wantErr != nil { + assert.EqualError(t, err, tt.wantErr.Error()) + } else { + assert.NoError(t, err) + assert.Equal(t, tt.want, got) + } + }) + } +} diff --git a/pkg/deploy/elbv2/load_balancer_manager.go b/pkg/deploy/elbv2/load_balancer_manager.go index fca93cefee..bc76c45aa1 100644 --- a/pkg/deploy/elbv2/load_balancer_manager.go +++ b/pkg/deploy/elbv2/load_balancer_manager.go @@ -10,6 +10,7 @@ import ( "github.com/go-logr/logr" "k8s.io/apimachinery/pkg/util/sets" "sigs.k8s.io/aws-load-balancer-controller/pkg/aws/services" + "sigs.k8s.io/aws-load-balancer-controller/pkg/config" "sigs.k8s.io/aws-load-balancer-controller/pkg/deploy/tracking" coremodel "sigs.k8s.io/aws-load-balancer-controller/pkg/model/core" elbv2model "sigs.k8s.io/aws-load-balancer-controller/pkg/model/elbv2" @@ -17,7 +18,7 @@ import ( // LoadBalancerManager is responsible for create/update/delete LoadBalancer resources. type LoadBalancerManager interface { - Create(ctx context.Context, resLB *elbv2model.LoadBalancer) (elbv2model.LoadBalancerStatus, error) + Create(ctx context.Context, resLB *elbv2model.LoadBalancer) (elbv2model.LoadBalancerStatus, LoadBalancerWithTags, error) Update(ctx context.Context, resLB *elbv2model.LoadBalancer, sdkLB LoadBalancerWithTags) (elbv2model.LoadBalancerStatus, error) @@ -26,14 +27,16 @@ type LoadBalancerManager interface { // NewDefaultLoadBalancerManager constructs new defaultLoadBalancerManager. func NewDefaultLoadBalancerManager(elbv2Client services.ELBV2, trackingProvider tracking.Provider, - taggingManager TaggingManager, externalManagedTags []string, logger logr.Logger) *defaultLoadBalancerManager { + taggingManager TaggingManager, externalManagedTags []string, featureGates config.FeatureGates, logger logr.Logger) *defaultLoadBalancerManager { return &defaultLoadBalancerManager{ - elbv2Client: elbv2Client, - trackingProvider: trackingProvider, - taggingManager: taggingManager, - attributesReconciler: NewDefaultLoadBalancerAttributeReconciler(elbv2Client, logger), - externalManagedTags: externalManagedTags, - logger: logger, + elbv2Client: elbv2Client, + trackingProvider: trackingProvider, + taggingManager: taggingManager, + attributesReconciler: NewDefaultLoadBalancerAttributeReconciler(elbv2Client, logger), + capacityReservationReconciler: NewDefaultLoadBalancerCapacityReservationReconciler(elbv2Client, featureGates, logger), + externalManagedTags: externalManagedTags, + featureGates: featureGates, + logger: logger, } } @@ -41,19 +44,20 @@ var _ LoadBalancerManager = &defaultLoadBalancerManager{} // defaultLoadBalancerManager implement LoadBalancerManager type defaultLoadBalancerManager struct { - elbv2Client services.ELBV2 - trackingProvider tracking.Provider - taggingManager TaggingManager - attributesReconciler LoadBalancerAttributeReconciler - externalManagedTags []string - - logger logr.Logger + elbv2Client services.ELBV2 + trackingProvider tracking.Provider + taggingManager TaggingManager + attributesReconciler LoadBalancerAttributeReconciler + capacityReservationReconciler LoadBalancerCapacityReservationReconciler + externalManagedTags []string + featureGates config.FeatureGates + logger logr.Logger } -func (m *defaultLoadBalancerManager) Create(ctx context.Context, resLB *elbv2model.LoadBalancer) (elbv2model.LoadBalancerStatus, error) { +func (m *defaultLoadBalancerManager) Create(ctx context.Context, resLB *elbv2model.LoadBalancer) (elbv2model.LoadBalancerStatus, LoadBalancerWithTags, error) { req, err := buildSDKCreateLoadBalancerInput(resLB.Spec) if err != nil { - return elbv2model.LoadBalancerStatus{}, err + return elbv2model.LoadBalancerStatus{}, LoadBalancerWithTags{}, err } lbTags := m.trackingProvider.ResourceTags(resLB.Stack(), resLB, resLB.Spec.Tags) req.Tags = convertTagsToSDKTags(lbTags) @@ -63,7 +67,7 @@ func (m *defaultLoadBalancerManager) Create(ctx context.Context, resLB *elbv2mod "resourceID", resLB.ID()) resp, err := m.elbv2Client.CreateLoadBalancerWithContext(ctx, req) if err != nil { - return elbv2model.LoadBalancerStatus{}, err + return elbv2model.LoadBalancerStatus{}, LoadBalancerWithTags{}, err } sdkLB := LoadBalancerWithTags{ LoadBalancer: &resp.LoadBalancers[0], @@ -74,16 +78,15 @@ func (m *defaultLoadBalancerManager) Create(ctx context.Context, resLB *elbv2mod "resourceID", resLB.ID(), "arn", awssdk.ToString(sdkLB.LoadBalancer.LoadBalancerArn)) if err := m.attributesReconciler.Reconcile(ctx, resLB, sdkLB); err != nil { - return elbv2model.LoadBalancerStatus{}, err + return elbv2model.LoadBalancerStatus{}, LoadBalancerWithTags{}, err } if resLB.Spec.Type == elbv2model.LoadBalancerTypeNetwork && resLB.Spec.SecurityGroupsInboundRulesOnPrivateLink != nil { if err := m.updateSDKLoadBalancerWithSecurityGroups(ctx, resLB, sdkLB); err != nil { - return elbv2model.LoadBalancerStatus{}, err + return elbv2model.LoadBalancerStatus{}, LoadBalancerWithTags{}, err } } - - return buildResLoadBalancerStatus(sdkLB), nil + return buildResLoadBalancerStatus(sdkLB), sdkLB, nil } func (m *defaultLoadBalancerManager) Update(ctx context.Context, resLB *elbv2model.LoadBalancer, sdkLB LoadBalancerWithTags) (elbv2model.LoadBalancerStatus, error) { @@ -187,14 +190,15 @@ func (m *defaultLoadBalancerManager) updateSDKLoadBalancerWithSubnetMappings(ctx isFirstTimeIPv6Setup := currentIPv6Addresses.Len() == 0 && desiredIPv6Addresses.Len() > 0 needsDualstackIPv6Update := isIPv4ToDualstackUpdate(resLB, sdkLB) && isFirstTimeIPv6Setup - if !needsDualstackIPv6Update && desiredSubnets.Equal(currentSubnets) && desiredSubnetsSourceNATPrefixes.Equal(currentSubnetsSourceNATPrefixes) && sdkLBEnablePrefixForIpv6SourceNatValue == resLBEnablePrefixForIpv6SourceNatValue { + if !needsDualstackIPv6Update && desiredSubnets.Equal(currentSubnets) && desiredSubnetsSourceNATPrefixes.Equal(currentSubnetsSourceNATPrefixes) && ((sdkLBEnablePrefixForIpv6SourceNatValue == resLBEnablePrefixForIpv6SourceNatValue) || (resLBEnablePrefixForIpv6SourceNatValue == "")) { return nil } - req := &elbv2sdk.SetSubnetsInput{ - LoadBalancerArn: sdkLB.LoadBalancer.LoadBalancerArn, - SubnetMappings: buildSDKSubnetMappings(resLB.Spec.SubnetMappings), - EnablePrefixForIpv6SourceNat: elbv2types.EnablePrefixForIpv6SourceNatEnum(resLBEnablePrefixForIpv6SourceNatValue), + LoadBalancerArn: sdkLB.LoadBalancer.LoadBalancerArn, + SubnetMappings: buildSDKSubnetMappings(resLB.Spec.SubnetMappings), + } + if resLB.Spec.Type == elbv2model.LoadBalancerTypeNetwork { + req.EnablePrefixForIpv6SourceNat = elbv2types.EnablePrefixForIpv6SourceNatEnum(resLBEnablePrefixForIpv6SourceNatValue) } changeDesc := fmt.Sprintf("%v => %v", currentSubnets.List(), desiredSubnets.List()) m.logger.Info("modifying loadBalancer subnetMappings", diff --git a/pkg/deploy/elbv2/load_balancer_manager_test.go b/pkg/deploy/elbv2/load_balancer_manager_test.go index eae5a9be23..e2b8567d1f 100644 --- a/pkg/deploy/elbv2/load_balancer_manager_test.go +++ b/pkg/deploy/elbv2/load_balancer_manager_test.go @@ -554,6 +554,7 @@ func Test_defaultLoadBalancerManager_updateSDKLoadBalancerWithSubnetMappings(t * resLB: &elbv2model.LoadBalancer{ ResourceMeta: coremodel.NewResourceMeta(stack, "AWS::ElasticLoadBalancingV2::LoadBalancer", "id-1"), Spec: elbv2model.LoadBalancerSpec{ + Type: elbv2model.LoadBalancerTypeNetwork, EnablePrefixForIpv6SourceNat: enablePrefixForIpv6SourceNatOn, SubnetMappings: []elbv2model.SubnetMapping{ { @@ -593,6 +594,7 @@ func Test_defaultLoadBalancerManager_updateSDKLoadBalancerWithSubnetMappings(t * resLB: &elbv2model.LoadBalancer{ ResourceMeta: coremodel.NewResourceMeta(stack, "AWS::ElasticLoadBalancingV2::LoadBalancer", "id-1"), Spec: elbv2model.LoadBalancerSpec{ + Type: elbv2model.LoadBalancerTypeNetwork, SubnetMappings: []elbv2model.SubnetMapping{ { SubnetID: "subnet-A", diff --git a/pkg/deploy/elbv2/load_balancer_synthesizer.go b/pkg/deploy/elbv2/load_balancer_synthesizer.go index 5758820476..629071cce8 100644 --- a/pkg/deploy/elbv2/load_balancer_synthesizer.go +++ b/pkg/deploy/elbv2/load_balancer_synthesizer.go @@ -2,18 +2,19 @@ package elbv2 import ( "context" - elbv2types "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2/types" - "strings" - awssdk "github.com/aws/aws-sdk-go-v2/aws" elbv2sdk "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2" + elbv2types "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2/types" "github.com/go-logr/logr" "github.com/pkg/errors" "k8s.io/apimachinery/pkg/util/sets" "sigs.k8s.io/aws-load-balancer-controller/pkg/aws/services" + "sigs.k8s.io/aws-load-balancer-controller/pkg/config" "sigs.k8s.io/aws-load-balancer-controller/pkg/deploy/tracking" "sigs.k8s.io/aws-load-balancer-controller/pkg/model/core" elbv2model "sigs.k8s.io/aws-load-balancer-controller/pkg/model/elbv2" + "sigs.k8s.io/aws-load-balancer-controller/pkg/runtime" + "strings" ) const ( @@ -22,26 +23,33 @@ const ( // NewLoadBalancerSynthesizer constructs loadBalancerSynthesizer func NewLoadBalancerSynthesizer(elbv2Client services.ELBV2, trackingProvider tracking.Provider, taggingManager TaggingManager, - lbManager LoadBalancerManager, logger logr.Logger, stack core.Stack) *loadBalancerSynthesizer { + lbManager LoadBalancerManager, logger logr.Logger, featureGates config.FeatureGates, controllerConfig config.ControllerConfig, stack core.Stack) *loadBalancerSynthesizer { return &loadBalancerSynthesizer{ - elbv2Client: elbv2Client, - trackingProvider: trackingProvider, - taggingManager: taggingManager, - lbManager: lbManager, - logger: logger, - stack: stack, + elbv2Client: elbv2Client, + trackingProvider: trackingProvider, + taggingManager: taggingManager, + lbManager: lbManager, + logger: logger, + stack: stack, + featureGates: featureGates, + controllerConfig: controllerConfig, + lbsNeedingCapacityModification: nil, + capacityReservationReconciler: NewDefaultLoadBalancerCapacityReservationReconciler(elbv2Client, featureGates, logger), } } // loadBalancerSynthesizer is responsible for synthesize LoadBalancer resources types for certain stack. type loadBalancerSynthesizer struct { - elbv2Client services.ELBV2 - trackingProvider tracking.Provider - taggingManager TaggingManager - lbManager LoadBalancerManager - logger logr.Logger - - stack core.Stack + elbv2Client services.ELBV2 + trackingProvider tracking.Provider + taggingManager TaggingManager + lbManager LoadBalancerManager + logger logr.Logger + stack core.Stack + featureGates config.FeatureGates + controllerConfig config.ControllerConfig + lbsNeedingCapacityModification []resAndSDKLoadBalancerPair + capacityReservationReconciler LoadBalancerCapacityReservationReconciler } func (s *loadBalancerSynthesizer) Synthesize(ctx context.Context) error { @@ -75,10 +83,17 @@ func (s *loadBalancerSynthesizer) Synthesize(ctx context.Context) error { } } for _, resLB := range unmatchedResLBs { - lbStatus, err := s.lbManager.Create(ctx, resLB) + lbStatus, sdkLB, err := s.lbManager.Create(ctx, resLB) if err != nil { return err } + if s.featureGates.Enabled(config.LBCapacityReservation) && + resLB.Spec.MinimumLoadBalancerCapacity != nil { + s.lbsNeedingCapacityModification = append(s.lbsNeedingCapacityModification, resAndSDKLoadBalancerPair{ + resLB: resLB, + sdkLB: sdkLB, + }) + } resLB.SetStatus(lbStatus) } for _, resAndSDKLB := range matchedResAndSDKLBs { @@ -86,6 +101,12 @@ func (s *loadBalancerSynthesizer) Synthesize(ctx context.Context) error { if err != nil { return err } + if s.featureGates.Enabled(config.LBCapacityReservation) { + s.lbsNeedingCapacityModification = append(s.lbsNeedingCapacityModification, resAndSDKLoadBalancerPair{ + resLB: resAndSDKLB.resLB, + sdkLB: resAndSDKLB.sdkLB, + }) + } resAndSDKLB.resLB.SetStatus(lbStatus) } return nil @@ -106,7 +127,19 @@ func (s *loadBalancerSynthesizer) disableDeletionProtection(ctx context.Context, } func (s *loadBalancerSynthesizer) PostSynthesize(ctx context.Context) error { - // nothing to do here. + for _, resAndSDKLB := range s.lbsNeedingCapacityModification { + isLoadBalancerProvisioning, err := s.isLoadBalancerInProvisioningState(ctx, resAndSDKLB.sdkLB) + if err != nil { + return err + } + if isLoadBalancerProvisioning { + requeueMsg := "monitor provisioning state for load balancer: " + awssdk.ToString(resAndSDKLB.sdkLB.LoadBalancer.LoadBalancerName) + return runtime.NewRequeueNeededAfter(requeueMsg, s.controllerConfig.LBStabilizationMonitorInterval) + } + if err := s.capacityReservationReconciler.Reconcile(ctx, resAndSDKLB.resLB, resAndSDKLB.sdkLB); err != nil { + return err + } + } return nil } @@ -197,3 +230,21 @@ func isSDKLoadBalancerRequiresReplacement(sdkLB LoadBalancerWithTags, resLB *elb } return false } + +func (s *loadBalancerSynthesizer) isLoadBalancerInProvisioningState(ctx context.Context, sdkLB LoadBalancerWithTags) (bool, error) { + lbArn := awssdk.ToString(sdkLB.LoadBalancer.LoadBalancerArn) + req := &elbv2sdk.DescribeLoadBalancersInput{ + LoadBalancerArns: []string{lbArn}, + } + elbv2Resp, err := s.elbv2Client.DescribeLoadBalancersAsList(ctx, req) + if err != nil { + return false, err + } + if len(elbv2Resp) == 0 { + return false, errors.Errorf("no load balancer found for the arn: %v to monitor load balancer state", lbArn) + } + if elbv2Resp[0].State.Code == elbv2types.LoadBalancerStateEnumProvisioning { + return true, nil + } + return false, nil +} diff --git a/pkg/deploy/elbv2/load_balancer_synthesizer_test.go b/pkg/deploy/elbv2/load_balancer_synthesizer_test.go index 50940fd7b9..236eddbb03 100644 --- a/pkg/deploy/elbv2/load_balancer_synthesizer_test.go +++ b/pkg/deploy/elbv2/load_balancer_synthesizer_test.go @@ -1,14 +1,17 @@ package elbv2 import ( - elbv2types "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2/types" - "testing" - + "context" awssdk "github.com/aws/aws-sdk-go-v2/aws" + elbv2sdk "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2" + elbv2types "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2/types" + "github.com/golang/mock/gomock" "github.com/pkg/errors" "github.com/stretchr/testify/assert" + "sigs.k8s.io/aws-load-balancer-controller/pkg/aws/services" coremodel "sigs.k8s.io/aws-load-balancer-controller/pkg/model/core" elbv2model "sigs.k8s.io/aws-load-balancer-controller/pkg/model/elbv2" + "testing" ) func Test_matchResAndSDKLoadBalancers(t *testing.T) { @@ -595,3 +598,122 @@ func Test_isSDKLoadBalancerRequiresReplacement(t *testing.T) { }) } } + +func Test_isLoadBalancerInProvisioningState(t *testing.T) { + type describeLoadBalancersAsListCall struct { + req *elbv2sdk.DescribeLoadBalancersInput + resp []elbv2types.LoadBalancer + err error + } + type fields struct { + describeLoadBalancersAsListCalls []describeLoadBalancersAsListCall + } + type args struct { + sdkLB LoadBalancerWithTags + } + tests := []struct { + name string + fields fields + args args + want bool + wantErr error + }{ + { + name: "load balancer in provisioning state case", + fields: fields{ + describeLoadBalancersAsListCalls: []describeLoadBalancersAsListCall{ + { + req: &elbv2sdk.DescribeLoadBalancersInput{ + LoadBalancerArns: []string{"my-arn"}, + }, + resp: []elbv2types.LoadBalancer{ + { + LoadBalancerArn: awssdk.String("lb-1"), + State: &elbv2types.LoadBalancerState{Code: elbv2types.LoadBalancerStateEnumProvisioning}, + }, + }, + }, + }, + }, + args: args{ + sdkLB: LoadBalancerWithTags{ + LoadBalancer: &elbv2types.LoadBalancer{ + LoadBalancerArn: awssdk.String("my-arn"), + }, + Tags: nil, + }, + }, + want: true, + }, + { + name: "load balancer in active state case", + fields: fields{ + describeLoadBalancersAsListCalls: []describeLoadBalancersAsListCall{ + { + req: &elbv2sdk.DescribeLoadBalancersInput{ + LoadBalancerArns: []string{"my-arn"}, + }, + resp: []elbv2types.LoadBalancer{ + { + LoadBalancerArn: awssdk.String("lb-1"), + State: &elbv2types.LoadBalancerState{Code: elbv2types.LoadBalancerStateEnumActive}, + }, + }, + }, + }, + }, + args: args{ + sdkLB: LoadBalancerWithTags{ + LoadBalancer: &elbv2types.LoadBalancer{ + LoadBalancerArn: awssdk.String("my-arn"), + }, + Tags: nil, + }, + }, + want: false, + }, + { + name: "error case", + fields: fields{ + describeLoadBalancersAsListCalls: []describeLoadBalancersAsListCall{ + { + req: &elbv2sdk.DescribeLoadBalancersInput{ + LoadBalancerArns: []string{"my-arn"}, + }, + err: errors.New("some error"), + }, + }, + }, + args: args{ + sdkLB: LoadBalancerWithTags{ + LoadBalancer: &elbv2types.LoadBalancer{ + LoadBalancerArn: awssdk.String("my-arn"), + }, + Tags: nil, + }, + }, + wantErr: errors.New("some error"), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + elbv2Client := services.NewMockELBV2(ctrl) + for _, call := range tt.fields.describeLoadBalancersAsListCalls { + elbv2Client.EXPECT().DescribeLoadBalancersAsList(gomock.Any(), call.req).Return(call.resp, call.err) + } + + r := &loadBalancerSynthesizer{ + elbv2Client: elbv2Client, + } + got, err := r.isLoadBalancerInProvisioningState(context.Background(), tt.args.sdkLB) + if tt.wantErr != nil { + assert.EqualError(t, err, tt.wantErr.Error()) + } else { + assert.NoError(t, err) + assert.Equal(t, tt.want, got) + } + }) + } +} diff --git a/pkg/deploy/stack_deployer.go b/pkg/deploy/stack_deployer.go index dda035adc3..bcfc6254fe 100644 --- a/pkg/deploy/stack_deployer.go +++ b/pkg/deploy/stack_deployer.go @@ -34,12 +34,13 @@ func NewDefaultStackDeployer(cloud aws.Cloud, k8sClient client.Client, return &defaultStackDeployer{ cloud: cloud, k8sClient: k8sClient, + controllerConfig: config, addonsConfig: config.AddonsConfig, trackingProvider: trackingProvider, ec2TaggingManager: ec2TaggingManager, ec2SGManager: ec2.NewDefaultSecurityGroupManager(cloud.EC2(), trackingProvider, ec2TaggingManager, networkingSGReconciler, cloud.VpcID(), config.ExternalManagedTags, logger), elbv2TaggingManager: elbv2TaggingManager, - elbv2LBManager: elbv2.NewDefaultLoadBalancerManager(cloud.ELBV2(), trackingProvider, elbv2TaggingManager, config.ExternalManagedTags, logger), + elbv2LBManager: elbv2.NewDefaultLoadBalancerManager(cloud.ELBV2(), trackingProvider, elbv2TaggingManager, config.ExternalManagedTags, config.FeatureGates, logger), elbv2LSManager: elbv2.NewDefaultListenerManager(cloud.ELBV2(), trackingProvider, elbv2TaggingManager, config.ExternalManagedTags, config.FeatureGates, logger), elbv2LRManager: elbv2.NewDefaultListenerRuleManager(cloud.ELBV2(), trackingProvider, elbv2TaggingManager, config.ExternalManagedTags, config.FeatureGates, logger), elbv2TGManager: elbv2.NewDefaultTargetGroupManager(cloud.ELBV2(), trackingProvider, elbv2TaggingManager, cloud.VpcID(), config.ExternalManagedTags, logger), @@ -59,6 +60,7 @@ var _ StackDeployer = &defaultStackDeployer{} type defaultStackDeployer struct { cloud aws.Cloud k8sClient client.Client + controllerConfig config.ControllerConfig addonsConfig config.AddonsConfig trackingProvider tracking.Provider ec2TaggingManager ec2.TaggingManager @@ -88,7 +90,7 @@ func (d *defaultStackDeployer) Deploy(ctx context.Context, stack core.Stack) err synthesizers := []ResourceSynthesizer{ ec2.NewSecurityGroupSynthesizer(d.cloud.EC2(), d.trackingProvider, d.ec2TaggingManager, d.ec2SGManager, d.vpcID, d.logger, stack), elbv2.NewTargetGroupSynthesizer(d.cloud.ELBV2(), d.trackingProvider, d.elbv2TaggingManager, d.elbv2TGManager, d.logger, d.featureGates, stack), - elbv2.NewLoadBalancerSynthesizer(d.cloud.ELBV2(), d.trackingProvider, d.elbv2TaggingManager, d.elbv2LBManager, d.logger, stack), + elbv2.NewLoadBalancerSynthesizer(d.cloud.ELBV2(), d.trackingProvider, d.elbv2TaggingManager, d.elbv2LBManager, d.logger, d.featureGates, d.controllerConfig, stack), elbv2.NewListenerSynthesizer(d.cloud.ELBV2(), d.elbv2TaggingManager, d.elbv2LSManager, d.logger, stack), elbv2.NewListenerRuleSynthesizer(d.cloud.ELBV2(), d.elbv2TaggingManager, d.elbv2LRManager, d.logger, stack), elbv2.NewTargetGroupBindingSynthesizer(d.k8sClient, d.trackingProvider, d.elbv2TGBManager, d.logger, stack), diff --git a/pkg/ingress/model_build_load_balancer.go b/pkg/ingress/model_build_load_balancer.go index fac48df9a9..111905cbf4 100644 --- a/pkg/ingress/model_build_load_balancer.go +++ b/pkg/ingress/model_build_load_balancer.go @@ -5,13 +5,12 @@ import ( "crypto/sha256" "encoding/hex" "fmt" - ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types" - "regexp" - awssdk "github.com/aws/aws-sdk-go-v2/aws" + ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types" "github.com/google/go-cmp/cmp" "github.com/pkg/errors" "k8s.io/apimachinery/pkg/util/sets" + "regexp" "sigs.k8s.io/aws-load-balancer-controller/apis/elbv2/v1beta1" "sigs.k8s.io/aws-load-balancer-controller/pkg/algorithm" "sigs.k8s.io/aws-load-balancer-controller/pkg/annotations" @@ -72,16 +71,21 @@ func (t *defaultModelBuildTask) buildLoadBalancerSpec(ctx context.Context, liste if err != nil { return elbv2model.LoadBalancerSpec{}, err } + lbMinimumCapacity, err := t.buildLoadBalancerMinimumCapacity(ctx) + if err != nil { + return elbv2model.LoadBalancerSpec{}, err + } return elbv2model.LoadBalancerSpec{ - Name: name, - Type: elbv2model.LoadBalancerTypeApplication, - Scheme: scheme, - IPAddressType: ipAddressType, - SubnetMappings: subnetMappings, - SecurityGroups: securityGroups, - CustomerOwnedIPv4Pool: coIPv4Pool, - LoadBalancerAttributes: loadBalancerAttributes, - Tags: tags, + Name: name, + Type: elbv2model.LoadBalancerTypeApplication, + Scheme: scheme, + IPAddressType: ipAddressType, + SubnetMappings: subnetMappings, + SecurityGroups: securityGroups, + CustomerOwnedIPv4Pool: coIPv4Pool, + LoadBalancerAttributes: loadBalancerAttributes, + MinimumLoadBalancerCapacity: lbMinimumCapacity, + Tags: tags, }, nil } diff --git a/pkg/ingress/model_build_load_balancer_capacity_reservation.go b/pkg/ingress/model_build_load_balancer_capacity_reservation.go new file mode 100644 index 0000000000..429e5ee20f --- /dev/null +++ b/pkg/ingress/model_build_load_balancer_capacity_reservation.go @@ -0,0 +1,81 @@ +package ingress + +import ( + "context" + "github.com/pkg/errors" + "sigs.k8s.io/aws-load-balancer-controller/pkg/annotations" + "sigs.k8s.io/aws-load-balancer-controller/pkg/config" + elbv2model "sigs.k8s.io/aws-load-balancer-controller/pkg/model/elbv2" + "strconv" +) + +// buildLoadBalancerMinimumCapacity builds the minimum load balancer capacity for load balancer +func (t *defaultModelBuildTask) buildLoadBalancerMinimumCapacity(_ context.Context) (*elbv2model.MinimumLoadBalancerCapacity, error) { + if !t.featureGates.Enabled(config.LBCapacityReservation) { + return nil, nil + } + ingGroupCapacityUnits, err := t.buildIngressGroupLoadBalancerMinimumCapacity(t.ingGroup.Members) + if err != nil { + return nil, err + } + var minimumLoadBalancerCapacity *elbv2model.MinimumLoadBalancerCapacity + var capacityUnits int64 + for key, value := range ingGroupCapacityUnits { + if key != elbv2model.CapacityUnits { + return nil, errors.Errorf("invalid key to set the capacity: %v, Expected key: %v", key, elbv2model.CapacityUnits) + } + capacityUnits, _ = strconv.ParseInt(value, 10, 64) + minimumLoadBalancerCapacity = &elbv2model.MinimumLoadBalancerCapacity{ + CapacityUnits: int32(capacityUnits), + } + + } + return minimumLoadBalancerCapacity, nil +} + +// buildIngressGroupLoadBalancerMinimumCapacity builds the minimum load balancer capacity for ingresses within a group. +// Note: the capacity reservation specified via IngressClass takes higher priority than the capacity specified via annotation on Ingress. +func (t *defaultModelBuildTask) buildIngressGroupLoadBalancerMinimumCapacity(ingList []ClassifiedIngress) (map[string]string, error) { + if len(ingList) > 0 { + ingClassCapacityUnits, _ := t.buildIngressClassLoadBalancerMinimumCapacity(ingList[0].IngClassConfig) + if ingClassCapacityUnits != nil { + return ingClassCapacityUnits, nil + } + } + ingGroupCapacityUnits := make(map[string]string) + for _, ing := range ingList { + ingGroupCapacity, err := t.buildIngressLoadBalancerMinimumCapacity(ing) + if err != nil { + return nil, err + } + // check for conflict capacity values + for capacityKey, capacityValue := range ingGroupCapacity { + existingCapacityValue, exists := ingGroupCapacityUnits[capacityKey] + if exists && existingCapacityValue != capacityValue { + return nil, errors.Errorf("conflicting capacity reservation %v: %v | %v", capacityKey, existingCapacityValue, capacityValue) + } + ingGroupCapacityUnits[capacityKey] = capacityValue + } + } + return ingGroupCapacityUnits, nil +} + +// buildIngressLoadBalancerMinimumCapacity builds the minimum load balancer capacity used for a single ingress within a group +func (t *defaultModelBuildTask) buildIngressLoadBalancerMinimumCapacity(ing ClassifiedIngress) (map[string]string, error) { + var annotationCapacity map[string]string + if _, err := t.annotationParser.ParseStringMapAnnotation(annotations.IngressSuffixLoadBalancerCapacityReservation, &annotationCapacity, ing.Ing.Annotations); err != nil { + return nil, err + } + return annotationCapacity, nil +} + +// buildIngressClassLoadBalancerMinimumCapacity builds the minimum load balancer capacity for an IngressClass. +func (t *defaultModelBuildTask) buildIngressClassLoadBalancerMinimumCapacity(ingClassConfig ClassConfiguration) (map[string]string, error) { + if ingClassConfig.IngClassParams == nil || ingClassConfig.IngClassParams.Spec.MinimumLoadBalancerCapacity == nil { + return nil, nil + } + capacityUnits := strconv.Itoa(int(ingClassConfig.IngClassParams.Spec.MinimumLoadBalancerCapacity.CapacityUnits)) + ingClassCapacityUnits := make(map[string]string) + ingClassCapacityUnits[elbv2model.CapacityUnits] = capacityUnits + return ingClassCapacityUnits, nil +} diff --git a/pkg/ingress/model_build_load_balancer_capacity_reservation_test.go b/pkg/ingress/model_build_load_balancer_capacity_reservation_test.go new file mode 100644 index 0000000000..5549e6ebe6 --- /dev/null +++ b/pkg/ingress/model_build_load_balancer_capacity_reservation_test.go @@ -0,0 +1,520 @@ +package ingress + +import ( + "context" + "fmt" + "github.com/pkg/errors" + "github.com/stretchr/testify/assert" + networking "k8s.io/api/networking/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + elbv2api "sigs.k8s.io/aws-load-balancer-controller/apis/elbv2/v1beta1" + "sigs.k8s.io/aws-load-balancer-controller/pkg/annotations" + "sigs.k8s.io/aws-load-balancer-controller/pkg/config" + elbv2model "sigs.k8s.io/aws-load-balancer-controller/pkg/model/elbv2" + "testing" +) + +func Test_defaultModelBuildTask_buildLoadBalancerMinimumCapacity(t *testing.T) { + type fields struct { + ingGroup Group + } + tests := []struct { + name string + featureGates map[config.Feature]bool + fields fields + want *elbv2model.MinimumLoadBalancerCapacity + wantErr error + }{ + { + name: "capacity reservation feature disabled", + featureGates: map[config.Feature]bool{ + config.LBCapacityReservation: false, + }, + fields: fields{ + ingGroup: Group{ + ID: GroupID{Name: "ig-group-1"}, + Members: []ClassifiedIngress{ + { + Ing: &networking.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "awesome-ns", + Name: "awesome-ing", + Annotations: map[string]string{ + "alb.ingress.kubernetes.io/minimum-load-balancer-capacity": "CapacityUnits=500", + }, + }, + }, + }, + { + Ing: &networking.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "awesome-ns", + Name: "awesome-ing", + Annotations: map[string]string{ + "alb.ingress.kubernetes.io/minimum-load-balancer-capacity": "CapacityUnits=500", + }, + }, + }, + }, + }, + }, + }, + want: nil, + }, + { + name: "capacity reservation from multiple Ingress that do not conflict", + featureGates: map[config.Feature]bool{ + config.LBCapacityReservation: true, + }, + fields: fields{ + ingGroup: Group{ + ID: GroupID{Name: "ig-group-1"}, + Members: []ClassifiedIngress{ + { + Ing: &networking.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "awesome-ns", + Name: "awesome-ing", + Annotations: map[string]string{ + "alb.ingress.kubernetes.io/minimum-load-balancer-capacity": "CapacityUnits=500", + }, + }, + }, + }, + { + Ing: &networking.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "awesome-ns", + Name: "awesome-ing", + Annotations: map[string]string{ + "alb.ingress.kubernetes.io/minimum-load-balancer-capacity": "CapacityUnits=500", + }, + }, + }, + }, + }, + }, + }, + want: &elbv2model.MinimumLoadBalancerCapacity{CapacityUnits: 500}, + }, + { + name: "capacity reservation from multiple Ingress that conflict", + featureGates: map[config.Feature]bool{ + config.LBCapacityReservation: true, + }, + fields: fields{ + ingGroup: Group{ + ID: GroupID{Name: "ig-group-2"}, + Members: []ClassifiedIngress{ + { + Ing: &networking.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "awesome-ns", + Name: "awesome-ing", + Annotations: map[string]string{ + "alb.ingress.kubernetes.io/minimum-load-balancer-capacity": "CapacityUnits=500", + }, + }, + }, + }, + { + Ing: &networking.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "awesome-ns", + Name: "awesome-ing", + Annotations: map[string]string{ + "alb.ingress.kubernetes.io/minimum-load-balancer-capacity": "CapacityUnits=200", + }, + }, + }, + }, + }, + }, + }, + wantErr: errors.New("conflicting capacity reservation CapacityUnits: 500 | 200"), + }, + { + name: "non-empty annotation capacity reservation from multiple Ingress, non-empty IngressClass capacity reservation", + featureGates: map[config.Feature]bool{ + config.LBCapacityReservation: true, + }, + fields: fields{ + ingGroup: Group{ + ID: GroupID{Name: "ig-group-3"}, + Members: []ClassifiedIngress{ + { + Ing: &networking.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "awesome-ns", + Name: "awesome-ing", + Annotations: map[string]string{ + "alb.ingress.kubernetes.io/minimum-load-balancer-capacity": "CapacityUnits=500", + }, + }, + }, + IngClassConfig: ClassConfiguration{ + IngClassParams: &elbv2api.IngressClassParams{ + ObjectMeta: metav1.ObjectMeta{ + Name: "awesome-class", + }, + Spec: elbv2api.IngressClassParamsSpec{ + MinimumLoadBalancerCapacity: &elbv2api.MinimumLoadBalancerCapacity{ + CapacityUnits: 1200, + }, + }, + }, + }, + }, + { + Ing: &networking.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "awesome-ns", + Name: "awesome-ing", + Annotations: map[string]string{ + "alb.ingress.kubernetes.io/minimum-load-balancer-capacity": "CapacityUnits=600", + }, + }, + }, + }, + }, + }, + }, + want: &elbv2model.MinimumLoadBalancerCapacity{CapacityUnits: 1200}, + }, + { + name: "capacity reservation from Ingress that does not have annotation for setting capacity reservation", + featureGates: map[config.Feature]bool{ + config.LBCapacityReservation: true, + }, + fields: fields{ + ingGroup: Group{ + ID: GroupID{Name: "ig-group-4"}, + Members: []ClassifiedIngress{ + { + Ing: &networking.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "awesome-ns", + Name: "awesome-ing", + Annotations: map[string]string{}, + }, + }, + }, + }, + }, + }, + want: nil, + }, + { + name: "invalid key to set the capacity reservation", + featureGates: map[config.Feature]bool{ + config.LBCapacityReservation: true, + }, + fields: fields{ + ingGroup: Group{ + ID: GroupID{Name: "ig-group-1"}, + Members: []ClassifiedIngress{ + { + Ing: &networking.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "awesome-ns", + Name: "awesome-ing", + Annotations: map[string]string{ + "alb.ingress.kubernetes.io/minimum-load-balancer-capacity": "InvalidUnits=500", + }, + }, + }, + }, + }, + }, + }, + wantErr: errors.New("invalid key to set the capacity: InvalidUnits, Expected key: CapacityUnits"), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + annotationParser := annotations.NewSuffixAnnotationParser("alb.ingress.kubernetes.io") + featureGates := config.NewFeatureGates() + for key, value := range tt.featureGates { + if value { + featureGates.Enable(key) + } else { + featureGates.Disable(key) + } + } + task := &defaultModelBuildTask{ + annotationParser: annotationParser, + ingGroup: tt.fields.ingGroup, + featureGates: featureGates, + } + got, err := task.buildLoadBalancerMinimumCapacity(context.Background()) + if tt.wantErr != nil { + assert.EqualError(t, err, tt.wantErr.Error()) + } else { + assert.NoError(t, err) + assert.Equal(t, tt.want, got) + } + }) + } +} + +func Test_defaultModelBuildTask_buildIngressGroupLoadBalancerMinimumCapacity(t *testing.T) { + type args struct { + ingList []ClassifiedIngress + } + tests := []struct { + name string + args args + want map[string]string + wantErr error + }{ + { + name: "capacity reservation from multiple Ingress that do not conflict", + args: args{ + ingList: []ClassifiedIngress{ + { + Ing: &networking.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "awesome-ns", + Name: "awesome-ing", + Annotations: map[string]string{ + "alb.ingress.kubernetes.io/minimum-load-balancer-capacity": "CapacityUnits=500", + }, + }, + }, + }, + { + Ing: &networking.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "awesome-ns", + Name: "awesome-ing", + Annotations: map[string]string{ + "alb.ingress.kubernetes.io/minimum-load-balancer-capacity": "CapacityUnits=500", + }, + }, + }, + }, + }, + }, + want: map[string]string{ + "CapacityUnits": "500", + }, + }, + { + name: "capacity reservation from multiple Ingress that conflict", + args: args{ + ingList: []ClassifiedIngress{ + { + Ing: &networking.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "awesome-ns", + Name: "awesome-ing", + Annotations: map[string]string{ + "alb.ingress.kubernetes.io/minimum-load-balancer-capacity": "CapacityUnits=500", + }, + }, + }, + }, + { + Ing: &networking.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "awesome-ns", + Name: "awesome-ing", + Annotations: map[string]string{ + "alb.ingress.kubernetes.io/minimum-load-balancer-capacity": "CapacityUnits=200", + }, + }, + }, + }, + }, + }, + wantErr: errors.New("conflicting capacity reservation CapacityUnits: 500 | 200"), + }, + { + name: "non-empty annotation capacity reservation from multiple Ingress, non-empty IngressClass capacity reservation", + args: args{ + ingList: []ClassifiedIngress{ + { + Ing: &networking.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "awesome-ns", + Name: "awesome-ing", + Annotations: map[string]string{ + "alb.ingress.kubernetes.io/minimum-load-balancer-capacity": "CapacityUnits=500", + }, + }, + }, + IngClassConfig: ClassConfiguration{ + IngClassParams: &elbv2api.IngressClassParams{ + ObjectMeta: metav1.ObjectMeta{ + Name: "awesome-class", + }, + Spec: elbv2api.IngressClassParamsSpec{ + MinimumLoadBalancerCapacity: &elbv2api.MinimumLoadBalancerCapacity{ + CapacityUnits: 1200, + }, + }, + }, + }, + }, + { + Ing: &networking.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "awesome-ns", + Name: "awesome-ing", + Annotations: map[string]string{ + "alb.ingress.kubernetes.io/minimum-load-balancer-capacity": "CapacityUnits=600", + }, + }, + }, + }, + }, + }, + want: map[string]string{ + "CapacityUnits": "1200", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + annotationParser := annotations.NewSuffixAnnotationParser("alb.ingress.kubernetes.io") + task := &defaultModelBuildTask{ + annotationParser: annotationParser, + } + got, err := task.buildIngressGroupLoadBalancerMinimumCapacity(tt.args.ingList) + if tt.wantErr != nil { + assert.EqualError(t, err, tt.wantErr.Error()) + } else { + assert.NoError(t, err) + assert.Equal(t, tt.want, got) + } + }) + } +} + +func Test_defaultModelBuildTask_buildIngressLoadBalancerMinimumCapacity(t *testing.T) { + type args struct { + ing ClassifiedIngress + } + tests := []struct { + name string + args args + want map[string]string + wantErr error + }{ + { + name: "non-empty annotation capacity reservation from Ingress", + args: args{ + ing: ClassifiedIngress{ + Ing: &networking.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "awesome-ns", + Name: "awesome-ing", + Annotations: map[string]string{ + "alb.ingress.kubernetes.io/minimum-load-balancer-capacity": "CapacityUnits=500", + }, + }, + }, + }, + }, + want: map[string]string{ + "CapacityUnits": "500", + }, + }, + { + name: "empty capacity reservation from Ingress", + args: args{ + ing: ClassifiedIngress{ + Ing: &networking.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "awesome-ns", + Name: "awesome-ing", + }, + }, + IngClassConfig: ClassConfiguration{}, + }, + }, + want: map[string]string(nil), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + annotationParser := annotations.NewSuffixAnnotationParser("alb.ingress.kubernetes.io") + task := &defaultModelBuildTask{ + annotationParser: annotationParser, + } + got, err := task.buildIngressLoadBalancerMinimumCapacity(tt.args.ing) + if tt.wantErr != nil { + fmt.Println(err) + assert.EqualError(t, err, tt.wantErr.Error()) + } else { + assert.NoError(t, err) + assert.Equal(t, tt.want, got) + } + }) + } +} + +func Test_defaultModelBuildTask_buildIngressClassLoadBalancerMinimumCapacity(t *testing.T) { + type args struct { + ingClassConfig ClassConfiguration + } + tests := []struct { + name string + args args + want map[string]string + }{ + { + name: "non-empty ingressClassParams, non-empty minimumLoadBalancerCapacity", + args: args{ + ingClassConfig: ClassConfiguration{ + IngClassParams: &elbv2api.IngressClassParams{ + ObjectMeta: metav1.ObjectMeta{ + Name: "awesome-class", + }, + Spec: elbv2api.IngressClassParamsSpec{ + MinimumLoadBalancerCapacity: &elbv2api.MinimumLoadBalancerCapacity{ + CapacityUnits: 1200, + }, + }, + }, + }, + }, + want: map[string]string{ + "CapacityUnits": "1200", + }, + }, + { + name: "non-empty ingressClassParams, empty minimumLoadBalancerCapacity", + args: args{ + ingClassConfig: ClassConfiguration{ + IngClassParams: &elbv2api.IngressClassParams{ + ObjectMeta: metav1.ObjectMeta{ + Name: "awesome-class", + }, + Spec: elbv2api.IngressClassParamsSpec{ + MinimumLoadBalancerCapacity: nil, + }, + }, + }, + }, + want: nil, + }, + { + name: "empty ingressClassParams", + args: args{ + ingClassConfig: ClassConfiguration{ + IngClassParams: nil, + }, + }, + want: nil, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + task := &defaultModelBuildTask{} + got, err := task.buildIngressClassLoadBalancerMinimumCapacity(tt.args.ingClassConfig) + assert.NoError(t, err) + assert.Equal(t, tt.want, got) + }) + } +} diff --git a/pkg/ingress/model_builder.go b/pkg/ingress/model_builder.go index 005f74e629..1f8aacb120 100644 --- a/pkg/ingress/model_builder.go +++ b/pkg/ingress/model_builder.go @@ -42,37 +42,38 @@ func NewDefaultModelBuilder(k8sClient client.Client, eventRecorder record.EventR annotationParser annotations.Parser, subnetsResolver networkingpkg.SubnetsResolver, authConfigBuilder AuthConfigBuilder, enhancedBackendBuilder EnhancedBackendBuilder, trackingProvider tracking.Provider, elbv2TaggingManager elbv2deploy.TaggingManager, featureGates config.FeatureGates, - vpcID string, clusterName string, defaultTags map[string]string, externalManagedTags []string, defaultSSLPolicy string, defaultTargetType string, + vpcID string, clusterName string, defaultTags map[string]string, externalManagedTags []string, defaultSSLPolicy string, defaultTargetType string, defaultLoadBalancerScheme string, backendSGProvider networkingpkg.BackendSGProvider, sgResolver networkingpkg.SecurityGroupResolver, enableBackendSG bool, disableRestrictedSGRules bool, allowedCAARNs []string, enableIPTargetType bool, logger logr.Logger) *defaultModelBuilder { certDiscovery := NewACMCertDiscovery(acmClient, allowedCAARNs, logger) ruleOptimizer := NewDefaultRuleOptimizer(logger) return &defaultModelBuilder{ - k8sClient: k8sClient, - eventRecorder: eventRecorder, - ec2Client: ec2Client, - elbv2Client: elbv2Client, - vpcID: vpcID, - clusterName: clusterName, - annotationParser: annotationParser, - subnetsResolver: subnetsResolver, - backendSGProvider: backendSGProvider, - sgResolver: sgResolver, - certDiscovery: certDiscovery, - authConfigBuilder: authConfigBuilder, - enhancedBackendBuilder: enhancedBackendBuilder, - ruleOptimizer: ruleOptimizer, - trackingProvider: trackingProvider, - elbv2TaggingManager: elbv2TaggingManager, - featureGates: featureGates, - defaultTags: defaultTags, - externalManagedTags: sets.NewString(externalManagedTags...), - defaultSSLPolicy: defaultSSLPolicy, - defaultTargetType: elbv2model.TargetType(defaultTargetType), - enableBackendSG: enableBackendSG, - disableRestrictedSGRules: disableRestrictedSGRules, - enableIPTargetType: enableIPTargetType, - logger: logger, + k8sClient: k8sClient, + eventRecorder: eventRecorder, + ec2Client: ec2Client, + elbv2Client: elbv2Client, + vpcID: vpcID, + clusterName: clusterName, + annotationParser: annotationParser, + subnetsResolver: subnetsResolver, + backendSGProvider: backendSGProvider, + sgResolver: sgResolver, + certDiscovery: certDiscovery, + authConfigBuilder: authConfigBuilder, + enhancedBackendBuilder: enhancedBackendBuilder, + ruleOptimizer: ruleOptimizer, + trackingProvider: trackingProvider, + elbv2TaggingManager: elbv2TaggingManager, + featureGates: featureGates, + defaultTags: defaultTags, + externalManagedTags: sets.NewString(externalManagedTags...), + defaultSSLPolicy: defaultSSLPolicy, + defaultTargetType: elbv2model.TargetType(defaultTargetType), + defaultLoadBalancerScheme: elbv2model.LoadBalancerScheme(defaultLoadBalancerScheme), + enableBackendSG: enableBackendSG, + disableRestrictedSGRules: disableRestrictedSGRules, + enableIPTargetType: enableIPTargetType, + logger: logger, } } @@ -88,24 +89,25 @@ type defaultModelBuilder struct { vpcID string clusterName string - annotationParser annotations.Parser - subnetsResolver networkingpkg.SubnetsResolver - backendSGProvider networkingpkg.BackendSGProvider - sgResolver networkingpkg.SecurityGroupResolver - certDiscovery CertDiscovery - authConfigBuilder AuthConfigBuilder - enhancedBackendBuilder EnhancedBackendBuilder - ruleOptimizer RuleOptimizer - trackingProvider tracking.Provider - elbv2TaggingManager elbv2deploy.TaggingManager - featureGates config.FeatureGates - defaultTags map[string]string - externalManagedTags sets.String - defaultSSLPolicy string - defaultTargetType elbv2model.TargetType - enableBackendSG bool - disableRestrictedSGRules bool - enableIPTargetType bool + annotationParser annotations.Parser + subnetsResolver networkingpkg.SubnetsResolver + backendSGProvider networkingpkg.BackendSGProvider + sgResolver networkingpkg.SecurityGroupResolver + certDiscovery CertDiscovery + authConfigBuilder AuthConfigBuilder + enhancedBackendBuilder EnhancedBackendBuilder + ruleOptimizer RuleOptimizer + trackingProvider tracking.Provider + elbv2TaggingManager elbv2deploy.TaggingManager + featureGates config.FeatureGates + defaultTags map[string]string + externalManagedTags sets.String + defaultSSLPolicy string + defaultTargetType elbv2model.TargetType + defaultLoadBalancerScheme elbv2model.LoadBalancerScheme + enableBackendSG bool + disableRestrictedSGRules bool + enableIPTargetType bool logger logr.Logger } @@ -142,7 +144,7 @@ func (b *defaultModelBuilder) Build(ctx context.Context, ingGroup Group) (core.S defaultTags: b.defaultTags, externalManagedTags: b.externalManagedTags, defaultIPAddressType: elbv2model.IPAddressTypeIPV4, - defaultScheme: elbv2model.LoadBalancerSchemeInternal, + defaultScheme: b.defaultLoadBalancerScheme, defaultSSLPolicy: b.defaultSSLPolicy, defaultTargetType: b.defaultTargetType, defaultBackendProtocol: elbv2model.ProtocolHTTP, diff --git a/pkg/ingress/model_builder_test.go b/pkg/ingress/model_builder_test.go index 99d9fb066d..c9040a4b3e 100644 --- a/pkg/ingress/model_builder_test.go +++ b/pkg/ingress/model_builder_test.go @@ -605,14 +605,15 @@ func Test_defaultModelBuilder_Build(t *testing.T) { } tests := []struct { - name string - env env - defaultTargetType string - enableIPTargetType *bool - args args - fields fields - wantStackPatch string - wantErr string + name string + env env + defaultTargetType string + defaultLoadBalancerScheme string + enableIPTargetType *bool + args args + fields fields + wantStackPatch string + wantErr string }{ { name: "Ingress - vanilla internal", @@ -3628,6 +3629,108 @@ func Test_defaultModelBuilder_Build(t *testing.T) { } } } +}`, + }, + { + name: "Ingress - vanilla with default-load-balancer-scheme internet-facing", + env: env{ + svcs: []*corev1.Service{ns_1_svc_1, ns_1_svc_2, ns_1_svc_3}, + }, + fields: fields{ + resolveViaDiscoveryCalls: []resolveViaDiscoveryCall{resolveViaDiscoveryCallForInternetFacingLB}, + listLoadBalancersCalls: []listLoadBalancersCall{listLoadBalancerCallForEmptyLB}, + enableBackendSG: true, + }, + defaultLoadBalancerScheme: string(elbv2model.LoadBalancerSchemeInternetFacing), + args: args{ + ingGroup: Group{ + ID: GroupID{Namespace: "ns-1", Name: "ing-1"}, + Members: []ClassifiedIngress{ + { + Ing: &networking.Ingress{ObjectMeta: metav1.ObjectMeta{ + Namespace: "ns-1", + Name: "ing-1", + }, + Spec: networking.IngressSpec{ + Rules: []networking.IngressRule{ + { + Host: "app-1.example.com", + IngressRuleValue: networking.IngressRuleValue{ + HTTP: &networking.HTTPIngressRuleValue{ + Paths: []networking.HTTPIngressPath{ + { + Path: "/svc-1", + Backend: networking.IngressBackend{ + Service: &networking.IngressServiceBackend{ + Name: ns_1_svc_1.Name, + Port: networking.ServiceBackendPort{ + Name: "http", + }, + }, + }, + }, + { + Path: "/svc-2", + Backend: networking.IngressBackend{ + Service: &networking.IngressServiceBackend{ + Name: ns_1_svc_2.Name, + Port: networking.ServiceBackendPort{ + Name: "http", + }, + }, + }, + }, + }, + }, + }, + }, + { + Host: "app-2.example.com", + IngressRuleValue: networking.IngressRuleValue{ + HTTP: &networking.HTTPIngressRuleValue{ + Paths: []networking.HTTPIngressPath{ + { + Path: "/svc-3", + Backend: networking.IngressBackend{ + Service: &networking.IngressServiceBackend{ + Name: ns_1_svc_3.Name, + Port: networking.ServiceBackendPort{ + Name: "https", + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + wantStackPatch: ` +{ + "resources": { + "AWS::ElasticLoadBalancingV2::LoadBalancer": { + "LoadBalancer": { + "spec": { + "name": "k8s-ns1-ing1-159dd7a143", + "scheme": "internet-facing", + "subnetMapping": [ + { + "subnetID": "subnet-c" + }, + { + "subnetID": "subnet-d" + } + ] + } + } + } + } }`, }, } @@ -3681,6 +3784,10 @@ func Test_defaultModelBuilder_Build(t *testing.T) { if defaultTargetType == "" { defaultTargetType = "instance" } + defaultLoadBalancerScheme := tt.defaultLoadBalancerScheme + if defaultLoadBalancerScheme == "" { + defaultLoadBalancerScheme = string(elbv2model.LoadBalancerSchemeInternal) + } b := &defaultModelBuilder{ k8sClient: k8sClient, @@ -3703,8 +3810,9 @@ func Test_defaultModelBuilder_Build(t *testing.T) { featureGates: config.NewFeatureGates(), logger: logr.New(&log.NullLogSink{}), - defaultSSLPolicy: "ELBSecurityPolicy-2016-08", - defaultTargetType: elbv2model.TargetType(defaultTargetType), + defaultSSLPolicy: "ELBSecurityPolicy-2016-08", + defaultTargetType: elbv2model.TargetType(defaultTargetType), + defaultLoadBalancerScheme: elbv2model.LoadBalancerScheme(defaultLoadBalancerScheme), } if tt.enableIPTargetType == nil { diff --git a/pkg/model/elbv2/load_balancer.go b/pkg/model/elbv2/load_balancer.go index 4e394f5a02..b521430968 100644 --- a/pkg/model/elbv2/load_balancer.go +++ b/pkg/model/elbv2/load_balancer.go @@ -142,6 +142,17 @@ type LoadBalancerAttribute struct { Value string `json:"value"` } +// Unit for setting the capacity on load balancer +const ( + CapacityUnits string = "CapacityUnits" +) + +// Information about a load balancer capacity reservation. +type MinimumLoadBalancerCapacity struct { + // The Capacity Units Value. + CapacityUnits int32 `json:"capacityUnits"` +} + // LoadBalancerSpec defines the desired state of LoadBalancer type LoadBalancerSpec struct { // The name of the load balancer. @@ -183,6 +194,10 @@ type LoadBalancerSpec struct { // +optional LoadBalancerAttributes []LoadBalancerAttribute `json:"loadBalancerAttributes,omitempty"` + // The load balancer capacity reservation + // +optional + MinimumLoadBalancerCapacity *MinimumLoadBalancerCapacity `json:"minimumLoadBalancerCapacity,omitempty"` + // The tags. // +optional Tags map[string]string `json:"tags,omitempty"` diff --git a/pkg/service/model_build_load_balancer.go b/pkg/service/model_build_load_balancer.go index 8142de5351..3fe52ba897 100644 --- a/pkg/service/model_build_load_balancer.go +++ b/pkg/service/model_build_load_balancer.go @@ -65,6 +65,10 @@ func (t *defaultModelBuildTask) buildLoadBalancerSpec(ctx context.Context, schem if err != nil { return elbv2model.LoadBalancerSpec{}, err } + lbMinimumCapacity, err := t.buildLoadBalancerMinimumCapacity(ctx) + if err != nil { + return elbv2model.LoadBalancerSpec{}, err + } securityGroups, err := t.buildLoadBalancerSecurityGroups(ctx, existingLB, ipAddressType) if err != nil { return elbv2model.LoadBalancerSpec{}, err @@ -95,6 +99,7 @@ func (t *defaultModelBuildTask) buildLoadBalancerSpec(ctx context.Context, schem SecurityGroups: securityGroups, SubnetMappings: subnetMappings, LoadBalancerAttributes: lbAttributes, + MinimumLoadBalancerCapacity: lbMinimumCapacity, Tags: tags, } @@ -245,7 +250,7 @@ func (t *defaultModelBuildTask) buildLoadBalancerScheme(ctx context.Context) (el return "", errors.New("invalid load balancer scheme") } } - return elbv2model.LoadBalancerSchemeInternal, nil + return t.defaultLoadBalancerScheme, nil } func (t *defaultModelBuildTask) buildLoadBalancerSchemeViaAnnotation(ctx context.Context) (elbv2model.LoadBalancerScheme, bool, error) { @@ -484,6 +489,33 @@ func (t *defaultModelBuildTask) buildLoadBalancerAttributes(_ context.Context) ( return makeAttributesSliceFromMap(mergedAttributes), nil } +func (t *defaultModelBuildTask) buildLoadBalancerMinimumCapacity(_ context.Context) (*elbv2model.MinimumLoadBalancerCapacity, error) { + if !t.featureGates.Enabled(config.LBCapacityReservation) { + return nil, nil + } + // Parse the annotation + var loadBalancerMinimumCapacityMap map[string]string + if _, err := t.annotationParser.ParseStringMapAnnotation(annotations.SvcLBSuffixLoadBalancerCapacityReservation, &loadBalancerMinimumCapacityMap, t.service.Annotations); err != nil { + return nil, err + } + if loadBalancerMinimumCapacityMap == nil { + return nil, nil + } + // Transform annotation to minimumLoadBalancerCapacity object + var minimumLoadBalancerCapacity *elbv2model.MinimumLoadBalancerCapacity + var capacityUnits int64 + for key, value := range loadBalancerMinimumCapacityMap { + if key != elbv2model.CapacityUnits { + return nil, errors.Errorf("invalid key to set the capacity: %v, Expected key: %v", key, elbv2model.CapacityUnits) + } + capacityUnits, _ = strconv.ParseInt(value, 10, 64) + minimumLoadBalancerCapacity = &elbv2model.MinimumLoadBalancerCapacity{ + CapacityUnits: int32(capacityUnits), + } + } + return minimumLoadBalancerCapacity, nil +} + func makeAttributesSliceFromMap(loadBalancerAttributesMap map[string]string) []elbv2model.LoadBalancerAttribute { attributes := make([]elbv2model.LoadBalancerAttribute, 0, len(loadBalancerAttributesMap)) for attrKey, attrValue := range loadBalancerAttributesMap { diff --git a/pkg/service/model_build_load_balancer_test.go b/pkg/service/model_build_load_balancer_test.go index 8b3370e746..4e681c6db3 100644 --- a/pkg/service/model_build_load_balancer_test.go +++ b/pkg/service/model_build_load_balancer_test.go @@ -1802,3 +1802,80 @@ func Test_defaultModelBuildTask_buildLoadBalancerName(t *testing.T) { }) } } + +func Test_defaultModelBuilderTask_buildLbCapacity(t *testing.T) { + tests := []struct { + testName string + svc *corev1.Service + wantError bool + wantValue *elbv2.MinimumLoadBalancerCapacity + }{ + { + testName: "Default value", + svc: &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + "service.beta.kubernetes.io/aws-load-balancer-type": "nlb-ip", + }, + }, + }, + wantError: false, + wantValue: nil, + }, + { + testName: "Annotation specified", + svc: &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + "service.beta.kubernetes.io/aws-load-balancer-type": "nlb-ip", + "service.beta.kubernetes.io/aws-load-balancer-minimum-load-balancer-capacity": "CapacityUnits=3000", + }, + }, + }, + wantError: false, + wantValue: &elbv2.MinimumLoadBalancerCapacity{ + CapacityUnits: int32(3000), + }, + }, + { + testName: "Annotation invalid", + svc: &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + "service.beta.kubernetes.io/aws-load-balancer-type": "nlb-ip", + "service.beta.kubernetes.io/aws-load-balancer-minimum-load-balancer-capacity": "InvalidUnits=3000", + }, + }, + }, + wantError: true, + }, + } + for _, tt := range tests { + t.Run(tt.testName, func(t *testing.T) { + parser := annotations.NewSuffixAnnotationParser("service.beta.kubernetes.io") + featureGates := config.NewFeatureGates() + builder := &defaultModelBuildTask{ + service: tt.svc, + annotationParser: parser, + defaultAccessLogsS3Bucket: "", + defaultAccessLogsS3Prefix: "", + defaultLoadBalancingCrossZoneEnabled: false, + defaultProxyProtocolV2Enabled: false, + defaultHealthCheckProtocol: elbv2.ProtocolTCP, + defaultHealthCheckPort: healthCheckPortTrafficPort, + defaultHealthCheckPath: "/", + defaultHealthCheckInterval: 10, + defaultHealthCheckTimeout: 10, + defaultHealthCheckHealthyThreshold: 3, + defaultHealthCheckUnhealthyThreshold: 3, + featureGates: featureGates, + } + lbMinimumCapacity, err := builder.buildLoadBalancerMinimumCapacity(context.Background()) + if tt.wantError { + assert.Error(t, err) + } else { + assert.Equal(t, tt.wantValue, lbMinimumCapacity) + } + }) + } +} diff --git a/pkg/service/model_builder.go b/pkg/service/model_builder.go index 9bd68c8cbc..76d6300f72 100644 --- a/pkg/service/model_builder.go +++ b/pkg/service/model_builder.go @@ -39,30 +39,31 @@ type ModelBuilder interface { func NewDefaultModelBuilder(annotationParser annotations.Parser, subnetsResolver networking.SubnetsResolver, vpcInfoProvider networking.VPCInfoProvider, vpcID string, trackingProvider tracking.Provider, elbv2TaggingManager elbv2deploy.TaggingManager, ec2Client services.EC2, featureGates config.FeatureGates, clusterName string, defaultTags map[string]string, - externalManagedTags []string, defaultSSLPolicy string, defaultTargetType string, enableIPTargetType bool, serviceUtils ServiceUtils, + externalManagedTags []string, defaultSSLPolicy string, defaultTargetType string, defaultLoadBalancerScheme string, enableIPTargetType bool, serviceUtils ServiceUtils, backendSGProvider networking.BackendSGProvider, sgResolver networking.SecurityGroupResolver, enableBackendSG bool, disableRestrictedSGRules bool, logger logr.Logger) *defaultModelBuilder { return &defaultModelBuilder{ - annotationParser: annotationParser, - subnetsResolver: subnetsResolver, - vpcInfoProvider: vpcInfoProvider, - trackingProvider: trackingProvider, - elbv2TaggingManager: elbv2TaggingManager, - featureGates: featureGates, - serviceUtils: serviceUtils, - clusterName: clusterName, - vpcID: vpcID, - defaultTags: defaultTags, - externalManagedTags: sets.NewString(externalManagedTags...), - defaultSSLPolicy: defaultSSLPolicy, - defaultTargetType: elbv2model.TargetType(defaultTargetType), - enableIPTargetType: enableIPTargetType, - backendSGProvider: backendSGProvider, - sgResolver: sgResolver, - ec2Client: ec2Client, - enableBackendSG: enableBackendSG, - disableRestrictedSGRules: disableRestrictedSGRules, - logger: logger, + annotationParser: annotationParser, + subnetsResolver: subnetsResolver, + vpcInfoProvider: vpcInfoProvider, + trackingProvider: trackingProvider, + elbv2TaggingManager: elbv2TaggingManager, + featureGates: featureGates, + serviceUtils: serviceUtils, + clusterName: clusterName, + vpcID: vpcID, + defaultTags: defaultTags, + externalManagedTags: sets.NewString(externalManagedTags...), + defaultSSLPolicy: defaultSSLPolicy, + defaultTargetType: elbv2model.TargetType(defaultTargetType), + defaultLoadBalancerScheme: elbv2model.LoadBalancerScheme(defaultLoadBalancerScheme), + enableIPTargetType: enableIPTargetType, + backendSGProvider: backendSGProvider, + sgResolver: sgResolver, + ec2Client: ec2Client, + enableBackendSG: enableBackendSG, + disableRestrictedSGRules: disableRestrictedSGRules, + logger: logger, } } @@ -82,14 +83,15 @@ type defaultModelBuilder struct { enableBackendSG bool disableRestrictedSGRules bool - clusterName string - vpcID string - defaultTags map[string]string - externalManagedTags sets.String - defaultSSLPolicy string - defaultTargetType elbv2model.TargetType - enableIPTargetType bool - logger logr.Logger + clusterName string + vpcID string + defaultTags map[string]string + externalManagedTags sets.String + defaultSSLPolicy string + defaultTargetType elbv2model.TargetType + defaultLoadBalancerScheme elbv2model.LoadBalancerScheme + enableIPTargetType bool + logger logr.Logger } func (b *defaultModelBuilder) Build(ctx context.Context, service *corev1.Service) (core.Stack, *elbv2model.LoadBalancer, bool, error) { @@ -126,6 +128,7 @@ func (b *defaultModelBuilder) Build(ctx context.Context, service *corev1.Service defaultLoadBalancingCrossZoneEnabled: false, defaultProxyProtocolV2Enabled: false, defaultTargetType: b.defaultTargetType, + defaultLoadBalancerScheme: b.defaultLoadBalancerScheme, defaultHealthCheckProtocol: elbv2model.ProtocolTCP, defaultHealthCheckPort: healthCheckPortTrafficPort, defaultHealthCheckPath: "/", @@ -193,6 +196,7 @@ type defaultModelBuildTask struct { defaultLoadBalancingCrossZoneEnabled bool defaultProxyProtocolV2Enabled bool defaultTargetType elbv2model.TargetType + defaultLoadBalancerScheme elbv2model.LoadBalancerScheme defaultHealthCheckProtocol elbv2model.Protocol defaultHealthCheckPort string defaultHealthCheckPath string diff --git a/pkg/service/model_builder_test.go b/pkg/service/model_builder_test.go index d0a0ca0b4b..0ed7f9b771 100644 --- a/pkg/service/model_builder_test.go +++ b/pkg/service/model_builder_test.go @@ -21,6 +21,7 @@ import ( "sigs.k8s.io/aws-load-balancer-controller/pkg/deploy" "sigs.k8s.io/aws-load-balancer-controller/pkg/deploy/elbv2" "sigs.k8s.io/aws-load-balancer-controller/pkg/deploy/tracking" + elbv2model "sigs.k8s.io/aws-load-balancer-controller/pkg/model/elbv2" "sigs.k8s.io/aws-load-balancer-controller/pkg/networking" "sigs.k8s.io/controller-runtime/pkg/log" ) @@ -110,6 +111,7 @@ func Test_defaultModelBuilderTask_Build(t *testing.T) { listLoadBalancerCalls []listLoadBalancerCall fetchVPCInfoCalls []fetchVPCInfoCall defaultTargetType string + defaultLoadBalancerScheme string enableIPTargetType *bool resolveSGViaNameOrIDCall []resolveSGViaNameOrIDCall backendSecurityGroup string @@ -6431,6 +6433,156 @@ func Test_defaultModelBuilderTask_Build(t *testing.T) { listLoadBalancerCalls: []listLoadBalancerCall{listLoadBalancerCallForEmptyLB}, wantError: true, }, + { + testName: "Simple service with default load balancer scheme internet-facing", + svc: &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "nlb-ip-svc-tls", + Namespace: "default", + UID: "bdca2bd0-bfc6-449a-88a3-03451f05f18c", + Annotations: map[string]string{ + "service.beta.kubernetes.io/aws-load-balancer-type": "nlb-ip", + "service.beta.kubernetes.io/aws-load-balancer-inbound-sg-rules-on-private-link-traffic": "on", + }, + }, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeLoadBalancer, + Selector: map[string]string{"app": "hello"}, + Ports: []corev1.ServicePort{ + { + Port: 80, + TargetPort: intstr.FromInt(80), + Protocol: corev1.ProtocolTCP, + }, + }, + }, + }, + resolveViaDiscoveryCalls: []resolveViaDiscoveryCall{resolveViaDiscoveryCallForOneSubnet}, + listLoadBalancerCalls: []listLoadBalancerCall{listLoadBalancerCallForEmptyLB}, + wantError: false, + wantNumResources: 4, + featureGates: map[config.Feature]bool{ + config.NLBSecurityGroup: false, + }, + defaultLoadBalancerScheme: string(elbv2model.LoadBalancerSchemeInternetFacing), + wantValue: ` +{ + "id":"default/nlb-ip-svc-tls", + "resources":{ + "AWS::ElasticLoadBalancingV2::Listener":{ + "80":{ + "spec":{ + "loadBalancerARN":{ + "$ref":"#/resources/AWS::ElasticLoadBalancingV2::LoadBalancer/LoadBalancer/status/loadBalancerARN" + }, + "port":80, + "protocol":"TCP", + "defaultActions":[ + { + "type":"forward", + "forwardConfig":{ + "targetGroups":[ + { + "targetGroupARN":{ + "$ref":"#/resources/AWS::ElasticLoadBalancingV2::TargetGroup/default/nlb-ip-svc-tls:80/status/targetGroupARN" + } + } + ] + } + } + ] + } + } + }, + "AWS::ElasticLoadBalancingV2::LoadBalancer":{ + "LoadBalancer":{ + "spec":{ + "name":"k8s-default-nlbipsvc-4d831c6ca6", + "type":"network", + "scheme":"internet-facing", + "securityGroupsInboundRulesOnPrivateLink":"on", + "ipAddressType":"ipv4", + "subnetMapping":[ + { + "subnetID":"subnet-1" + } + ] + } + } + }, + "AWS::ElasticLoadBalancingV2::TargetGroup":{ + "default/nlb-ip-svc-tls:80":{ + "spec":{ + "name":"k8s-default-nlbipsvc-d4818dcd51", + "targetType":"ip", + "ipAddressType":"ipv4", + "port":80, + "protocol":"TCP", + "healthCheckConfig":{ + "port":"traffic-port", + "protocol":"TCP", + "intervalSeconds":10, + "timeoutSeconds":10, + "healthyThresholdCount":3, + "unhealthyThresholdCount":3 + }, + "targetGroupAttributes":[ + { + "key":"proxy_protocol_v2.enabled", + "value":"false" + } + ] + } + } + }, + "K8S::ElasticLoadBalancingV2::TargetGroupBinding":{ + "default/nlb-ip-svc-tls:80":{ + "spec":{ + "template":{ + "metadata":{ + "name":"k8s-default-nlbipsvc-d4818dcd51", + "namespace":"default", + "creationTimestamp":null + }, + "spec":{ + "targetGroupARN":{ + "$ref":"#/resources/AWS::ElasticLoadBalancingV2::TargetGroup/default/nlb-ip-svc-tls:80/status/targetGroupARN" + }, + "targetType":"ip", + "ipAddressType":"ipv4", + "vpcID": "vpc-xxx", + "serviceRef":{ + "name":"nlb-ip-svc-tls", + "port":80 + }, + "networking":{ + "ingress":[ + { + "from":[ + { + "ipBlock":{ + "cidr":"192.168.0.0/19" + } + } + ], + "ports":[ + { + "protocol":"TCP", + "port":80 + } + ] + } + ] + } + } + } + } + } + } + } +} +`, + }, } for _, tt := range tests { @@ -6471,6 +6623,10 @@ func Test_defaultModelBuilderTask_Build(t *testing.T) { if defaultTargetType == "" { defaultTargetType = "instance" } + defaultLoadBalancerScheme := tt.defaultLoadBalancerScheme + if defaultLoadBalancerScheme == "" { + defaultLoadBalancerScheme = string(elbv2model.LoadBalancerSchemeInternal) + } backendSGProvider := networking.NewMockBackendSGProvider(ctrl) if tt.enableBackendSG { backendSGProvider.EXPECT().Get(gomock.Any(), networking.ResourceType(networking.ResourceTypeService), gomock.Any()).Return(tt.backendSecurityGroup, nil).AnyTimes() @@ -6487,7 +6643,7 @@ func Test_defaultModelBuilderTask_Build(t *testing.T) { enableIPTargetType = *tt.enableIPTargetType } builder := NewDefaultModelBuilder(annotationParser, subnetsResolver, vpcInfoProvider, "vpc-xxx", trackingProvider, elbv2TaggingManager, ec2Client, featureGates, - "my-cluster", nil, nil, "ELBSecurityPolicy-2016-08", defaultTargetType, enableIPTargetType, serviceUtils, + "my-cluster", nil, nil, "ELBSecurityPolicy-2016-08", defaultTargetType, defaultLoadBalancerScheme, enableIPTargetType, serviceUtils, backendSGProvider, sgResolver, tt.enableBackendSG, tt.disableRestrictedSGRules, logr.New(&log.NullLogSink{})) ctx := context.Background() stack, _, _, err := builder.Build(ctx, tt.svc) diff --git a/scripts/aws_sdk_model_override/cleanup.sh b/scripts/aws_sdk_model_override/cleanup.sh index d73ddd3a52..e13562a8ba 100755 --- a/scripts/aws_sdk_model_override/cleanup.sh +++ b/scripts/aws_sdk_model_override/cleanup.sh @@ -1,5 +1,5 @@ #!/bin/bash -SDK_VENDOR_PATH="./scripts/aws_sdk_model_override/aws-sdk-go" +SDK_VENDOR_PATH="./scripts/aws_sdk_model_override/awsSdkGoV2" rm -rf "${SDK_VENDOR_PATH}" -go mod edit -dropreplace github.com/aws/aws-sdk-go +go mod edit -dropreplace github.com/aws/aws-sdk-go-v2 diff --git a/scripts/aws_sdk_model_override/setup.sh b/scripts/aws_sdk_model_override/setup.sh index 6692146e6a..337c8542b7 100755 --- a/scripts/aws_sdk_model_override/setup.sh +++ b/scripts/aws_sdk_model_override/setup.sh @@ -2,21 +2,8 @@ set -e -SDK_VENDOR_PATH="./scripts/aws_sdk_model_override/aws-sdk-go" -SDK_MODEL_OVERRIDE_DST_PATH="${SDK_VENDOR_PATH}/models" -SDK_MODEL_OVERRIDE_SRC_PATH="./scripts/aws_sdk_model_override/models" - -# Clone the SDK to the vendor path (removing an old one if necessary) -rm -rf "${SDK_VENDOR_PATH}" -git clone --depth 1 https://github.com/aws/aws-sdk-go.git "${SDK_VENDOR_PATH}" - -# Override the SDK models -cp -r "${SDK_MODEL_OVERRIDE_SRC_PATH}"/* "${SDK_MODEL_OVERRIDE_DST_PATH}"/. - -# Generate the SDK -pushd "${SDK_VENDOR_PATH}" -make generate -popd +SDK_VENDOR_PATH="./scripts/aws_sdk_model_override" +SDK_MODEL_OVERRIDE_DST_PATH="${SDK_VENDOR_PATH}/awsSdkGoV2/service/elasticloadbalancingv2" # Use the vendored version of aws-sdk-go -go mod edit -replace github.com/aws/aws-sdk-go="${SDK_VENDOR_PATH}" +go mod edit -replace github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2="${SDK_MODEL_OVERRIDE_DST_PATH}" diff --git a/webhooks/elbv2/targetgroupbinding_mutator.go b/webhooks/elbv2/targetgroupbinding_mutator.go index 8a005927cb..dd9fb557e9 100644 --- a/webhooks/elbv2/targetgroupbinding_mutator.go +++ b/webhooks/elbv2/targetgroupbinding_mutator.go @@ -39,6 +39,12 @@ func (m *targetGroupBindingMutator) Prototype(_ admission.Request) (runtime.Obje func (m *targetGroupBindingMutator) MutateCreate(ctx context.Context, obj runtime.Object) (runtime.Object, error) { tgb := obj.(*elbv2api.TargetGroupBinding) + if tgb.Spec.TargetGroupARN == "" && tgb.Spec.TargetGroupName == "" { + return nil, errors.Errorf("must provide either TargetGroupARN or TargetGroupName") + } + if err := m.getArnFromNameIfNeeded(ctx, tgb); err != nil { + return nil, err + } if err := m.defaultingTargetType(ctx, tgb); err != nil { return nil, err } @@ -51,6 +57,17 @@ func (m *targetGroupBindingMutator) MutateCreate(ctx context.Context, obj runtim return tgb, nil } +func (m *targetGroupBindingMutator) getArnFromNameIfNeeded(ctx context.Context, tgb *elbv2api.TargetGroupBinding) error { + if tgb.Spec.TargetGroupARN == "" && tgb.Spec.TargetGroupName != "" { + tgObj, err := m.getTargetGroupsByNameFromAWS(ctx, tgb.Spec.TargetGroupName) + if err != nil { + return err + } + tgb.Spec.TargetGroupARN = *tgObj.TargetGroupArn + } + return nil +} + func (m *targetGroupBindingMutator) MutateUpdate(ctx context.Context, obj runtime.Object, oldObj runtime.Object) (runtime.Object, error) { return obj, nil } @@ -142,6 +159,20 @@ func (m *targetGroupBindingMutator) getTargetGroupFromAWS(ctx context.Context, t return &tgList[0], nil } +func (m *targetGroupBindingMutator) getTargetGroupsByNameFromAWS(ctx context.Context, tgName string) (*elbv2types.TargetGroup, error) { + req := &elbv2sdk.DescribeTargetGroupsInput{ + Names: []string{tgName}, + } + tgList, err := m.elbv2Client.DescribeTargetGroupsAsList(ctx, req) + if err != nil { + return nil, err + } + if len(tgList) != 1 { + return nil, errors.Errorf("expecting a single targetGroup with name [%s] but got %v", tgName, len(tgList)) + } + return &tgList[0], nil +} + func (m *targetGroupBindingMutator) getVpcIDFromAWS(ctx context.Context, tgARN string) (string, error) { targetGroup, err := m.getTargetGroupFromAWS(ctx, tgARN) if err != nil { diff --git a/webhooks/elbv2/targetgroupbinding_mutator_test.go b/webhooks/elbv2/targetgroupbinding_mutator_test.go index 1b6df32614..f23692f808 100644 --- a/webhooks/elbv2/targetgroupbinding_mutator_test.go +++ b/webhooks/elbv2/targetgroupbinding_mutator_test.go @@ -239,6 +239,53 @@ func Test_targetGroupBindingMutator_MutateCreate(t *testing.T) { }, wantErr: errors.New("unable to get target group VpcID: vpcid not found"), }, + { + name: "targetGroupBinding with TargetGroupName instead of TargetGroupARN", + fields: fields{ + describeTargetGroupsAsListCalls: []describeTargetGroupsAsListCall{ + { + req: &elbv2sdk.DescribeTargetGroupsInput{ + Names: []string{"tg-name"}, + }, + resp: []elbv2types.TargetGroup{ + { + TargetGroupArn: awssdk.String("tg-arn"), + TargetGroupName: awssdk.String("tg-name"), + TargetType: elbv2types.TargetTypeEnumInstance, + }, + }, + }, + { + req: &elbv2sdk.DescribeTargetGroupsInput{ + TargetGroupArns: []string{"tg-arn"}, + }, + resp: []elbv2types.TargetGroup{ + { + TargetGroupArn: awssdk.String("tg-arn"), + TargetType: elbv2types.TargetTypeEnumInstance, + }, + }, + }, + }, + }, + args: args{ + obj: &elbv2api.TargetGroupBinding{ + Spec: elbv2api.TargetGroupBindingSpec{ + TargetGroupName: "tg-name", + TargetType: &instanceTargetType, + IPAddressType: &targetGroupIPAddressTypeIPv4, + }, + }, + }, + want: &elbv2api.TargetGroupBinding{ + Spec: elbv2api.TargetGroupBindingSpec{ + TargetGroupARN: "tg-arn", + TargetGroupName: "tg-name", + TargetType: &instanceTargetType, + IPAddressType: &targetGroupIPAddressTypeIPv4, + }, + }, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/webhooks/elbv2/targetgroupbinding_validator.go b/webhooks/elbv2/targetgroupbinding_validator.go index 367c52a381..c4aa4df909 100644 --- a/webhooks/elbv2/targetgroupbinding_validator.go +++ b/webhooks/elbv2/targetgroupbinding_validator.go @@ -2,6 +2,7 @@ package elbv2 import ( "context" + "fmt" elbv2types "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2/types" "regexp" "strings" @@ -53,7 +54,7 @@ func (v *targetGroupBindingValidator) Prototype(_ admission.Request) (runtime.Ob func (v *targetGroupBindingValidator) ValidateCreate(ctx context.Context, obj runtime.Object) error { tgb := obj.(*elbv2api.TargetGroupBinding) - if err := v.checkRequiredFields(tgb); err != nil { + if err := v.checkRequiredFields(ctx, tgb); err != nil { return err } if err := v.checkNodeSelector(tgb); err != nil { @@ -74,7 +75,7 @@ func (v *targetGroupBindingValidator) ValidateCreate(ctx context.Context, obj ru func (v *targetGroupBindingValidator) ValidateUpdate(ctx context.Context, obj runtime.Object, oldObj runtime.Object) error { tgb := obj.(*elbv2api.TargetGroupBinding) oldTgb := oldObj.(*elbv2api.TargetGroupBinding) - if err := v.checkRequiredFields(tgb); err != nil { + if err := v.checkRequiredFields(ctx, tgb); err != nil { return err } if err := v.checkImmutableFields(tgb, oldTgb); err != nil { @@ -91,8 +92,30 @@ func (v *targetGroupBindingValidator) ValidateDelete(ctx context.Context, obj ru } // checkRequiredFields will check required fields are not absent. -func (v *targetGroupBindingValidator) checkRequiredFields(tgb *elbv2api.TargetGroupBinding) error { +func (v *targetGroupBindingValidator) checkRequiredFields(ctx context.Context, tgb *elbv2api.TargetGroupBinding) error { var absentRequiredFields []string + if tgb.Spec.TargetGroupARN == "" { + if tgb.Spec.TargetGroupName == "" { + absentRequiredFields = append(absentRequiredFields, "either TargetGroupARN or TargetGroupName") + } else if tgb.Spec.TargetGroupName != "" { + /* + The purpose of this code is to guarantee that the either the ARN of the TargetGroup exists + or it's possible to infer the ARN by the name of the TargetGroup (since it's unique). + + And even though the validator can't mutate, I added tgb.Spec.TargetGroupARN = *tgObj.TargetGroupArn + to guarantee the object is in a consistent state though the rest of the process. + + The whole code of aws-load-balancer-controller was written assuming there is an ARN. + By changing the object here I guarantee as early as possible that that assumption is true. + */ + + tgObj, err := v.getTargetGroupsByNameFromAWS(ctx, tgb.Spec.TargetGroupName) + if err != nil { + return fmt.Errorf("searching TargetGroup with name %s: %w", tgb.Spec.TargetGroupName, err) + } + tgb.Spec.TargetGroupARN = *tgObj.TargetGroupArn + } + } if tgb.Spec.TargetType == nil { absentRequiredFields = append(absentRequiredFields, "spec.targetType") } @@ -227,6 +250,21 @@ func (v *targetGroupBindingValidator) getVpcIDFromAWS(ctx context.Context, tgARN return awssdk.ToString(targetGroup.VpcId), nil } +// getTargetGroupFromAWS returns the AWS target group corresponding to the tgName +func (v *targetGroupBindingValidator) getTargetGroupsByNameFromAWS(ctx context.Context, tgName string) (*elbv2types.TargetGroup, error) { + req := &elbv2sdk.DescribeTargetGroupsInput{ + Names: []string{tgName}, + } + tgList, err := v.elbv2Client.DescribeTargetGroupsAsList(ctx, req) + if err != nil { + return nil, err + } + if len(tgList) != 1 { + return nil, errors.Errorf("expecting a single targetGroup with name [%s] but got %v", tgName, len(tgList)) + } + return &tgList[0], nil +} + // +kubebuilder:webhook:path=/validate-elbv2-k8s-aws-v1beta1-targetgroupbinding,mutating=false,failurePolicy=fail,groups=elbv2.k8s.aws,resources=targetgroupbindings,verbs=create;update,versions=v1beta1,name=vtargetgroupbinding.elbv2.k8s.aws,sideEffects=None,webhookVersions=v1,admissionReviewVersions=v1beta1 func (v *targetGroupBindingValidator) SetupWithManager(mgr ctrl.Manager) { diff --git a/webhooks/elbv2/targetgroupbinding_validator_test.go b/webhooks/elbv2/targetgroupbinding_validator_test.go index 7ce250047b..a023e526a2 100644 --- a/webhooks/elbv2/targetgroupbinding_validator_test.go +++ b/webhooks/elbv2/targetgroupbinding_validator_test.go @@ -96,8 +96,9 @@ func Test_targetGroupBindingValidator_ValidateCreate(t *testing.T) { args: args{ obj: &elbv2api.TargetGroupBinding{ Spec: elbv2api.TargetGroupBindingSpec{ - TargetType: &ipTargetType, - NodeSelector: &v1.LabelSelector{}, + TargetGroupARN: "tg-1", + TargetType: &ipTargetType, + NodeSelector: &v1.LabelSelector{}, }, }, }, @@ -131,6 +132,47 @@ func Test_targetGroupBindingValidator_ValidateCreate(t *testing.T) { }, wantErr: nil, }, + { + name: "TargetGroupName can be resolved", + fields: fields{ + describeTargetGroupsAsListCalls: []describeTargetGroupsAsListCall{ + { + req: &elbv2sdk.DescribeTargetGroupsInput{ + Names: []string{"tg-name"}, + }, + resp: []elbv2types.TargetGroup{ + { + TargetGroupArn: awssdk.String("tg-arn"), + TargetGroupName: awssdk.String("tg-name"), + TargetType: elbv2types.TargetTypeEnumInstance, + }, + }, + }, + { + req: &elbv2sdk.DescribeTargetGroupsInput{ + TargetGroupArns: []string{"tg-arn"}, + }, + resp: []elbv2types.TargetGroup{ + { + TargetGroupArn: awssdk.String("tg-arn"), + TargetGroupName: awssdk.String("tg-name"), + TargetType: elbv2types.TargetTypeEnumInstance, + }, + }, + }, + }, + }, + args: args{ + obj: &elbv2api.TargetGroupBinding{ + Spec: elbv2api.TargetGroupBindingSpec{ + TargetGroupName: "tg-name", + TargetType: &instanceTargetType, + IPAddressType: &targetGroupIPAddressTypeIPv4, + }, + }, + }, + wantErr: nil, + }, { name: "ipAddressType mismatch with TargetGroup", fields: fields{ @@ -360,14 +402,16 @@ func Test_targetGroupBindingValidator_ValidateUpdate(t *testing.T) { args: args{ obj: &elbv2api.TargetGroupBinding{ Spec: elbv2api.TargetGroupBindingSpec{ - TargetType: &ipTargetType, - NodeSelector: &v1.LabelSelector{}, + TargetGroupARN: "tg-1", + TargetType: &ipTargetType, + NodeSelector: &v1.LabelSelector{}, }, }, oldObj: &elbv2api.TargetGroupBinding{ Spec: elbv2api.TargetGroupBindingSpec{ - TargetType: &ipTargetType, - NodeSelector: &v1.LabelSelector{}, + TargetGroupARN: "tg-1", + TargetType: &ipTargetType, + NodeSelector: &v1.LabelSelector{}, }, }, }, @@ -449,6 +493,20 @@ func Test_targetGroupBindingValidator_checkRequiredFields(t *testing.T) { }, wantErr: errors.New("TargetGroupBinding must specify these fields: spec.targetType"), }, + { + name: "either TargetGroupARN or TargetGroupName must be specified", + args: args{ + tgb: &elbv2api.TargetGroupBinding{ + Spec: elbv2api.TargetGroupBindingSpec{ + TargetGroupARN: "", + TargetGroupName: "", + // TargetType: &ipTargetType, + TargetType: &instanceTargetType, + }, + }, + }, + wantErr: errors.New("TargetGroupBinding must specify these fields: either TargetGroupARN or TargetGroupName"), + }, { name: "targetType is set", args: args{ @@ -467,7 +525,7 @@ func Test_targetGroupBindingValidator_checkRequiredFields(t *testing.T) { v := &targetGroupBindingValidator{ logger: logr.New(&log.NullLogSink{}), } - err := v.checkRequiredFields(tt.args.tgb) + err := v.checkRequiredFields(context.Background(), tt.args.tgb) if tt.wantErr != nil { assert.EqualError(t, err, tt.wantErr.Error()) } else { @@ -807,7 +865,8 @@ func Test_targetGroupBindingValidator_checkNodeSelector(t *testing.T) { args: args{ tgb: &elbv2api.TargetGroupBinding{ Spec: elbv2api.TargetGroupBindingSpec{ - TargetType: &ipTargetType, + TargetGroupARN: "tg-4", + TargetType: &ipTargetType, }, }, }, @@ -818,7 +877,8 @@ func Test_targetGroupBindingValidator_checkNodeSelector(t *testing.T) { args: args{ tgb: &elbv2api.TargetGroupBinding{ Spec: elbv2api.TargetGroupBindingSpec{ - TargetType: &instanceTargetType, + TargetGroupARN: "tg-5", + TargetType: &instanceTargetType, }, }, }, @@ -829,8 +889,9 @@ func Test_targetGroupBindingValidator_checkNodeSelector(t *testing.T) { args: args{ tgb: &elbv2api.TargetGroupBinding{ Spec: elbv2api.TargetGroupBindingSpec{ - TargetType: &instanceTargetType, - NodeSelector: &nodeSelector, + TargetGroupARN: "tg-6", + TargetType: &instanceTargetType, + NodeSelector: &nodeSelector, }, }, }, @@ -841,8 +902,9 @@ func Test_targetGroupBindingValidator_checkNodeSelector(t *testing.T) { args: args{ tgb: &elbv2api.TargetGroupBinding{ Spec: elbv2api.TargetGroupBindingSpec{ - TargetType: &ipTargetType, - NodeSelector: &nodeSelector, + TargetGroupARN: "tg-7", + TargetType: &ipTargetType, + NodeSelector: &nodeSelector, }, }, },