diff --git a/.gitignore b/.gitignore index 0f2180c1af..d9917959af 100644 --- a/.gitignore +++ b/.gitignore @@ -24,3 +24,4 @@ site *.swo *~ *.bak +scripts/aws_sdk_model_override/* diff --git a/apis/elbv2/v1beta1/ingressclassparams_types.go b/apis/elbv2/v1beta1/ingressclassparams_types.go index 23479ff8f1..4fc9216946 100644 --- a/apis/elbv2/v1beta1/ingressclassparams_types.go +++ b/apis/elbv2/v1beta1/ingressclassparams_types.go @@ -101,6 +101,12 @@ type Listener struct { ListenerAttributes []Attribute `json:"listenerAttributes,omitempty"` } +// Information about a load balancer capacity reservation. +type MinimumLoadBalancerCapacity struct { + // The Capacity Units Value. + CapacityUnits int32 `json:"capacityUnits"` +} + // IngressClassParamsSpec defines the desired state of IngressClassParams type IngressClassParamsSpec struct { // CertificateArn specifies the ARN of the certificates for all Ingresses that belong to IngressClass with this IngressClassParams. @@ -146,6 +152,10 @@ type IngressClassParamsSpec struct { // Listeners define a list of listeners with their protocol, port and attributes. // +optional Listeners []Listener `json:"listeners,omitempty"` + + // MinimumLoadBalancerCapacity define the capacity reservation for LoadBalancers for all Ingress that belong to IngressClass with this IngressClassParams. + // +optional + MinimumLoadBalancerCapacity *MinimumLoadBalancerCapacity `json:"minimumLoadBalancerCapacity,omitempty"` } // +kubebuilder:object:root=true diff --git a/apis/elbv2/v1beta1/zz_generated.deepcopy.go b/apis/elbv2/v1beta1/zz_generated.deepcopy.go index d4630526ab..108289a522 100644 --- a/apis/elbv2/v1beta1/zz_generated.deepcopy.go +++ b/apis/elbv2/v1beta1/zz_generated.deepcopy.go @@ -169,6 +169,11 @@ func (in *IngressClassParamsSpec) DeepCopyInto(out *IngressClassParamsSpec) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.MinimumLoadBalancerCapacity != nil { + in, out := &in.MinimumLoadBalancerCapacity, &out.MinimumLoadBalancerCapacity + *out = new(MinimumLoadBalancerCapacity) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new IngressClassParamsSpec. @@ -216,6 +221,21 @@ func (in *Listener) DeepCopy() *Listener { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MinimumLoadBalancerCapacity) DeepCopyInto(out *MinimumLoadBalancerCapacity) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MinimumLoadBalancerCapacity. +func (in *MinimumLoadBalancerCapacity) DeepCopy() *MinimumLoadBalancerCapacity { + if in == nil { + return nil + } + out := new(MinimumLoadBalancerCapacity) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *NetworkingIngressRule) DeepCopyInto(out *NetworkingIngressRule) { *out = *in diff --git a/config/crd/bases/elbv2.k8s.aws_ingressclassparams.yaml b/config/crd/bases/elbv2.k8s.aws_ingressclassparams.yaml index 26b39f0e8c..7f65f28ad7 100644 --- a/config/crd/bases/elbv2.k8s.aws_ingressclassparams.yaml +++ b/config/crd/bases/elbv2.k8s.aws_ingressclassparams.yaml @@ -133,6 +133,18 @@ spec: - value type: object type: array + minimumLoadBalancerCapacity: + description: MinimumLoadBalancerCapacity define the capacity reservation + for LoadBalancers for all Ingress that belong to IngressClass with + this IngressClassParams. + properties: + capacityUnits: + description: The Capacity Units Value. + format: int32 + type: integer + required: + - capacityUnits + type: object namespaceSelector: description: |- NamespaceSelector restrict the namespaces of Ingresses that are allowed to specify the IngressClass with this IngressClassParams. diff --git a/controllers/ingress/group_controller.go b/controllers/ingress/group_controller.go index 175bbb6906..266bbbba61 100644 --- a/controllers/ingress/group_controller.go +++ b/controllers/ingress/group_controller.go @@ -170,6 +170,10 @@ func (r *groupReconciler) buildAndDeployModel(ctx context.Context, ingGroup ingr r.logger.Info("successfully built model", "model", stackJSON) if err := r.stackDeployer.Deploy(ctx, stack); err != nil { + var requeueNeededAfter *runtime.RequeueNeededAfter + if errors.As(err, &requeueNeededAfter) { + return nil, nil, err + } r.recordIngressGroupEvent(ctx, ingGroup, corev1.EventTypeWarning, k8s.IngressEventReasonFailedDeployModel, fmt.Sprintf("Failed deploy model due to %v", err)) return nil, nil, err } diff --git a/controllers/service/service_controller.go b/controllers/service/service_controller.go index 2ed7612b01..fe77f61f0b 100644 --- a/controllers/service/service_controller.go +++ b/controllers/service/service_controller.go @@ -124,6 +124,10 @@ func (r *serviceReconciler) buildModel(ctx context.Context, svc *corev1.Service) func (r *serviceReconciler) deployModel(ctx context.Context, svc *corev1.Service, stack core.Stack) error { if err := r.stackDeployer.Deploy(ctx, stack); err != nil { + var requeueNeededAfter *runtime.RequeueNeededAfter + if errors.As(err, &requeueNeededAfter) { + return err + } r.eventRecorder.Event(svc, corev1.EventTypeWarning, k8s.ServiceEventReasonFailedDeployModel, fmt.Sprintf("Failed deploy model due to %v", err)) return err } diff --git a/docs/deploy/configurations.md b/docs/deploy/configurations.md index 44751238bc..a5796e6410 100644 --- a/docs/deploy/configurations.md +++ b/docs/deploy/configurations.md @@ -104,6 +104,7 @@ Currently, you can set only 1 namespace to watch in this flag. See [this Kuberne | [sync-period](#sync-period) | duration | 10h0m0s | Period at which the controller forces the repopulation of its local object stores | | targetgroupbinding-max-concurrent-reconciles | int | 3 | Maximum number of concurrently running reconcile loops for targetGroupBinding | | targetgroupbinding-max-exponential-backoff-delay | duration | 16m40s | Maximum duration of exponential backoff for targetGroupBinding reconcile failures | +| [lb-stabilization-monitor-interval](#lb-stabilization-monitor-interval) | duration | 2m | Interval at which the controller monitors the state of load balancer after creation | tolerate-non-existent-backend-service | boolean | true | Whether to allow rules which refer to backend services that do not exist (When enabled, it will return 503 error if backend service not exist) | | tolerate-non-existent-backend-action | boolean | true | Whether to allow rules which refer to backend actions that do not exist (When enabled, it will return 503 error if backend action not exist) | | watch-namespace | string | | Namespace the controller watches for updates to Kubernetes objects, If empty, all namespaces are watched. | @@ -137,6 +138,9 @@ Once disabled: As best practice, we do not recommend users to manually modify the resources managed by the controller. And users should not depend on the controller auto-reconciliation to revert the manual modification, or to mitigate any security risks. +### lb-stabilization-monitor-interval +`--lb-stabilization-monitor-interval` defines a fixed interval for the controller to monitor the state of load balancer after the creation for stabilization, default to 2m. It monitors the load balancer state so that once it becomes active it can make the required updates like capacity reservation for the active load balancer. It calls DescribeLoadBalancer API at a fixed interval to monitor the state. Please be mindful that lower value will result into frequent calls which may incur unnecessary AWS API usage. + ### waf-addons By default, the controller assumes sole ownership of the WAF addons associated to the provisioned ALBs, via the flag `--enable-waf` and `--enable-wafv2`. And the users should disable them accordingly if they want a third party like AWS Firewall Manager to associate or remove the WAF-ACL of the ALBs. @@ -177,3 +181,4 @@ There are a set of key=value pairs that describe AWS load balancer controller fe | NLBHealthCheckAdvancedConfiguration | string | true | Enable or disable advanced health check configuration for NLB, for example health check timeout | | ALBSingleSubnet | string | false | If enabled, controller will allow using only 1 subnet for provisioning ALB, which need to get whitelisted by ELB in advance | | NLBSecurityGroup | string | true | Enable or disable all NLB security groups actions including frontend sg creation, backend sg creation, and backend sg modifications | +| LBCapacityReservation | string | true | Enable or disable the capacity reservation feature on ALB and NLB \ No newline at end of file diff --git a/docs/guide/ingress/annotations.md b/docs/guide/ingress/annotations.md index 2dc2947ede..7035dc80f8 100644 --- a/docs/guide/ingress/annotations.md +++ b/docs/guide/ingress/annotations.md @@ -61,6 +61,7 @@ You can add annotations to kubernetes Ingress and Service objects to customize t | [alb.ingress.kubernetes.io/mutual-authentication](#mutual-authentication) | json |N/A| Ingress | Exclusive | | [alb.ingress.kubernetes.io/multi-cluster-target-group](#multi-cluster-target-group) | boolean |N/A| Ingress, Service | N/A | | [alb.ingress.kubernetes.io/listener-attributes.${Protocol}-${Port}](#listener-attributes) | stringMap |N/A| Ingress |Merge| +| [alb.ingress.kubernetes.io/minimum-load-balancer-capacity](#load-balancer-capacity-reservation) | stringMap |N/A| Ingress | Exclusive | ## IngressGroup IngressGroup feature enables you to group multiple Ingress resources together. @@ -931,6 +932,26 @@ In addition, you can use annotations to specify additional tags alb.ingress.kubernetes.io/tags: Environment=dev,Team=test ``` +## Capacity Unit Reservation +Load balancer capacity unit reservation can be configured via following annotations: + +- `alb.ingress.kubernetes.io/minimum-load-balancer-capacity` specifies the + [Capacity Unit Reservation](https://docs.aws.amazon.com/elasticloadbalancing/latest/application/capacity-unit-reservation.html) to be configured. + + !!!example + - set the capacity unit reservation to 1000 + ``` + alb.ingress.kubernetes.io/minimum-load-balancer-capacity: CapacityUnits=1000 + ``` + - reset the capacity unit reservation + ``` + alb.ingress.kubernetes.io/minimum-load-balancer-capacity: CapacityUnits=0 + ``` + + !!!note "Notes" + - If you specify this annotation, but remove it later, the capacity unit reservation is not reset. You need to reset the capacity by setting the capacity units to zero as show in the example above. + - If users do not want the controller to manage the capacity unit reservation on load balancer, they can disable the feature by setting controller command line feature gate flag ```--feature-gates=LBCapacityReservation=true``` + ## Addons - `alb.ingress.kubernetes.io/waf-acl-id` specifies the identifier for the Amazon WAF Classic web ACL. diff --git a/docs/guide/ingress/ingress_class.md b/docs/guide/ingress/ingress_class.md index 3322cc7cfb..fd5d6ee954 100644 --- a/docs/guide/ingress/ingress_class.md +++ b/docs/guide/ingress/ingress_class.md @@ -140,6 +140,16 @@ You can use IngressClassParams to enforce settings for a set of Ingresses. spec: certificateArn: ['arn:aws:acm:us-east-1:123456789:certificate/test-arn-1','arn:aws:acm:us-east-1:123456789:certificate/test-arn-2'] ``` + - with minimumLoadBalancerCapacity.capacityUnits + ``` + apiVersion: elbv2.k8s.aws/v1beta1 + kind: IngressClassParams + metadata: + name: class2048-config + spec: + minimumLoadBalancerCapacity: + capacityUnits: 1000 + ``` ### IngressClassParams specification @@ -233,3 +243,12 @@ Cluster administrators can use `loadBalancerAttributes` field to specify the [Lo 1. If `loadBalancerAttributes` is set, the attributes defined will be applied to the load balancer that belong to this IngressClass. If you specify invalid keys or values for the load balancer attributes, the controller will fail to reconcile ingresses belonging to the particular ingress class. 2. If `loadBalancerAttributes` un-specified, Ingresses with this IngressClass can continue to use `alb.ingress.kubernetes.io/load-balancer-attributes` annotation to specify the load balancer attributes. + +#### spec.minimumLoadBalancerCapacity + +Cluster administrators can use the optional `minimumLoadBalancerCapacity` field to specify the capacity reservation for the load balancers that belong to this IngressClass. +They may specify `capacityUnits`. If the field is specified, LBC will ignore the `alb.ingress.kubernetes.io/minimum-load-balancer-capacity annotation` annotation. + +##### spec.minimumLoadBalancerCapacity.capacityUnits + +If `capacityUnits` is specified, it must be to valid positive value greater than 0. If set to 0, the LBC will reset the capacity reservation for the load balancer. \ No newline at end of file diff --git a/docs/guide/service/annotations.md b/docs/guide/service/annotations.md index e9c494288c..a27c8c7813 100644 --- a/docs/guide/service/annotations.md +++ b/docs/guide/service/annotations.md @@ -56,6 +56,7 @@ | [service.beta.kubernetes.io/aws-load-balancer-multi-cluster-target-group](#multi-cluster-target-group) | boolean | false | If specified, the controller will only operate on targets that exist within the cluster, ignoring targets from other sources. | | [service.beta.kubernetes.io/aws-load-balancer-enable-prefix-for-ipv6-source-nat](#enable-prefix-for-ipv6-source-nat) | string | off | Optional annotation. dualstack lb only. Allowed values - on and off | | [service.beta.kubernetes.io/aws-load-balancer-source-nat-ipv6-prefixes](#source-nat-ipv6-prefixes) | stringList | | Optional annotation. dualstack lb only. This annotation is only applicable when user has to set the service.beta.kubernetes.io/aws-load-balancer-enable-prefix-for-ipv6-source-nat to "on". Length must match the number of subnets | +| [service.beta.kubernetes.io/aws-load-balancer-minimum-load-balancer-capacity](#load-balancer-capacity-reservation) | stringMap | | ## Traffic Routing Traffic Routing can be controlled with following annotations: @@ -579,6 +580,25 @@ Load balancer access can be controlled via following annotations: service.beta.kubernetes.io/aws-load-balancer-inbound-sg-rules-on-private-link-traffic: "off" ``` +## Capacity Unit Reservation +Load balancer capacity unit reservation can be configured via following annotations: + +- `service.beta.kubernetes.io/aws-load-balancer-minimum-load-balancer-capacity` specifies the + [Capacity Unit Reservation](https://docs.aws.amazon.com/elasticloadbalancing/latest/network/capacity-unit-reservation.html) to be configured. + + !!!example + - set the capacity unit reservation to 1000 + ``` + service.beta.kubernetes.io/aws-load-balancer-minimum-load-balancer-capacity: CapacityUnits=3000 + ``` + - reset the capacity unit reservation + ``` + service.beta.kubernetes.io/aws-load-balancer-minimum-load-balancer-capacity: CapacityUnits=0 + ``` + + !!!note "Notes" + - If you specify this annotation, but remove it later, the capacity unit reservation is not reset. You need to reset the capacity by setting the capacity units to zero as show in the example above. + - If users do not want the controller to manage the capacity unit reservation on load balancer, they can disable the feature by setting controller command line feature gate flag ```--feature-gates=LBCapacityReservation=true``` ## Legacy Cloud Provider The AWS Load Balancer Controller manages Kubernetes Services in a compatible way with the AWS cloud provider's legacy service controller. diff --git a/docs/install/iam_policy.json b/docs/install/iam_policy.json index 603c5b280b..1a5b4d614b 100644 --- a/docs/install/iam_policy.json +++ b/docs/install/iam_policy.json @@ -41,7 +41,8 @@ "elasticloadbalancing:DescribeTargetHealth", "elasticloadbalancing:DescribeTags", "elasticloadbalancing:DescribeTrustStores", - "elasticloadbalancing:DescribeListenerAttributes" + "elasticloadbalancing:DescribeListenerAttributes", + "elasticloadbalancing:DescribeCapacityReservation" ], "Resource": "*" }, @@ -191,7 +192,8 @@ "elasticloadbalancing:ModifyTargetGroup", "elasticloadbalancing:ModifyTargetGroupAttributes", "elasticloadbalancing:DeleteTargetGroup", - "elasticloadbalancing:ModifyListenerAttributes" + "elasticloadbalancing:ModifyListenerAttributes", + "elasticloadbalancing:ModifyCapacityReservation" ], "Resource": "*", "Condition": { diff --git a/docs/install/iam_policy_cn.json b/docs/install/iam_policy_cn.json index ae1ad1793f..ba8a39fa79 100644 --- a/docs/install/iam_policy_cn.json +++ b/docs/install/iam_policy_cn.json @@ -41,7 +41,8 @@ "elasticloadbalancing:DescribeTargetHealth", "elasticloadbalancing:DescribeTags", "elasticloadbalancing:DescribeTrustStores", - "elasticloadbalancing:DescribeListenerAttributes" + "elasticloadbalancing:DescribeListenerAttributes", + "elasticloadbalancing:DescribeCapacityReservation" ], "Resource": "*" }, @@ -213,7 +214,8 @@ "elasticloadbalancing:ModifyTargetGroup", "elasticloadbalancing:ModifyTargetGroupAttributes", "elasticloadbalancing:DeleteTargetGroup", - "elasticloadbalancing:ModifyListenerAttributes" + "elasticloadbalancing:ModifyListenerAttributes", + "elasticloadbalancing:ModifyCapacityReservation" ], "Resource": "*", "Condition": { diff --git a/docs/install/iam_policy_us-gov.json b/docs/install/iam_policy_us-gov.json index f2d3d1d293..828f77f4d8 100644 --- a/docs/install/iam_policy_us-gov.json +++ b/docs/install/iam_policy_us-gov.json @@ -41,7 +41,8 @@ "elasticloadbalancing:DescribeTargetHealth", "elasticloadbalancing:DescribeTags", "elasticloadbalancing:DescribeTrustStores", - "elasticloadbalancing:DescribeListenerAttributes" + "elasticloadbalancing:DescribeListenerAttributes", + "elasticloadbalancing:DescribeCapacityReservation" ], "Resource": "*" }, @@ -213,7 +214,8 @@ "elasticloadbalancing:ModifyTargetGroup", "elasticloadbalancing:ModifyTargetGroupAttributes", "elasticloadbalancing:DeleteTargetGroup", - "elasticloadbalancing:ModifyListenerAttributes" + "elasticloadbalancing:ModifyListenerAttributes", + "elasticloadbalancing:ModifyCapacityReservation" ], "Resource": "*", "Condition": { diff --git a/go.mod b/go.mod index a042291590..1e05dbf7d8 100644 --- a/go.mod +++ b/go.mod @@ -4,19 +4,19 @@ go 1.22.8 require ( github.com/aws/aws-sdk-go v1.55.5 - github.com/aws/aws-sdk-go-v2 v1.32.3 + github.com/aws/aws-sdk-go-v2 v1.32.5 github.com/aws/aws-sdk-go-v2/config v1.27.27 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.11 github.com/aws/aws-sdk-go-v2/service/acm v1.28.4 github.com/aws/aws-sdk-go-v2/service/appmesh v1.27.7 github.com/aws/aws-sdk-go-v2/service/ec2 v1.173.0 - github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2 v1.41.0 + github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2 v1.42.0 github.com/aws/aws-sdk-go-v2/service/resourcegroupstaggingapi v1.23.3 github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.31.7 github.com/aws/aws-sdk-go-v2/service/shield v1.27.3 github.com/aws/aws-sdk-go-v2/service/wafregional v1.23.3 github.com/aws/aws-sdk-go-v2/service/wafv2 v1.51.4 - github.com/aws/smithy-go v1.22.0 + github.com/aws/smithy-go v1.22.1 github.com/evanphx/json-patch v5.7.0+incompatible github.com/gavv/httpexpect/v2 v2.9.0 github.com/go-logr/logr v1.4.1 @@ -57,8 +57,8 @@ require ( github.com/andybalholm/brotli v1.0.4 // indirect github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535 // indirect github.com/aws/aws-sdk-go-v2/credentials v1.17.27 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.22 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.22 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.24 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.24 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.3 // indirect github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.17 // indirect diff --git a/go.sum b/go.sum index c8386e14df..b13e212624 100644 --- a/go.sum +++ b/go.sum @@ -40,6 +40,8 @@ github.com/aws/aws-sdk-go v1.55.5 h1:KKUZBfBoyqy5d3swXyiC7Q76ic40rYcbqH7qjh59kzU github.com/aws/aws-sdk-go v1.55.5/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU= github.com/aws/aws-sdk-go-v2 v1.32.3 h1:T0dRlFBKcdaUPGNtkBSwHZxrtis8CQU17UpNBZYd0wk= github.com/aws/aws-sdk-go-v2 v1.32.3/go.mod h1:2SK5n0a2karNTv5tbP1SjsX0uhttou00v/HpXKM1ZUo= +github.com/aws/aws-sdk-go-v2 v1.32.5 h1:U8vdWJuY7ruAkzaOdD7guwJjD06YSKmnKCJs7s3IkIo= +github.com/aws/aws-sdk-go-v2 v1.32.5/go.mod h1:P5WJBrYqqbWVaOxgH0X/FYYD47/nooaPOZPlQdmiN2U= github.com/aws/aws-sdk-go-v2/config v1.27.27 h1:HdqgGt1OAP0HkEDDShEl0oSYa9ZZBSOmKpdpsDMdO90= github.com/aws/aws-sdk-go-v2/config v1.27.27/go.mod h1:MVYamCg76dFNINkZFu4n4RjDixhVr51HLj4ErWzrVwg= github.com/aws/aws-sdk-go-v2/credentials v1.17.27 h1:2raNba6gr2IfA0eqqiP2XiQ0UVOpGPgDSi0I9iAP+UI= @@ -48,8 +50,12 @@ github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.11 h1:KreluoV8FZDEtI6Co2xuNk github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.11/go.mod h1:SeSUYBLsMYFoRvHE0Tjvn7kbxaUhl75CJi1sbfhMxkU= github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.22 h1:Jw50LwEkVjuVzE1NzkhNKkBf9cRN7MtE1F/b2cOKTUM= github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.22/go.mod h1:Y/SmAyPcOTmpeVaWSzSKiILfXTVJwrGmYZhcRbhWuEY= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.24 h1:4usbeaes3yJnCFC7kfeyhkdkPtoRYPa/hTmCqMpKpLI= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.24/go.mod h1:5CI1JemjVwde8m2WG3cz23qHKPOxbpkq0HaoreEgLIY= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.22 h1:981MHwBaRZM7+9QSR6XamDzF/o7ouUGxFzr+nVSIhrs= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.22/go.mod h1:1RA1+aBEfn+CAB/Mh0MB6LsdCYCnjZm7tKXtnk499ZQ= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.24 h1:N1zsICrQglfzaBnrfM0Ys00860C+QFwu6u/5+LomP+o= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.24/go.mod h1:dCn9HbJ8+K31i8IQ8EWmWj0EiIk0+vKiHNMxTTYveAg= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 h1:hT8rVHwugYE2lEfdFE0QWVo81lF7jMrYJVDWI+f+VxU= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0/go.mod h1:8tu/lYfQfFe6IGnaOdrpVgEL2IrrDOf6/m9RQum4NkY= github.com/aws/aws-sdk-go-v2/service/acm v1.28.4 h1:wiW1Y6/1lysA0eJZRq0I53YYKuV9MNAzL15z2eZRlEE= @@ -58,8 +64,8 @@ github.com/aws/aws-sdk-go-v2/service/appmesh v1.27.7 h1:q44a6kysAfej9zZwRnraOg9s github.com/aws/aws-sdk-go-v2/service/appmesh v1.27.7/go.mod h1:ZYSmrgAMp0rTCHH+SGsoxZo+PPbgsDqBzewTp3tSJ60= github.com/aws/aws-sdk-go-v2/service/ec2 v1.173.0 h1:ta62lid9JkIpKZtZZXSj6rP2AqY5x1qYGq53ffxqD9Q= github.com/aws/aws-sdk-go-v2/service/ec2 v1.173.0/go.mod h1:o6QDjdVKpP5EF0dp/VlvqckzuSDATr1rLdHt3A5m0YY= -github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2 v1.41.0 h1:W+xNfPS8dQ8YoszdkHqTDYIgCrWJvyUU/ZgdJ0frCRE= -github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2 v1.41.0/go.mod h1:6WuvTcPjB9gff93p/2LNBg09d8xK99jpVO6+fRSCKEU= +github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2 v1.42.0 h1:C4/D90/j3EF/SokpC4HO1aPMkZV1dgqUbmejdpxQiAE= +github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2 v1.42.0/go.mod h1:pZP3I+Ts+XuhJJtZE49+ABVjfxm7u9/hxcNUYSpY3OE= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.3 h1:dT3MqvGhSoaIhRseqw2I0yH81l7wiR2vjs57O51EAm8= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.3/go.mod h1:GlAeCkHwugxdHaueRr4nhPuY+WW+gR8UjlcqzPr1SPI= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.17 h1:HGErhhrxZlQ044RiM+WdoZxp0p+EGM62y3L6pwA4olE= @@ -82,6 +88,8 @@ github.com/aws/aws-sdk-go-v2/service/wafv2 v1.51.4 h1:1khBA5uryBRJoCb4G2iR5RT06B github.com/aws/aws-sdk-go-v2/service/wafv2 v1.51.4/go.mod h1:QpFImaPGKNwa+MiZ+oo6LbV1PVQBapc0CnrAMRScoxM= github.com/aws/smithy-go v1.22.0 h1:uunKnWlcoL3zO7q+gG2Pk53joueEOsnNB28QdMsmiMM= github.com/aws/smithy-go v1.22.0/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= +github.com/aws/smithy-go v1.22.1 h1:/HPHZQ0g7f4eUeK6HKglFz8uwVfZKgoI25rb/J+dnro= +github.com/aws/smithy-go v1.22.1/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= diff --git a/helm/aws-load-balancer-controller/crds/crds.yaml b/helm/aws-load-balancer-controller/crds/crds.yaml index 048d51e841..b7b7845302 100644 --- a/helm/aws-load-balancer-controller/crds/crds.yaml +++ b/helm/aws-load-balancer-controller/crds/crds.yaml @@ -132,6 +132,18 @@ spec: - value type: object type: array + minimumLoadBalancerCapacity: + description: MinimumLoadBalancerCapacity define the capacity reservation + for LoadBalancers for all Ingress that belong to IngressClass with + this IngressClassParams. + properties: + capacityUnits: + description: The Capacity Units Value. + format: int32 + type: integer + required: + - capacityUnits + type: object namespaceSelector: description: |- NamespaceSelector restrict the namespaces of Ingresses that are allowed to specify the IngressClass with this IngressClassParams. diff --git a/helm/aws-load-balancer-controller/templates/deployment.yaml b/helm/aws-load-balancer-controller/templates/deployment.yaml index da672ab34d..4506d489e8 100644 --- a/helm/aws-load-balancer-controller/templates/deployment.yaml +++ b/helm/aws-load-balancer-controller/templates/deployment.yaml @@ -110,6 +110,9 @@ spec: {{- if .Values.targetgroupbindingMaxExponentialBackoffDelay }} - --targetgroupbinding-max-exponential-backoff-delay={{ .Values.targetgroupbindingMaxExponentialBackoffDelay }} {{- end }} + {{- if .Values.lbStabilizationMonitorInterval }} + - --lb-stabilization-monitor-interval={{ .Values.lbStabilizationMonitorInterval }} + {{- end }} {{- if .Values.logLevel }} - --log-level={{ .Values.logLevel }} {{- end }} diff --git a/helm/aws-load-balancer-controller/values.yaml b/helm/aws-load-balancer-controller/values.yaml index a33542b8b5..d7fcb02fab 100644 --- a/helm/aws-load-balancer-controller/values.yaml +++ b/helm/aws-load-balancer-controller/values.yaml @@ -233,6 +233,9 @@ targetgroupbindingMaxConcurrentReconciles: # Maximum duration of exponential backoff for targetGroupBinding reconcile failures targetgroupbindingMaxExponentialBackoffDelay: +# Interval at which the controller monitors the state of load balancer after creation for stabilization +lbStabilizationMonitorInterval: + # Period at which the controller forces the repopulation of its local object stores. (default 10h0m0s) syncPeriod: @@ -358,6 +361,7 @@ controllerConfig: # SubnetsClusterTagCheck: true # NLBHealthCheckAdvancedConfig: true # ALBSingleSubnet: false + # LBCapacityReservation: true certDiscovery: allowedCertificateAuthorityARNs: "" # empty means all CAs are in scope diff --git a/pkg/annotations/constants.go b/pkg/annotations/constants.go index 3bad8a76b5..b2bc9aad1e 100644 --- a/pkg/annotations/constants.go +++ b/pkg/annotations/constants.go @@ -13,50 +13,51 @@ const ( AnnotationPrefixIngress = "alb.ingress.kubernetes.io" // Ingress annotation suffixes - IngressSuffixLoadBalancerName = "load-balancer-name" - IngressSuffixGroupName = "group.name" - IngressSuffixGroupOrder = "group.order" - IngressSuffixTags = "tags" - IngressSuffixIPAddressType = "ip-address-type" - IngressSuffixScheme = "scheme" - IngressSuffixSubnets = "subnets" - IngressSuffixCustomerOwnedIPv4Pool = "customer-owned-ipv4-pool" - IngressSuffixLoadBalancerAttributes = "load-balancer-attributes" - IngressSuffixWAFv2ACLARN = "wafv2-acl-arn" - IngressSuffixWAFACLID = "waf-acl-id" - IngressSuffixWebACLID = "web-acl-id" // deprecated, use "waf-acl-id" instead. - IngressSuffixShieldAdvancedProtection = "shield-advanced-protection" - IngressSuffixSecurityGroups = "security-groups" - IngressSuffixListenPorts = "listen-ports" - IngressSuffixSSLRedirect = "ssl-redirect" - IngressSuffixInboundCIDRs = "inbound-cidrs" - IngressSuffixCertificateARN = "certificate-arn" - IngressSuffixSSLPolicy = "ssl-policy" - IngressSuffixTargetType = "target-type" - IngressSuffixBackendProtocol = "backend-protocol" - IngressSuffixBackendProtocolVersion = "backend-protocol-version" - IngressSuffixTargetGroupAttributes = "target-group-attributes" - IngressSuffixHealthCheckPort = "healthcheck-port" - IngressSuffixHealthCheckProtocol = "healthcheck-protocol" - IngressSuffixHealthCheckPath = "healthcheck-path" - IngressSuffixHealthCheckIntervalSeconds = "healthcheck-interval-seconds" - IngressSuffixHealthCheckTimeoutSeconds = "healthcheck-timeout-seconds" - IngressSuffixHealthyThresholdCount = "healthy-threshold-count" - IngressSuffixUnhealthyThresholdCount = "unhealthy-threshold-count" - IngressSuffixSuccessCodes = "success-codes" - IngressSuffixAuthType = "auth-type" - IngressSuffixAuthIDPCognito = "auth-idp-cognito" - IngressSuffixAuthIDPOIDC = "auth-idp-oidc" - IngressSuffixAuthOnUnauthenticatedRequest = "auth-on-unauthenticated-request" - IngressSuffixAuthScope = "auth-scope" - IngressSuffixAuthSessionCookie = "auth-session-cookie" - IngressSuffixAuthSessionTimeout = "auth-session-timeout" - IngressSuffixTargetNodeLabels = "target-node-labels" - IngressSuffixManageSecurityGroupRules = "manage-backend-security-group-rules" - IngressSuffixMutualAuthentication = "mutual-authentication" - IngressSuffixSecurityGroupPrefixLists = "security-group-prefix-lists" - IngressSuffixlsAttsAnnotationPrefix = "listener-attributes" - IngressLBSuffixMultiClusterTargetGroup = "multi-cluster-target-group" + IngressSuffixLoadBalancerName = "load-balancer-name" + IngressSuffixGroupName = "group.name" + IngressSuffixGroupOrder = "group.order" + IngressSuffixTags = "tags" + IngressSuffixIPAddressType = "ip-address-type" + IngressSuffixScheme = "scheme" + IngressSuffixSubnets = "subnets" + IngressSuffixCustomerOwnedIPv4Pool = "customer-owned-ipv4-pool" + IngressSuffixLoadBalancerAttributes = "load-balancer-attributes" + IngressSuffixWAFv2ACLARN = "wafv2-acl-arn" + IngressSuffixWAFACLID = "waf-acl-id" + IngressSuffixWebACLID = "web-acl-id" // deprecated, use "waf-acl-id" instead. + IngressSuffixShieldAdvancedProtection = "shield-advanced-protection" + IngressSuffixSecurityGroups = "security-groups" + IngressSuffixListenPorts = "listen-ports" + IngressSuffixSSLRedirect = "ssl-redirect" + IngressSuffixInboundCIDRs = "inbound-cidrs" + IngressSuffixCertificateARN = "certificate-arn" + IngressSuffixSSLPolicy = "ssl-policy" + IngressSuffixTargetType = "target-type" + IngressSuffixBackendProtocol = "backend-protocol" + IngressSuffixBackendProtocolVersion = "backend-protocol-version" + IngressSuffixTargetGroupAttributes = "target-group-attributes" + IngressSuffixHealthCheckPort = "healthcheck-port" + IngressSuffixHealthCheckProtocol = "healthcheck-protocol" + IngressSuffixHealthCheckPath = "healthcheck-path" + IngressSuffixHealthCheckIntervalSeconds = "healthcheck-interval-seconds" + IngressSuffixHealthCheckTimeoutSeconds = "healthcheck-timeout-seconds" + IngressSuffixHealthyThresholdCount = "healthy-threshold-count" + IngressSuffixUnhealthyThresholdCount = "unhealthy-threshold-count" + IngressSuffixSuccessCodes = "success-codes" + IngressSuffixAuthType = "auth-type" + IngressSuffixAuthIDPCognito = "auth-idp-cognito" + IngressSuffixAuthIDPOIDC = "auth-idp-oidc" + IngressSuffixAuthOnUnauthenticatedRequest = "auth-on-unauthenticated-request" + IngressSuffixAuthScope = "auth-scope" + IngressSuffixAuthSessionCookie = "auth-session-cookie" + IngressSuffixAuthSessionTimeout = "auth-session-timeout" + IngressSuffixTargetNodeLabels = "target-node-labels" + IngressSuffixManageSecurityGroupRules = "manage-backend-security-group-rules" + IngressSuffixMutualAuthentication = "mutual-authentication" + IngressSuffixSecurityGroupPrefixLists = "security-group-prefix-lists" + IngressSuffixlsAttsAnnotationPrefix = "listener-attributes" + IngressLBSuffixMultiClusterTargetGroup = "multi-cluster-target-group" + IngressSuffixLoadBalancerCapacityReservation = "minimum-load-balancer-capacity" // NLB annotation suffixes // prefixes service.beta.kubernetes.io, service.kubernetes.io @@ -101,4 +102,5 @@ const ( SvcLBSuffixMultiClusterTargetGroup = "aws-load-balancer-multi-cluster-target-group" ScvLBSuffixEnablePrefixForIpv6SourceNat = "aws-load-balancer-enable-prefix-for-ipv6-source-nat" ScvLBSuffixSourceNatIpv6Prefixes = "aws-load-balancer-source-nat-ipv6-prefixes" + SvcLBSuffixLoadBalancerCapacityReservation = "aws-load-balancer-minimum-load-balancer-capacity" ) diff --git a/pkg/aws/services/elbv2.go b/pkg/aws/services/elbv2.go index 4b49990656..b89b83d00c 100644 --- a/pkg/aws/services/elbv2.go +++ b/pkg/aws/services/elbv2.go @@ -59,6 +59,8 @@ type ELBV2 interface { AddListenerCertificatesWithContext(ctx context.Context, input *elasticloadbalancingv2.AddListenerCertificatesInput) (*elasticloadbalancingv2.AddListenerCertificatesOutput, error) DescribeListenerAttributesWithContext(ctx context.Context, input *elasticloadbalancingv2.DescribeListenerAttributesInput) (*elasticloadbalancingv2.DescribeListenerAttributesOutput, error) ModifyListenerAttributesWithContext(ctx context.Context, input *elasticloadbalancingv2.ModifyListenerAttributesInput) (*elasticloadbalancingv2.ModifyListenerAttributesOutput, error) + ModifyCapacityReservationWithContext(ctx context.Context, input *elasticloadbalancingv2.ModifyCapacityReservationInput) (*elasticloadbalancingv2.ModifyCapacityReservationOutput, error) + DescribeCapacityReservationWithContext(ctx context.Context, input *elasticloadbalancingv2.DescribeCapacityReservationInput) (*elasticloadbalancingv2.DescribeCapacityReservationOutput, error) } func NewELBV2(awsClientsProvider provider.AWSClientsProvider) ELBV2 { @@ -440,3 +442,19 @@ func (c *elbv2Client) ModifyListenerAttributesWithContext(ctx context.Context, i } return client.ModifyListenerAttributes(ctx, input) } + +func (c *elbv2Client) ModifyCapacityReservationWithContext(ctx context.Context, input *elasticloadbalancingv2.ModifyCapacityReservationInput) (*elasticloadbalancingv2.ModifyCapacityReservationOutput, error) { + client, err := c.awsClientsProvider.GetELBv2Client(ctx, "ModifyCapacityReservation") + if err != nil { + return nil, err + } + return client.ModifyCapacityReservation(ctx, input) +} + +func (c *elbv2Client) DescribeCapacityReservationWithContext(ctx context.Context, input *elasticloadbalancingv2.DescribeCapacityReservationInput) (*elasticloadbalancingv2.DescribeCapacityReservationOutput, error) { + client, err := c.awsClientsProvider.GetELBv2Client(ctx, "DescribeCapacityReservation") + if err != nil { + return nil, err + } + return client.DescribeCapacityReservation(ctx, input) +} diff --git a/pkg/aws/services/elbv2_mocks.go b/pkg/aws/services/elbv2_mocks.go index 42c82464c6..805c0e8ac4 100644 --- a/pkg/aws/services/elbv2_mocks.go +++ b/pkg/aws/services/elbv2_mocks.go @@ -201,6 +201,21 @@ func (mr *MockELBV2MockRecorder) DeregisterTargetsWithContext(arg0, arg1 interfa return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeregisterTargetsWithContext", reflect.TypeOf((*MockELBV2)(nil).DeregisterTargetsWithContext), arg0, arg1) } +// DescribeCapacityReservationWithContext mocks base method. +func (m *MockELBV2) DescribeCapacityReservationWithContext(arg0 context.Context, arg1 *elasticloadbalancingv2.DescribeCapacityReservationInput) (*elasticloadbalancingv2.DescribeCapacityReservationOutput, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DescribeCapacityReservationWithContext", arg0, arg1) + ret0, _ := ret[0].(*elasticloadbalancingv2.DescribeCapacityReservationOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// DescribeCapacityReservationWithContext indicates an expected call of DescribeCapacityReservationWithContext. +func (mr *MockELBV2MockRecorder) DescribeCapacityReservationWithContext(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DescribeCapacityReservationWithContext", reflect.TypeOf((*MockELBV2)(nil).DescribeCapacityReservationWithContext), arg0, arg1) +} + // DescribeListenerAttributesWithContext mocks base method. func (m *MockELBV2) DescribeListenerAttributesWithContext(arg0 context.Context, arg1 *elasticloadbalancingv2.DescribeListenerAttributesInput) (*elasticloadbalancingv2.DescribeListenerAttributesOutput, error) { m.ctrl.T.Helper() @@ -426,6 +441,21 @@ func (mr *MockELBV2MockRecorder) DescribeTrustStoresWithContext(arg0, arg1 inter return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DescribeTrustStoresWithContext", reflect.TypeOf((*MockELBV2)(nil).DescribeTrustStoresWithContext), arg0, arg1) } +// ModifyCapacityReservationWithContext mocks base method. +func (m *MockELBV2) ModifyCapacityReservationWithContext(arg0 context.Context, arg1 *elasticloadbalancingv2.ModifyCapacityReservationInput) (*elasticloadbalancingv2.ModifyCapacityReservationOutput, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ModifyCapacityReservationWithContext", arg0, arg1) + ret0, _ := ret[0].(*elasticloadbalancingv2.ModifyCapacityReservationOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ModifyCapacityReservationWithContext indicates an expected call of ModifyCapacityReservationWithContext. +func (mr *MockELBV2MockRecorder) ModifyCapacityReservationWithContext(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ModifyCapacityReservationWithContext", reflect.TypeOf((*MockELBV2)(nil).ModifyCapacityReservationWithContext), arg0, arg1) +} + // ModifyListenerAttributesWithContext mocks base method. func (m *MockELBV2) ModifyListenerAttributesWithContext(arg0 context.Context, arg1 *elasticloadbalancingv2.ModifyListenerAttributesInput) (*elasticloadbalancingv2.ModifyListenerAttributesOutput, error) { m.ctrl.T.Helper() diff --git a/pkg/config/controller_config.go b/pkg/config/controller_config.go index 7d6b42e44d..6de85e1fd7 100644 --- a/pkg/config/controller_config.go +++ b/pkg/config/controller_config.go @@ -22,6 +22,7 @@ const ( flagServiceMaxConcurrentReconciles = "service-max-concurrent-reconciles" flagTargetGroupBindingMaxConcurrentReconciles = "targetgroupbinding-max-concurrent-reconciles" flagTargetGroupBindingMaxExponentialBackoffDelay = "targetgroupbinding-max-exponential-backoff-delay" + flagLbStabilizationMonitorInterval = "lb-stabilization-monitor-interval" flagDefaultSSLPolicy = "default-ssl-policy" flagEnableBackendSG = "enable-backend-security-group" flagBackendSecurityGroup = "backend-security-group" @@ -34,6 +35,7 @@ const ( defaultEnableBackendSG = true defaultEnableEndpointSlices = false defaultDisableRestrictedSGRules = false + defaultLbStabilizationMonitorInterval = time.Second * 120 ) var ( @@ -102,6 +104,9 @@ type ControllerConfig struct { // DisableRestrictedSGRules specifies whether to use restricted security group rules DisableRestrictedSGRules bool + // LBStabilizationMonitorInterval specifies the duration of interval to monitor the load balancer state for stabilization + LBStabilizationMonitorInterval time.Duration + FeatureGates FeatureGates } @@ -122,6 +127,8 @@ func (cfg *ControllerConfig) BindFlags(fs *pflag.FlagSet) { "Maximum number of concurrently running reconcile loops for targetGroupBinding") fs.DurationVar(&cfg.TargetGroupBindingMaxExponentialBackoffDelay, flagTargetGroupBindingMaxExponentialBackoffDelay, defaultMaxExponentialBackoffDelay, "Maximum duration of exponential backoff for targetGroupBinding reconcile failures") + fs.DurationVar(&cfg.LBStabilizationMonitorInterval, flagLbStabilizationMonitorInterval, defaultLbStabilizationMonitorInterval, + "Duration of interval to monitor the load balancer state for stabilization") fs.StringVar(&cfg.DefaultSSLPolicy, flagDefaultSSLPolicy, defaultSSLPolicy, "Default SSL policy for load balancers listeners") fs.BoolVar(&cfg.EnableBackendSecurityGroup, flagEnableBackendSG, defaultEnableBackendSG, diff --git a/pkg/config/feature_gates.go b/pkg/config/feature_gates.go index 6835f907f6..e220faff7a 100644 --- a/pkg/config/feature_gates.go +++ b/pkg/config/feature_gates.go @@ -22,6 +22,7 @@ const ( NLBHealthCheckAdvancedConfig Feature = "NLBHealthCheckAdvancedConfig" NLBSecurityGroup Feature = "NLBSecurityGroup" ALBSingleSubnet Feature = "ALBSingleSubnet" + LBCapacityReservation Feature = "LBCapacityReservation" ) type FeatureGates interface { @@ -60,6 +61,7 @@ func NewFeatureGates() FeatureGates { NLBHealthCheckAdvancedConfig: true, NLBSecurityGroup: true, ALBSingleSubnet: false, + LBCapacityReservation: true, }, } } diff --git a/pkg/deploy/elbv2/load_balancer_capacity_reservation_reconciler.go b/pkg/deploy/elbv2/load_balancer_capacity_reservation_reconciler.go new file mode 100644 index 0000000000..0a42db648e --- /dev/null +++ b/pkg/deploy/elbv2/load_balancer_capacity_reservation_reconciler.go @@ -0,0 +1,112 @@ +package elbv2 + +import ( + "context" + awssdk "github.com/aws/aws-sdk-go-v2/aws" + elbv2sdk "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2" + elbv2types "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2/types" + "github.com/go-logr/logr" + "reflect" + "sigs.k8s.io/aws-load-balancer-controller/pkg/aws/services" + "sigs.k8s.io/aws-load-balancer-controller/pkg/config" + elbv2model "sigs.k8s.io/aws-load-balancer-controller/pkg/model/elbv2" +) + +// reconciler for LoadBalancer Capacity Reservation +type LoadBalancerCapacityReservationReconciler interface { + // Reconcile loadBalancer capacity reservation + Reconcile(ctx context.Context, resLB *elbv2model.LoadBalancer, sdkLB LoadBalancerWithTags) error +} + +// NewDefaultLoadBalancerCapacityReservationReconciler constructs new defaultLoadBalancerCapacityReservationReconciler. +func NewDefaultLoadBalancerCapacityReservationReconciler(elbv2Client services.ELBV2, featureGates config.FeatureGates, logger logr.Logger) *defaultLoadBalancerCapacityReservationReconciler { + return &defaultLoadBalancerCapacityReservationReconciler{ + elbv2Client: elbv2Client, + logger: logger, + featureGates: featureGates, + } +} + +var _ LoadBalancerCapacityReservationReconciler = &defaultLoadBalancerCapacityReservationReconciler{} + +// default implementation for LoadBalancerCapacityReservationReconciler +type defaultLoadBalancerCapacityReservationReconciler struct { + elbv2Client services.ELBV2 + logger logr.Logger + featureGates config.FeatureGates +} + +func (r *defaultLoadBalancerCapacityReservationReconciler) Reconcile(ctx context.Context, resLB *elbv2model.LoadBalancer, sdkLB LoadBalancerWithTags) error { + desiredCapacityReservation := resLB.Spec.MinimumLoadBalancerCapacity + // If the annotation is missing or not set, skip the capacity reservation + if desiredCapacityReservation == nil { + return nil + } + //If the value of desired capacityUnits is zero then set desiredCapacityReservation to nil to reset the capacity + if desiredCapacityReservation.CapacityUnits == 0 { + desiredCapacityReservation = nil + } + currentCapacityReservation, err := r.getCurrentCapacityReservation(ctx, sdkLB) + if err != nil { + return err + } + isLBCapacityReservationDrifted := !reflect.DeepEqual(desiredCapacityReservation, currentCapacityReservation) + if !isLBCapacityReservationDrifted { + return nil + } + if desiredCapacityReservation == nil { + //If the value of desired capacityUnits is nil then reset capacity + req := &elbv2sdk.ModifyCapacityReservationInput{ + LoadBalancerArn: sdkLB.LoadBalancer.LoadBalancerArn, + ResetCapacityReservation: awssdk.Bool(true), + } + + r.logger.Info("resetting loadBalancer capacity reservation", + "stackID", resLB.Stack().StackID(), + "resourceID", resLB.ID(), + "arn", awssdk.ToString(sdkLB.LoadBalancer.LoadBalancerArn)) + if _, err := r.elbv2Client.ModifyCapacityReservationWithContext(ctx, req); err != nil { + return err + } + r.logger.Info("reset successful for loadBalancer capacity reservation", + "stackID", resLB.Stack().StackID(), + "resourceID", resLB.ID(), + "arn", awssdk.ToString(sdkLB.LoadBalancer.LoadBalancerArn)) + } else { + req := &elbv2sdk.ModifyCapacityReservationInput{ + LoadBalancerArn: sdkLB.LoadBalancer.LoadBalancerArn, + MinimumLoadBalancerCapacity: &elbv2types.MinimumLoadBalancerCapacity{ + CapacityUnits: awssdk.Int32(desiredCapacityReservation.CapacityUnits), + }, + } + r.logger.Info("modifying loadBalancer capacity reservation", + "stackID", resLB.Stack().StackID(), + "resourceID", resLB.ID(), + "arn", awssdk.ToString(sdkLB.LoadBalancer.LoadBalancerArn), + "change", desiredCapacityReservation) + if _, err := r.elbv2Client.ModifyCapacityReservationWithContext(ctx, req); err != nil { + return err + } + r.logger.Info("modified loadBalancer capacity reservation", + "stackID", resLB.Stack().StackID(), + "resourceID", resLB.ID(), + "arn", awssdk.ToString(sdkLB.LoadBalancer.LoadBalancerArn)) + } + return nil +} + +func (r *defaultLoadBalancerCapacityReservationReconciler) getCurrentCapacityReservation(ctx context.Context, sdkLB LoadBalancerWithTags) (*elbv2model.MinimumLoadBalancerCapacity, error) { + req := &elbv2sdk.DescribeCapacityReservationInput{ + LoadBalancerArn: sdkLB.LoadBalancer.LoadBalancerArn, + } + resp, err := r.elbv2Client.DescribeCapacityReservationWithContext(ctx, req) + if err != nil { + return nil, err + } + var sdkLBMinimumCapacity = &elbv2model.MinimumLoadBalancerCapacity{} + if (resp.CapacityReservationState == nil || len(resp.CapacityReservationState) == 0) && resp.MinimumLoadBalancerCapacity == nil { + return nil, nil + } + sdkLBMinimumCapacity.CapacityUnits = awssdk.ToInt32(resp.MinimumLoadBalancerCapacity.CapacityUnits) + return sdkLBMinimumCapacity, nil +} diff --git a/pkg/deploy/elbv2/load_balancer_capacity_reservation_reconciler_test.go b/pkg/deploy/elbv2/load_balancer_capacity_reservation_reconciler_test.go new file mode 100644 index 0000000000..4e258cf087 --- /dev/null +++ b/pkg/deploy/elbv2/load_balancer_capacity_reservation_reconciler_test.go @@ -0,0 +1,328 @@ +package elbv2 + +import ( + "context" + awssdk "github.com/aws/aws-sdk-go-v2/aws" + elbv2sdk "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2" + elbv2types "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2/types" + "github.com/go-logr/logr" + "github.com/golang/mock/gomock" + "github.com/pkg/errors" + "github.com/stretchr/testify/assert" + "sigs.k8s.io/aws-load-balancer-controller/pkg/aws/services" + "sigs.k8s.io/aws-load-balancer-controller/pkg/config" + coremodel "sigs.k8s.io/aws-load-balancer-controller/pkg/model/core" + elbv2model "sigs.k8s.io/aws-load-balancer-controller/pkg/model/elbv2" + "sigs.k8s.io/controller-runtime/pkg/log" + "testing" +) + +func Test_defaultLoadBalancerCapacityReservationReconciler_updateSDKLoadBalancerWithCapacityReservation(t *testing.T) { + type describeCapacityReservationWithContextCall struct { + req *elbv2sdk.DescribeCapacityReservationInput + resp *elbv2sdk.DescribeCapacityReservationOutput + err error + } + + type modifyCapacityReservationWithContextCall struct { + req *elbv2sdk.ModifyCapacityReservationInput + resp *elbv2sdk.ModifyCapacityReservationOutput + err error + } + + type fields struct { + describeCapacityReservationWithContextCalls []describeCapacityReservationWithContextCall + modifyCapacityReservationWithContextCalls []modifyCapacityReservationWithContextCall + } + type args struct { + sdkLB LoadBalancerWithTags + resLB *elbv2model.LoadBalancer + } + + stack := coremodel.NewDefaultStack(coremodel.StackID{Namespace: "namespace", Name: "name"}) + tests := []struct { + name string + featureGates map[config.Feature]bool + fields fields + args args + wantErr error + }{ + { + name: "capacity reservation should be updated", + featureGates: map[config.Feature]bool{ + config.LBCapacityReservation: true, + }, + fields: fields{ + describeCapacityReservationWithContextCalls: []describeCapacityReservationWithContextCall{ + { + req: &elbv2sdk.DescribeCapacityReservationInput{ + LoadBalancerArn: awssdk.String("my-arn"), + }, + resp: &elbv2sdk.DescribeCapacityReservationOutput{ + CapacityReservationState: nil, + MinimumLoadBalancerCapacity: nil, + }, + }, + }, + modifyCapacityReservationWithContextCalls: []modifyCapacityReservationWithContextCall{ + { + req: &elbv2sdk.ModifyCapacityReservationInput{ + LoadBalancerArn: awssdk.String("my-arn"), + MinimumLoadBalancerCapacity: &elbv2types.MinimumLoadBalancerCapacity{CapacityUnits: awssdk.Int32(1200)}, + }, + }, + }, + }, + args: args{ + sdkLB: LoadBalancerWithTags{ + LoadBalancer: &elbv2types.LoadBalancer{ + LoadBalancerArn: awssdk.String("my-arn"), + }, + }, + resLB: &elbv2model.LoadBalancer{ + ResourceMeta: coremodel.NewResourceMeta(stack, "AWS::ElasticLoadBalancingV2::LoadBalancer", "id-1"), + Spec: elbv2model.LoadBalancerSpec{ + MinimumLoadBalancerCapacity: &elbv2model.MinimumLoadBalancerCapacity{CapacityUnits: 1200}, + }, + }, + }, + }, + { + name: "reset capacity with zero value", + featureGates: map[config.Feature]bool{ + config.LBCapacityReservation: true, + }, + fields: fields{ + describeCapacityReservationWithContextCalls: []describeCapacityReservationWithContextCall{ + { + req: &elbv2sdk.DescribeCapacityReservationInput{ + LoadBalancerArn: awssdk.String("my-arn"), + }, + resp: &elbv2sdk.DescribeCapacityReservationOutput{ + CapacityReservationState: []elbv2types.ZonalCapacityReservationState{}, + MinimumLoadBalancerCapacity: &elbv2types.MinimumLoadBalancerCapacity{CapacityUnits: awssdk.Int32(1200)}, + }, + }, + }, + modifyCapacityReservationWithContextCalls: []modifyCapacityReservationWithContextCall{ + { + req: &elbv2sdk.ModifyCapacityReservationInput{ + LoadBalancerArn: awssdk.String("my-arn"), + ResetCapacityReservation: awssdk.Bool(true), + }, + }, + }, + }, + args: args{ + sdkLB: LoadBalancerWithTags{ + LoadBalancer: &elbv2types.LoadBalancer{ + LoadBalancerArn: awssdk.String("my-arn"), + }, + }, + resLB: &elbv2model.LoadBalancer{ + ResourceMeta: coremodel.NewResourceMeta(stack, "AWS::ElasticLoadBalancingV2::LoadBalancer", "id-1"), + Spec: elbv2model.LoadBalancerSpec{ + MinimumLoadBalancerCapacity: &elbv2model.MinimumLoadBalancerCapacity{CapacityUnits: 0}, + }, + }, + }, + }, + { + name: "no capacity reservation should be updated as their is no change", + featureGates: map[config.Feature]bool{ + config.LBCapacityReservation: true, + }, + fields: fields{ + describeCapacityReservationWithContextCalls: []describeCapacityReservationWithContextCall{ + { + req: &elbv2sdk.DescribeCapacityReservationInput{ + LoadBalancerArn: awssdk.String("my-arn"), + }, + resp: &elbv2sdk.DescribeCapacityReservationOutput{ + CapacityReservationState: []elbv2types.ZonalCapacityReservationState{}, + MinimumLoadBalancerCapacity: &elbv2types.MinimumLoadBalancerCapacity{CapacityUnits: awssdk.Int32(1200)}, + }, + }, + }, + modifyCapacityReservationWithContextCalls: nil, + }, + args: args{ + sdkLB: LoadBalancerWithTags{ + LoadBalancer: &elbv2types.LoadBalancer{ + LoadBalancerArn: awssdk.String("my-arn"), + }, + }, + resLB: &elbv2model.LoadBalancer{ + ResourceMeta: coremodel.NewResourceMeta(stack, "AWS::ElasticLoadBalancingV2::LoadBalancer", "id-1"), + Spec: elbv2model.LoadBalancerSpec{ + MinimumLoadBalancerCapacity: &elbv2model.MinimumLoadBalancerCapacity{CapacityUnits: 1200}, + }, + }, + }, + }, + { + name: "no capacity reservation should be updated as their is no specification on resource", + featureGates: map[config.Feature]bool{ + config.LBCapacityReservation: true, + }, + fields: fields{ + describeCapacityReservationWithContextCalls: nil, + modifyCapacityReservationWithContextCalls: nil, + }, + args: args{ + sdkLB: LoadBalancerWithTags{ + LoadBalancer: &elbv2types.LoadBalancer{ + LoadBalancerArn: awssdk.String("my-arn"), + }, + }, + resLB: &elbv2model.LoadBalancer{ + ResourceMeta: coremodel.NewResourceMeta(stack, "AWS::ElasticLoadBalancingV2::LoadBalancer", "id-1"), + Spec: elbv2model.LoadBalancerSpec{ + MinimumLoadBalancerCapacity: nil, + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + elbv2Client := services.NewMockELBV2(ctrl) + + for _, call := range tt.fields.describeCapacityReservationWithContextCalls { + elbv2Client.EXPECT().DescribeCapacityReservationWithContext(gomock.Any(), call.req).Return(call.resp, call.err) + } + for _, call := range tt.fields.modifyCapacityReservationWithContextCalls { + elbv2Client.EXPECT().ModifyCapacityReservationWithContext(gomock.Any(), call.req).Return(call.resp, call.err) + } + r := &defaultLoadBalancerCapacityReservationReconciler{ + elbv2Client: elbv2Client, + logger: logr.New(&log.NullLogSink{}), + } + err := r.Reconcile(context.Background(), tt.args.resLB, tt.args.sdkLB) + if tt.wantErr != nil { + assert.EqualError(t, err, tt.wantErr.Error()) + } else { + assert.NoError(t, err) + } + }) + } +} + +func Test_defaultLoadBalancerCapacityReservationReconciler_getCurrentLoadBalancerCapacityReservation(t *testing.T) { + type describeCapacityReservationWithContextCall struct { + req *elbv2sdk.DescribeCapacityReservationInput + resp *elbv2sdk.DescribeCapacityReservationOutput + err error + } + type fields struct { + describeCapacityReservationWithContextCalls []describeCapacityReservationWithContextCall + } + type args struct { + sdkLB LoadBalancerWithTags + } + tests := []struct { + name string + fields fields + args args + want *elbv2model.MinimumLoadBalancerCapacity + wantErr error + }{ + { + name: "no capacity reservation case", + fields: fields{ + describeCapacityReservationWithContextCalls: []describeCapacityReservationWithContextCall{ + { + req: &elbv2sdk.DescribeCapacityReservationInput{ + LoadBalancerArn: awssdk.String("my-arn"), + }, + resp: &elbv2sdk.DescribeCapacityReservationOutput{ + CapacityReservationState: nil, + MinimumLoadBalancerCapacity: nil, + }, + }, + }, + }, + args: args{ + sdkLB: LoadBalancerWithTags{ + LoadBalancer: &elbv2types.LoadBalancer{ + LoadBalancerArn: awssdk.String("my-arn"), + }, + Tags: nil, + }, + }, + want: nil, + }, + { + name: "standard case", + fields: fields{ + describeCapacityReservationWithContextCalls: []describeCapacityReservationWithContextCall{ + { + req: &elbv2sdk.DescribeCapacityReservationInput{ + LoadBalancerArn: awssdk.String("my-arn"), + }, + resp: &elbv2sdk.DescribeCapacityReservationOutput{ + CapacityReservationState: []elbv2types.ZonalCapacityReservationState{}, + MinimumLoadBalancerCapacity: &elbv2types.MinimumLoadBalancerCapacity{CapacityUnits: awssdk.Int32(3000)}, + }, + }, + }, + }, + args: args{ + sdkLB: LoadBalancerWithTags{ + LoadBalancer: &elbv2types.LoadBalancer{ + LoadBalancerArn: awssdk.String("my-arn"), + }, + Tags: nil, + }, + }, + want: &elbv2model.MinimumLoadBalancerCapacity{CapacityUnits: 3000}, + }, + { + name: "error case", + fields: fields{ + describeCapacityReservationWithContextCalls: []describeCapacityReservationWithContextCall{ + { + req: &elbv2sdk.DescribeCapacityReservationInput{ + LoadBalancerArn: awssdk.String("my-arn"), + }, + err: errors.New("some error"), + }, + }, + }, + args: args{ + sdkLB: LoadBalancerWithTags{ + LoadBalancer: &elbv2types.LoadBalancer{ + LoadBalancerArn: awssdk.String("my-arn"), + }, + Tags: nil, + }, + }, + wantErr: errors.New("some error"), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + elbv2Client := services.NewMockELBV2(ctrl) + featureGates := config.NewFeatureGates() + for _, call := range tt.fields.describeCapacityReservationWithContextCalls { + elbv2Client.EXPECT().DescribeCapacityReservationWithContext(gomock.Any(), call.req).Return(call.resp, call.err) + } + + r := &defaultLoadBalancerCapacityReservationReconciler{ + elbv2Client: elbv2Client, + featureGates: featureGates, + } + got, err := r.getCurrentCapacityReservation(context.Background(), tt.args.sdkLB) + if tt.wantErr != nil { + assert.EqualError(t, err, tt.wantErr.Error()) + } else { + assert.NoError(t, err) + assert.Equal(t, tt.want, got) + } + }) + } +} diff --git a/pkg/deploy/elbv2/load_balancer_manager.go b/pkg/deploy/elbv2/load_balancer_manager.go index 70df7f2268..0727c16439 100644 --- a/pkg/deploy/elbv2/load_balancer_manager.go +++ b/pkg/deploy/elbv2/load_balancer_manager.go @@ -9,6 +9,7 @@ import ( "github.com/go-logr/logr" "k8s.io/apimachinery/pkg/util/sets" "sigs.k8s.io/aws-load-balancer-controller/pkg/aws/services" + "sigs.k8s.io/aws-load-balancer-controller/pkg/config" "sigs.k8s.io/aws-load-balancer-controller/pkg/deploy/tracking" coremodel "sigs.k8s.io/aws-load-balancer-controller/pkg/model/core" elbv2model "sigs.k8s.io/aws-load-balancer-controller/pkg/model/elbv2" @@ -16,7 +17,7 @@ import ( // LoadBalancerManager is responsible for create/update/delete LoadBalancer resources. type LoadBalancerManager interface { - Create(ctx context.Context, resLB *elbv2model.LoadBalancer) (elbv2model.LoadBalancerStatus, error) + Create(ctx context.Context, resLB *elbv2model.LoadBalancer) (elbv2model.LoadBalancerStatus, LoadBalancerWithTags, error) Update(ctx context.Context, resLB *elbv2model.LoadBalancer, sdkLB LoadBalancerWithTags) (elbv2model.LoadBalancerStatus, error) @@ -25,14 +26,16 @@ type LoadBalancerManager interface { // NewDefaultLoadBalancerManager constructs new defaultLoadBalancerManager. func NewDefaultLoadBalancerManager(elbv2Client services.ELBV2, trackingProvider tracking.Provider, - taggingManager TaggingManager, externalManagedTags []string, logger logr.Logger) *defaultLoadBalancerManager { + taggingManager TaggingManager, externalManagedTags []string, featureGates config.FeatureGates, logger logr.Logger) *defaultLoadBalancerManager { return &defaultLoadBalancerManager{ - elbv2Client: elbv2Client, - trackingProvider: trackingProvider, - taggingManager: taggingManager, - attributesReconciler: NewDefaultLoadBalancerAttributeReconciler(elbv2Client, logger), - externalManagedTags: externalManagedTags, - logger: logger, + elbv2Client: elbv2Client, + trackingProvider: trackingProvider, + taggingManager: taggingManager, + attributesReconciler: NewDefaultLoadBalancerAttributeReconciler(elbv2Client, logger), + capacityReservationReconciler: NewDefaultLoadBalancerCapacityReservationReconciler(elbv2Client, featureGates, logger), + externalManagedTags: externalManagedTags, + featureGates: featureGates, + logger: logger, } } @@ -40,19 +43,20 @@ var _ LoadBalancerManager = &defaultLoadBalancerManager{} // defaultLoadBalancerManager implement LoadBalancerManager type defaultLoadBalancerManager struct { - elbv2Client services.ELBV2 - trackingProvider tracking.Provider - taggingManager TaggingManager - attributesReconciler LoadBalancerAttributeReconciler - externalManagedTags []string - - logger logr.Logger + elbv2Client services.ELBV2 + trackingProvider tracking.Provider + taggingManager TaggingManager + attributesReconciler LoadBalancerAttributeReconciler + capacityReservationReconciler LoadBalancerCapacityReservationReconciler + externalManagedTags []string + featureGates config.FeatureGates + logger logr.Logger } -func (m *defaultLoadBalancerManager) Create(ctx context.Context, resLB *elbv2model.LoadBalancer) (elbv2model.LoadBalancerStatus, error) { +func (m *defaultLoadBalancerManager) Create(ctx context.Context, resLB *elbv2model.LoadBalancer) (elbv2model.LoadBalancerStatus, LoadBalancerWithTags, error) { req, err := buildSDKCreateLoadBalancerInput(resLB.Spec) if err != nil { - return elbv2model.LoadBalancerStatus{}, err + return elbv2model.LoadBalancerStatus{}, LoadBalancerWithTags{}, err } lbTags := m.trackingProvider.ResourceTags(resLB.Stack(), resLB, resLB.Spec.Tags) req.Tags = convertTagsToSDKTags(lbTags) @@ -62,7 +66,7 @@ func (m *defaultLoadBalancerManager) Create(ctx context.Context, resLB *elbv2mod "resourceID", resLB.ID()) resp, err := m.elbv2Client.CreateLoadBalancerWithContext(ctx, req) if err != nil { - return elbv2model.LoadBalancerStatus{}, err + return elbv2model.LoadBalancerStatus{}, LoadBalancerWithTags{}, err } sdkLB := LoadBalancerWithTags{ LoadBalancer: &resp.LoadBalancers[0], @@ -73,16 +77,15 @@ func (m *defaultLoadBalancerManager) Create(ctx context.Context, resLB *elbv2mod "resourceID", resLB.ID(), "arn", awssdk.ToString(sdkLB.LoadBalancer.LoadBalancerArn)) if err := m.attributesReconciler.Reconcile(ctx, resLB, sdkLB); err != nil { - return elbv2model.LoadBalancerStatus{}, err + return elbv2model.LoadBalancerStatus{}, LoadBalancerWithTags{}, err } if resLB.Spec.Type == elbv2model.LoadBalancerTypeNetwork && resLB.Spec.SecurityGroupsInboundRulesOnPrivateLink != nil { if err := m.updateSDKLoadBalancerWithSecurityGroups(ctx, resLB, sdkLB); err != nil { - return elbv2model.LoadBalancerStatus{}, err + return elbv2model.LoadBalancerStatus{}, LoadBalancerWithTags{}, err } } - - return buildResLoadBalancerStatus(sdkLB), nil + return buildResLoadBalancerStatus(sdkLB), sdkLB, nil } func (m *defaultLoadBalancerManager) Update(ctx context.Context, resLB *elbv2model.LoadBalancer, sdkLB LoadBalancerWithTags) (elbv2model.LoadBalancerStatus, error) { diff --git a/pkg/deploy/elbv2/load_balancer_synthesizer.go b/pkg/deploy/elbv2/load_balancer_synthesizer.go index 5758820476..629071cce8 100644 --- a/pkg/deploy/elbv2/load_balancer_synthesizer.go +++ b/pkg/deploy/elbv2/load_balancer_synthesizer.go @@ -2,18 +2,19 @@ package elbv2 import ( "context" - elbv2types "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2/types" - "strings" - awssdk "github.com/aws/aws-sdk-go-v2/aws" elbv2sdk "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2" + elbv2types "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2/types" "github.com/go-logr/logr" "github.com/pkg/errors" "k8s.io/apimachinery/pkg/util/sets" "sigs.k8s.io/aws-load-balancer-controller/pkg/aws/services" + "sigs.k8s.io/aws-load-balancer-controller/pkg/config" "sigs.k8s.io/aws-load-balancer-controller/pkg/deploy/tracking" "sigs.k8s.io/aws-load-balancer-controller/pkg/model/core" elbv2model "sigs.k8s.io/aws-load-balancer-controller/pkg/model/elbv2" + "sigs.k8s.io/aws-load-balancer-controller/pkg/runtime" + "strings" ) const ( @@ -22,26 +23,33 @@ const ( // NewLoadBalancerSynthesizer constructs loadBalancerSynthesizer func NewLoadBalancerSynthesizer(elbv2Client services.ELBV2, trackingProvider tracking.Provider, taggingManager TaggingManager, - lbManager LoadBalancerManager, logger logr.Logger, stack core.Stack) *loadBalancerSynthesizer { + lbManager LoadBalancerManager, logger logr.Logger, featureGates config.FeatureGates, controllerConfig config.ControllerConfig, stack core.Stack) *loadBalancerSynthesizer { return &loadBalancerSynthesizer{ - elbv2Client: elbv2Client, - trackingProvider: trackingProvider, - taggingManager: taggingManager, - lbManager: lbManager, - logger: logger, - stack: stack, + elbv2Client: elbv2Client, + trackingProvider: trackingProvider, + taggingManager: taggingManager, + lbManager: lbManager, + logger: logger, + stack: stack, + featureGates: featureGates, + controllerConfig: controllerConfig, + lbsNeedingCapacityModification: nil, + capacityReservationReconciler: NewDefaultLoadBalancerCapacityReservationReconciler(elbv2Client, featureGates, logger), } } // loadBalancerSynthesizer is responsible for synthesize LoadBalancer resources types for certain stack. type loadBalancerSynthesizer struct { - elbv2Client services.ELBV2 - trackingProvider tracking.Provider - taggingManager TaggingManager - lbManager LoadBalancerManager - logger logr.Logger - - stack core.Stack + elbv2Client services.ELBV2 + trackingProvider tracking.Provider + taggingManager TaggingManager + lbManager LoadBalancerManager + logger logr.Logger + stack core.Stack + featureGates config.FeatureGates + controllerConfig config.ControllerConfig + lbsNeedingCapacityModification []resAndSDKLoadBalancerPair + capacityReservationReconciler LoadBalancerCapacityReservationReconciler } func (s *loadBalancerSynthesizer) Synthesize(ctx context.Context) error { @@ -75,10 +83,17 @@ func (s *loadBalancerSynthesizer) Synthesize(ctx context.Context) error { } } for _, resLB := range unmatchedResLBs { - lbStatus, err := s.lbManager.Create(ctx, resLB) + lbStatus, sdkLB, err := s.lbManager.Create(ctx, resLB) if err != nil { return err } + if s.featureGates.Enabled(config.LBCapacityReservation) && + resLB.Spec.MinimumLoadBalancerCapacity != nil { + s.lbsNeedingCapacityModification = append(s.lbsNeedingCapacityModification, resAndSDKLoadBalancerPair{ + resLB: resLB, + sdkLB: sdkLB, + }) + } resLB.SetStatus(lbStatus) } for _, resAndSDKLB := range matchedResAndSDKLBs { @@ -86,6 +101,12 @@ func (s *loadBalancerSynthesizer) Synthesize(ctx context.Context) error { if err != nil { return err } + if s.featureGates.Enabled(config.LBCapacityReservation) { + s.lbsNeedingCapacityModification = append(s.lbsNeedingCapacityModification, resAndSDKLoadBalancerPair{ + resLB: resAndSDKLB.resLB, + sdkLB: resAndSDKLB.sdkLB, + }) + } resAndSDKLB.resLB.SetStatus(lbStatus) } return nil @@ -106,7 +127,19 @@ func (s *loadBalancerSynthesizer) disableDeletionProtection(ctx context.Context, } func (s *loadBalancerSynthesizer) PostSynthesize(ctx context.Context) error { - // nothing to do here. + for _, resAndSDKLB := range s.lbsNeedingCapacityModification { + isLoadBalancerProvisioning, err := s.isLoadBalancerInProvisioningState(ctx, resAndSDKLB.sdkLB) + if err != nil { + return err + } + if isLoadBalancerProvisioning { + requeueMsg := "monitor provisioning state for load balancer: " + awssdk.ToString(resAndSDKLB.sdkLB.LoadBalancer.LoadBalancerName) + return runtime.NewRequeueNeededAfter(requeueMsg, s.controllerConfig.LBStabilizationMonitorInterval) + } + if err := s.capacityReservationReconciler.Reconcile(ctx, resAndSDKLB.resLB, resAndSDKLB.sdkLB); err != nil { + return err + } + } return nil } @@ -197,3 +230,21 @@ func isSDKLoadBalancerRequiresReplacement(sdkLB LoadBalancerWithTags, resLB *elb } return false } + +func (s *loadBalancerSynthesizer) isLoadBalancerInProvisioningState(ctx context.Context, sdkLB LoadBalancerWithTags) (bool, error) { + lbArn := awssdk.ToString(sdkLB.LoadBalancer.LoadBalancerArn) + req := &elbv2sdk.DescribeLoadBalancersInput{ + LoadBalancerArns: []string{lbArn}, + } + elbv2Resp, err := s.elbv2Client.DescribeLoadBalancersAsList(ctx, req) + if err != nil { + return false, err + } + if len(elbv2Resp) == 0 { + return false, errors.Errorf("no load balancer found for the arn: %v to monitor load balancer state", lbArn) + } + if elbv2Resp[0].State.Code == elbv2types.LoadBalancerStateEnumProvisioning { + return true, nil + } + return false, nil +} diff --git a/pkg/deploy/elbv2/load_balancer_synthesizer_test.go b/pkg/deploy/elbv2/load_balancer_synthesizer_test.go index 50940fd7b9..236eddbb03 100644 --- a/pkg/deploy/elbv2/load_balancer_synthesizer_test.go +++ b/pkg/deploy/elbv2/load_balancer_synthesizer_test.go @@ -1,14 +1,17 @@ package elbv2 import ( - elbv2types "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2/types" - "testing" - + "context" awssdk "github.com/aws/aws-sdk-go-v2/aws" + elbv2sdk "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2" + elbv2types "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2/types" + "github.com/golang/mock/gomock" "github.com/pkg/errors" "github.com/stretchr/testify/assert" + "sigs.k8s.io/aws-load-balancer-controller/pkg/aws/services" coremodel "sigs.k8s.io/aws-load-balancer-controller/pkg/model/core" elbv2model "sigs.k8s.io/aws-load-balancer-controller/pkg/model/elbv2" + "testing" ) func Test_matchResAndSDKLoadBalancers(t *testing.T) { @@ -595,3 +598,122 @@ func Test_isSDKLoadBalancerRequiresReplacement(t *testing.T) { }) } } + +func Test_isLoadBalancerInProvisioningState(t *testing.T) { + type describeLoadBalancersAsListCall struct { + req *elbv2sdk.DescribeLoadBalancersInput + resp []elbv2types.LoadBalancer + err error + } + type fields struct { + describeLoadBalancersAsListCalls []describeLoadBalancersAsListCall + } + type args struct { + sdkLB LoadBalancerWithTags + } + tests := []struct { + name string + fields fields + args args + want bool + wantErr error + }{ + { + name: "load balancer in provisioning state case", + fields: fields{ + describeLoadBalancersAsListCalls: []describeLoadBalancersAsListCall{ + { + req: &elbv2sdk.DescribeLoadBalancersInput{ + LoadBalancerArns: []string{"my-arn"}, + }, + resp: []elbv2types.LoadBalancer{ + { + LoadBalancerArn: awssdk.String("lb-1"), + State: &elbv2types.LoadBalancerState{Code: elbv2types.LoadBalancerStateEnumProvisioning}, + }, + }, + }, + }, + }, + args: args{ + sdkLB: LoadBalancerWithTags{ + LoadBalancer: &elbv2types.LoadBalancer{ + LoadBalancerArn: awssdk.String("my-arn"), + }, + Tags: nil, + }, + }, + want: true, + }, + { + name: "load balancer in active state case", + fields: fields{ + describeLoadBalancersAsListCalls: []describeLoadBalancersAsListCall{ + { + req: &elbv2sdk.DescribeLoadBalancersInput{ + LoadBalancerArns: []string{"my-arn"}, + }, + resp: []elbv2types.LoadBalancer{ + { + LoadBalancerArn: awssdk.String("lb-1"), + State: &elbv2types.LoadBalancerState{Code: elbv2types.LoadBalancerStateEnumActive}, + }, + }, + }, + }, + }, + args: args{ + sdkLB: LoadBalancerWithTags{ + LoadBalancer: &elbv2types.LoadBalancer{ + LoadBalancerArn: awssdk.String("my-arn"), + }, + Tags: nil, + }, + }, + want: false, + }, + { + name: "error case", + fields: fields{ + describeLoadBalancersAsListCalls: []describeLoadBalancersAsListCall{ + { + req: &elbv2sdk.DescribeLoadBalancersInput{ + LoadBalancerArns: []string{"my-arn"}, + }, + err: errors.New("some error"), + }, + }, + }, + args: args{ + sdkLB: LoadBalancerWithTags{ + LoadBalancer: &elbv2types.LoadBalancer{ + LoadBalancerArn: awssdk.String("my-arn"), + }, + Tags: nil, + }, + }, + wantErr: errors.New("some error"), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + elbv2Client := services.NewMockELBV2(ctrl) + for _, call := range tt.fields.describeLoadBalancersAsListCalls { + elbv2Client.EXPECT().DescribeLoadBalancersAsList(gomock.Any(), call.req).Return(call.resp, call.err) + } + + r := &loadBalancerSynthesizer{ + elbv2Client: elbv2Client, + } + got, err := r.isLoadBalancerInProvisioningState(context.Background(), tt.args.sdkLB) + if tt.wantErr != nil { + assert.EqualError(t, err, tt.wantErr.Error()) + } else { + assert.NoError(t, err) + assert.Equal(t, tt.want, got) + } + }) + } +} diff --git a/pkg/deploy/stack_deployer.go b/pkg/deploy/stack_deployer.go index dda035adc3..bcfc6254fe 100644 --- a/pkg/deploy/stack_deployer.go +++ b/pkg/deploy/stack_deployer.go @@ -34,12 +34,13 @@ func NewDefaultStackDeployer(cloud aws.Cloud, k8sClient client.Client, return &defaultStackDeployer{ cloud: cloud, k8sClient: k8sClient, + controllerConfig: config, addonsConfig: config.AddonsConfig, trackingProvider: trackingProvider, ec2TaggingManager: ec2TaggingManager, ec2SGManager: ec2.NewDefaultSecurityGroupManager(cloud.EC2(), trackingProvider, ec2TaggingManager, networkingSGReconciler, cloud.VpcID(), config.ExternalManagedTags, logger), elbv2TaggingManager: elbv2TaggingManager, - elbv2LBManager: elbv2.NewDefaultLoadBalancerManager(cloud.ELBV2(), trackingProvider, elbv2TaggingManager, config.ExternalManagedTags, logger), + elbv2LBManager: elbv2.NewDefaultLoadBalancerManager(cloud.ELBV2(), trackingProvider, elbv2TaggingManager, config.ExternalManagedTags, config.FeatureGates, logger), elbv2LSManager: elbv2.NewDefaultListenerManager(cloud.ELBV2(), trackingProvider, elbv2TaggingManager, config.ExternalManagedTags, config.FeatureGates, logger), elbv2LRManager: elbv2.NewDefaultListenerRuleManager(cloud.ELBV2(), trackingProvider, elbv2TaggingManager, config.ExternalManagedTags, config.FeatureGates, logger), elbv2TGManager: elbv2.NewDefaultTargetGroupManager(cloud.ELBV2(), trackingProvider, elbv2TaggingManager, cloud.VpcID(), config.ExternalManagedTags, logger), @@ -59,6 +60,7 @@ var _ StackDeployer = &defaultStackDeployer{} type defaultStackDeployer struct { cloud aws.Cloud k8sClient client.Client + controllerConfig config.ControllerConfig addonsConfig config.AddonsConfig trackingProvider tracking.Provider ec2TaggingManager ec2.TaggingManager @@ -88,7 +90,7 @@ func (d *defaultStackDeployer) Deploy(ctx context.Context, stack core.Stack) err synthesizers := []ResourceSynthesizer{ ec2.NewSecurityGroupSynthesizer(d.cloud.EC2(), d.trackingProvider, d.ec2TaggingManager, d.ec2SGManager, d.vpcID, d.logger, stack), elbv2.NewTargetGroupSynthesizer(d.cloud.ELBV2(), d.trackingProvider, d.elbv2TaggingManager, d.elbv2TGManager, d.logger, d.featureGates, stack), - elbv2.NewLoadBalancerSynthesizer(d.cloud.ELBV2(), d.trackingProvider, d.elbv2TaggingManager, d.elbv2LBManager, d.logger, stack), + elbv2.NewLoadBalancerSynthesizer(d.cloud.ELBV2(), d.trackingProvider, d.elbv2TaggingManager, d.elbv2LBManager, d.logger, d.featureGates, d.controllerConfig, stack), elbv2.NewListenerSynthesizer(d.cloud.ELBV2(), d.elbv2TaggingManager, d.elbv2LSManager, d.logger, stack), elbv2.NewListenerRuleSynthesizer(d.cloud.ELBV2(), d.elbv2TaggingManager, d.elbv2LRManager, d.logger, stack), elbv2.NewTargetGroupBindingSynthesizer(d.k8sClient, d.trackingProvider, d.elbv2TGBManager, d.logger, stack), diff --git a/pkg/ingress/model_build_load_balancer.go b/pkg/ingress/model_build_load_balancer.go index fac48df9a9..111905cbf4 100644 --- a/pkg/ingress/model_build_load_balancer.go +++ b/pkg/ingress/model_build_load_balancer.go @@ -5,13 +5,12 @@ import ( "crypto/sha256" "encoding/hex" "fmt" - ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types" - "regexp" - awssdk "github.com/aws/aws-sdk-go-v2/aws" + ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types" "github.com/google/go-cmp/cmp" "github.com/pkg/errors" "k8s.io/apimachinery/pkg/util/sets" + "regexp" "sigs.k8s.io/aws-load-balancer-controller/apis/elbv2/v1beta1" "sigs.k8s.io/aws-load-balancer-controller/pkg/algorithm" "sigs.k8s.io/aws-load-balancer-controller/pkg/annotations" @@ -72,16 +71,21 @@ func (t *defaultModelBuildTask) buildLoadBalancerSpec(ctx context.Context, liste if err != nil { return elbv2model.LoadBalancerSpec{}, err } + lbMinimumCapacity, err := t.buildLoadBalancerMinimumCapacity(ctx) + if err != nil { + return elbv2model.LoadBalancerSpec{}, err + } return elbv2model.LoadBalancerSpec{ - Name: name, - Type: elbv2model.LoadBalancerTypeApplication, - Scheme: scheme, - IPAddressType: ipAddressType, - SubnetMappings: subnetMappings, - SecurityGroups: securityGroups, - CustomerOwnedIPv4Pool: coIPv4Pool, - LoadBalancerAttributes: loadBalancerAttributes, - Tags: tags, + Name: name, + Type: elbv2model.LoadBalancerTypeApplication, + Scheme: scheme, + IPAddressType: ipAddressType, + SubnetMappings: subnetMappings, + SecurityGroups: securityGroups, + CustomerOwnedIPv4Pool: coIPv4Pool, + LoadBalancerAttributes: loadBalancerAttributes, + MinimumLoadBalancerCapacity: lbMinimumCapacity, + Tags: tags, }, nil } diff --git a/pkg/ingress/model_build_load_balancer_capacity_reservation.go b/pkg/ingress/model_build_load_balancer_capacity_reservation.go new file mode 100644 index 0000000000..429e5ee20f --- /dev/null +++ b/pkg/ingress/model_build_load_balancer_capacity_reservation.go @@ -0,0 +1,81 @@ +package ingress + +import ( + "context" + "github.com/pkg/errors" + "sigs.k8s.io/aws-load-balancer-controller/pkg/annotations" + "sigs.k8s.io/aws-load-balancer-controller/pkg/config" + elbv2model "sigs.k8s.io/aws-load-balancer-controller/pkg/model/elbv2" + "strconv" +) + +// buildLoadBalancerMinimumCapacity builds the minimum load balancer capacity for load balancer +func (t *defaultModelBuildTask) buildLoadBalancerMinimumCapacity(_ context.Context) (*elbv2model.MinimumLoadBalancerCapacity, error) { + if !t.featureGates.Enabled(config.LBCapacityReservation) { + return nil, nil + } + ingGroupCapacityUnits, err := t.buildIngressGroupLoadBalancerMinimumCapacity(t.ingGroup.Members) + if err != nil { + return nil, err + } + var minimumLoadBalancerCapacity *elbv2model.MinimumLoadBalancerCapacity + var capacityUnits int64 + for key, value := range ingGroupCapacityUnits { + if key != elbv2model.CapacityUnits { + return nil, errors.Errorf("invalid key to set the capacity: %v, Expected key: %v", key, elbv2model.CapacityUnits) + } + capacityUnits, _ = strconv.ParseInt(value, 10, 64) + minimumLoadBalancerCapacity = &elbv2model.MinimumLoadBalancerCapacity{ + CapacityUnits: int32(capacityUnits), + } + + } + return minimumLoadBalancerCapacity, nil +} + +// buildIngressGroupLoadBalancerMinimumCapacity builds the minimum load balancer capacity for ingresses within a group. +// Note: the capacity reservation specified via IngressClass takes higher priority than the capacity specified via annotation on Ingress. +func (t *defaultModelBuildTask) buildIngressGroupLoadBalancerMinimumCapacity(ingList []ClassifiedIngress) (map[string]string, error) { + if len(ingList) > 0 { + ingClassCapacityUnits, _ := t.buildIngressClassLoadBalancerMinimumCapacity(ingList[0].IngClassConfig) + if ingClassCapacityUnits != nil { + return ingClassCapacityUnits, nil + } + } + ingGroupCapacityUnits := make(map[string]string) + for _, ing := range ingList { + ingGroupCapacity, err := t.buildIngressLoadBalancerMinimumCapacity(ing) + if err != nil { + return nil, err + } + // check for conflict capacity values + for capacityKey, capacityValue := range ingGroupCapacity { + existingCapacityValue, exists := ingGroupCapacityUnits[capacityKey] + if exists && existingCapacityValue != capacityValue { + return nil, errors.Errorf("conflicting capacity reservation %v: %v | %v", capacityKey, existingCapacityValue, capacityValue) + } + ingGroupCapacityUnits[capacityKey] = capacityValue + } + } + return ingGroupCapacityUnits, nil +} + +// buildIngressLoadBalancerMinimumCapacity builds the minimum load balancer capacity used for a single ingress within a group +func (t *defaultModelBuildTask) buildIngressLoadBalancerMinimumCapacity(ing ClassifiedIngress) (map[string]string, error) { + var annotationCapacity map[string]string + if _, err := t.annotationParser.ParseStringMapAnnotation(annotations.IngressSuffixLoadBalancerCapacityReservation, &annotationCapacity, ing.Ing.Annotations); err != nil { + return nil, err + } + return annotationCapacity, nil +} + +// buildIngressClassLoadBalancerMinimumCapacity builds the minimum load balancer capacity for an IngressClass. +func (t *defaultModelBuildTask) buildIngressClassLoadBalancerMinimumCapacity(ingClassConfig ClassConfiguration) (map[string]string, error) { + if ingClassConfig.IngClassParams == nil || ingClassConfig.IngClassParams.Spec.MinimumLoadBalancerCapacity == nil { + return nil, nil + } + capacityUnits := strconv.Itoa(int(ingClassConfig.IngClassParams.Spec.MinimumLoadBalancerCapacity.CapacityUnits)) + ingClassCapacityUnits := make(map[string]string) + ingClassCapacityUnits[elbv2model.CapacityUnits] = capacityUnits + return ingClassCapacityUnits, nil +} diff --git a/pkg/ingress/model_build_load_balancer_capacity_reservation_test.go b/pkg/ingress/model_build_load_balancer_capacity_reservation_test.go new file mode 100644 index 0000000000..5549e6ebe6 --- /dev/null +++ b/pkg/ingress/model_build_load_balancer_capacity_reservation_test.go @@ -0,0 +1,520 @@ +package ingress + +import ( + "context" + "fmt" + "github.com/pkg/errors" + "github.com/stretchr/testify/assert" + networking "k8s.io/api/networking/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + elbv2api "sigs.k8s.io/aws-load-balancer-controller/apis/elbv2/v1beta1" + "sigs.k8s.io/aws-load-balancer-controller/pkg/annotations" + "sigs.k8s.io/aws-load-balancer-controller/pkg/config" + elbv2model "sigs.k8s.io/aws-load-balancer-controller/pkg/model/elbv2" + "testing" +) + +func Test_defaultModelBuildTask_buildLoadBalancerMinimumCapacity(t *testing.T) { + type fields struct { + ingGroup Group + } + tests := []struct { + name string + featureGates map[config.Feature]bool + fields fields + want *elbv2model.MinimumLoadBalancerCapacity + wantErr error + }{ + { + name: "capacity reservation feature disabled", + featureGates: map[config.Feature]bool{ + config.LBCapacityReservation: false, + }, + fields: fields{ + ingGroup: Group{ + ID: GroupID{Name: "ig-group-1"}, + Members: []ClassifiedIngress{ + { + Ing: &networking.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "awesome-ns", + Name: "awesome-ing", + Annotations: map[string]string{ + "alb.ingress.kubernetes.io/minimum-load-balancer-capacity": "CapacityUnits=500", + }, + }, + }, + }, + { + Ing: &networking.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "awesome-ns", + Name: "awesome-ing", + Annotations: map[string]string{ + "alb.ingress.kubernetes.io/minimum-load-balancer-capacity": "CapacityUnits=500", + }, + }, + }, + }, + }, + }, + }, + want: nil, + }, + { + name: "capacity reservation from multiple Ingress that do not conflict", + featureGates: map[config.Feature]bool{ + config.LBCapacityReservation: true, + }, + fields: fields{ + ingGroup: Group{ + ID: GroupID{Name: "ig-group-1"}, + Members: []ClassifiedIngress{ + { + Ing: &networking.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "awesome-ns", + Name: "awesome-ing", + Annotations: map[string]string{ + "alb.ingress.kubernetes.io/minimum-load-balancer-capacity": "CapacityUnits=500", + }, + }, + }, + }, + { + Ing: &networking.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "awesome-ns", + Name: "awesome-ing", + Annotations: map[string]string{ + "alb.ingress.kubernetes.io/minimum-load-balancer-capacity": "CapacityUnits=500", + }, + }, + }, + }, + }, + }, + }, + want: &elbv2model.MinimumLoadBalancerCapacity{CapacityUnits: 500}, + }, + { + name: "capacity reservation from multiple Ingress that conflict", + featureGates: map[config.Feature]bool{ + config.LBCapacityReservation: true, + }, + fields: fields{ + ingGroup: Group{ + ID: GroupID{Name: "ig-group-2"}, + Members: []ClassifiedIngress{ + { + Ing: &networking.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "awesome-ns", + Name: "awesome-ing", + Annotations: map[string]string{ + "alb.ingress.kubernetes.io/minimum-load-balancer-capacity": "CapacityUnits=500", + }, + }, + }, + }, + { + Ing: &networking.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "awesome-ns", + Name: "awesome-ing", + Annotations: map[string]string{ + "alb.ingress.kubernetes.io/minimum-load-balancer-capacity": "CapacityUnits=200", + }, + }, + }, + }, + }, + }, + }, + wantErr: errors.New("conflicting capacity reservation CapacityUnits: 500 | 200"), + }, + { + name: "non-empty annotation capacity reservation from multiple Ingress, non-empty IngressClass capacity reservation", + featureGates: map[config.Feature]bool{ + config.LBCapacityReservation: true, + }, + fields: fields{ + ingGroup: Group{ + ID: GroupID{Name: "ig-group-3"}, + Members: []ClassifiedIngress{ + { + Ing: &networking.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "awesome-ns", + Name: "awesome-ing", + Annotations: map[string]string{ + "alb.ingress.kubernetes.io/minimum-load-balancer-capacity": "CapacityUnits=500", + }, + }, + }, + IngClassConfig: ClassConfiguration{ + IngClassParams: &elbv2api.IngressClassParams{ + ObjectMeta: metav1.ObjectMeta{ + Name: "awesome-class", + }, + Spec: elbv2api.IngressClassParamsSpec{ + MinimumLoadBalancerCapacity: &elbv2api.MinimumLoadBalancerCapacity{ + CapacityUnits: 1200, + }, + }, + }, + }, + }, + { + Ing: &networking.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "awesome-ns", + Name: "awesome-ing", + Annotations: map[string]string{ + "alb.ingress.kubernetes.io/minimum-load-balancer-capacity": "CapacityUnits=600", + }, + }, + }, + }, + }, + }, + }, + want: &elbv2model.MinimumLoadBalancerCapacity{CapacityUnits: 1200}, + }, + { + name: "capacity reservation from Ingress that does not have annotation for setting capacity reservation", + featureGates: map[config.Feature]bool{ + config.LBCapacityReservation: true, + }, + fields: fields{ + ingGroup: Group{ + ID: GroupID{Name: "ig-group-4"}, + Members: []ClassifiedIngress{ + { + Ing: &networking.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "awesome-ns", + Name: "awesome-ing", + Annotations: map[string]string{}, + }, + }, + }, + }, + }, + }, + want: nil, + }, + { + name: "invalid key to set the capacity reservation", + featureGates: map[config.Feature]bool{ + config.LBCapacityReservation: true, + }, + fields: fields{ + ingGroup: Group{ + ID: GroupID{Name: "ig-group-1"}, + Members: []ClassifiedIngress{ + { + Ing: &networking.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "awesome-ns", + Name: "awesome-ing", + Annotations: map[string]string{ + "alb.ingress.kubernetes.io/minimum-load-balancer-capacity": "InvalidUnits=500", + }, + }, + }, + }, + }, + }, + }, + wantErr: errors.New("invalid key to set the capacity: InvalidUnits, Expected key: CapacityUnits"), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + annotationParser := annotations.NewSuffixAnnotationParser("alb.ingress.kubernetes.io") + featureGates := config.NewFeatureGates() + for key, value := range tt.featureGates { + if value { + featureGates.Enable(key) + } else { + featureGates.Disable(key) + } + } + task := &defaultModelBuildTask{ + annotationParser: annotationParser, + ingGroup: tt.fields.ingGroup, + featureGates: featureGates, + } + got, err := task.buildLoadBalancerMinimumCapacity(context.Background()) + if tt.wantErr != nil { + assert.EqualError(t, err, tt.wantErr.Error()) + } else { + assert.NoError(t, err) + assert.Equal(t, tt.want, got) + } + }) + } +} + +func Test_defaultModelBuildTask_buildIngressGroupLoadBalancerMinimumCapacity(t *testing.T) { + type args struct { + ingList []ClassifiedIngress + } + tests := []struct { + name string + args args + want map[string]string + wantErr error + }{ + { + name: "capacity reservation from multiple Ingress that do not conflict", + args: args{ + ingList: []ClassifiedIngress{ + { + Ing: &networking.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "awesome-ns", + Name: "awesome-ing", + Annotations: map[string]string{ + "alb.ingress.kubernetes.io/minimum-load-balancer-capacity": "CapacityUnits=500", + }, + }, + }, + }, + { + Ing: &networking.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "awesome-ns", + Name: "awesome-ing", + Annotations: map[string]string{ + "alb.ingress.kubernetes.io/minimum-load-balancer-capacity": "CapacityUnits=500", + }, + }, + }, + }, + }, + }, + want: map[string]string{ + "CapacityUnits": "500", + }, + }, + { + name: "capacity reservation from multiple Ingress that conflict", + args: args{ + ingList: []ClassifiedIngress{ + { + Ing: &networking.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "awesome-ns", + Name: "awesome-ing", + Annotations: map[string]string{ + "alb.ingress.kubernetes.io/minimum-load-balancer-capacity": "CapacityUnits=500", + }, + }, + }, + }, + { + Ing: &networking.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "awesome-ns", + Name: "awesome-ing", + Annotations: map[string]string{ + "alb.ingress.kubernetes.io/minimum-load-balancer-capacity": "CapacityUnits=200", + }, + }, + }, + }, + }, + }, + wantErr: errors.New("conflicting capacity reservation CapacityUnits: 500 | 200"), + }, + { + name: "non-empty annotation capacity reservation from multiple Ingress, non-empty IngressClass capacity reservation", + args: args{ + ingList: []ClassifiedIngress{ + { + Ing: &networking.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "awesome-ns", + Name: "awesome-ing", + Annotations: map[string]string{ + "alb.ingress.kubernetes.io/minimum-load-balancer-capacity": "CapacityUnits=500", + }, + }, + }, + IngClassConfig: ClassConfiguration{ + IngClassParams: &elbv2api.IngressClassParams{ + ObjectMeta: metav1.ObjectMeta{ + Name: "awesome-class", + }, + Spec: elbv2api.IngressClassParamsSpec{ + MinimumLoadBalancerCapacity: &elbv2api.MinimumLoadBalancerCapacity{ + CapacityUnits: 1200, + }, + }, + }, + }, + }, + { + Ing: &networking.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "awesome-ns", + Name: "awesome-ing", + Annotations: map[string]string{ + "alb.ingress.kubernetes.io/minimum-load-balancer-capacity": "CapacityUnits=600", + }, + }, + }, + }, + }, + }, + want: map[string]string{ + "CapacityUnits": "1200", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + annotationParser := annotations.NewSuffixAnnotationParser("alb.ingress.kubernetes.io") + task := &defaultModelBuildTask{ + annotationParser: annotationParser, + } + got, err := task.buildIngressGroupLoadBalancerMinimumCapacity(tt.args.ingList) + if tt.wantErr != nil { + assert.EqualError(t, err, tt.wantErr.Error()) + } else { + assert.NoError(t, err) + assert.Equal(t, tt.want, got) + } + }) + } +} + +func Test_defaultModelBuildTask_buildIngressLoadBalancerMinimumCapacity(t *testing.T) { + type args struct { + ing ClassifiedIngress + } + tests := []struct { + name string + args args + want map[string]string + wantErr error + }{ + { + name: "non-empty annotation capacity reservation from Ingress", + args: args{ + ing: ClassifiedIngress{ + Ing: &networking.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "awesome-ns", + Name: "awesome-ing", + Annotations: map[string]string{ + "alb.ingress.kubernetes.io/minimum-load-balancer-capacity": "CapacityUnits=500", + }, + }, + }, + }, + }, + want: map[string]string{ + "CapacityUnits": "500", + }, + }, + { + name: "empty capacity reservation from Ingress", + args: args{ + ing: ClassifiedIngress{ + Ing: &networking.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "awesome-ns", + Name: "awesome-ing", + }, + }, + IngClassConfig: ClassConfiguration{}, + }, + }, + want: map[string]string(nil), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + annotationParser := annotations.NewSuffixAnnotationParser("alb.ingress.kubernetes.io") + task := &defaultModelBuildTask{ + annotationParser: annotationParser, + } + got, err := task.buildIngressLoadBalancerMinimumCapacity(tt.args.ing) + if tt.wantErr != nil { + fmt.Println(err) + assert.EqualError(t, err, tt.wantErr.Error()) + } else { + assert.NoError(t, err) + assert.Equal(t, tt.want, got) + } + }) + } +} + +func Test_defaultModelBuildTask_buildIngressClassLoadBalancerMinimumCapacity(t *testing.T) { + type args struct { + ingClassConfig ClassConfiguration + } + tests := []struct { + name string + args args + want map[string]string + }{ + { + name: "non-empty ingressClassParams, non-empty minimumLoadBalancerCapacity", + args: args{ + ingClassConfig: ClassConfiguration{ + IngClassParams: &elbv2api.IngressClassParams{ + ObjectMeta: metav1.ObjectMeta{ + Name: "awesome-class", + }, + Spec: elbv2api.IngressClassParamsSpec{ + MinimumLoadBalancerCapacity: &elbv2api.MinimumLoadBalancerCapacity{ + CapacityUnits: 1200, + }, + }, + }, + }, + }, + want: map[string]string{ + "CapacityUnits": "1200", + }, + }, + { + name: "non-empty ingressClassParams, empty minimumLoadBalancerCapacity", + args: args{ + ingClassConfig: ClassConfiguration{ + IngClassParams: &elbv2api.IngressClassParams{ + ObjectMeta: metav1.ObjectMeta{ + Name: "awesome-class", + }, + Spec: elbv2api.IngressClassParamsSpec{ + MinimumLoadBalancerCapacity: nil, + }, + }, + }, + }, + want: nil, + }, + { + name: "empty ingressClassParams", + args: args{ + ingClassConfig: ClassConfiguration{ + IngClassParams: nil, + }, + }, + want: nil, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + task := &defaultModelBuildTask{} + got, err := task.buildIngressClassLoadBalancerMinimumCapacity(tt.args.ingClassConfig) + assert.NoError(t, err) + assert.Equal(t, tt.want, got) + }) + } +} diff --git a/pkg/model/elbv2/load_balancer.go b/pkg/model/elbv2/load_balancer.go index 4e394f5a02..b521430968 100644 --- a/pkg/model/elbv2/load_balancer.go +++ b/pkg/model/elbv2/load_balancer.go @@ -142,6 +142,17 @@ type LoadBalancerAttribute struct { Value string `json:"value"` } +// Unit for setting the capacity on load balancer +const ( + CapacityUnits string = "CapacityUnits" +) + +// Information about a load balancer capacity reservation. +type MinimumLoadBalancerCapacity struct { + // The Capacity Units Value. + CapacityUnits int32 `json:"capacityUnits"` +} + // LoadBalancerSpec defines the desired state of LoadBalancer type LoadBalancerSpec struct { // The name of the load balancer. @@ -183,6 +194,10 @@ type LoadBalancerSpec struct { // +optional LoadBalancerAttributes []LoadBalancerAttribute `json:"loadBalancerAttributes,omitempty"` + // The load balancer capacity reservation + // +optional + MinimumLoadBalancerCapacity *MinimumLoadBalancerCapacity `json:"minimumLoadBalancerCapacity,omitempty"` + // The tags. // +optional Tags map[string]string `json:"tags,omitempty"` diff --git a/pkg/service/model_build_load_balancer.go b/pkg/service/model_build_load_balancer.go index 8142de5351..b1087c6c57 100644 --- a/pkg/service/model_build_load_balancer.go +++ b/pkg/service/model_build_load_balancer.go @@ -65,6 +65,10 @@ func (t *defaultModelBuildTask) buildLoadBalancerSpec(ctx context.Context, schem if err != nil { return elbv2model.LoadBalancerSpec{}, err } + lbMinimumCapacity, err := t.buildLoadBalancerMinimumCapacity(ctx) + if err != nil { + return elbv2model.LoadBalancerSpec{}, err + } securityGroups, err := t.buildLoadBalancerSecurityGroups(ctx, existingLB, ipAddressType) if err != nil { return elbv2model.LoadBalancerSpec{}, err @@ -95,6 +99,7 @@ func (t *defaultModelBuildTask) buildLoadBalancerSpec(ctx context.Context, schem SecurityGroups: securityGroups, SubnetMappings: subnetMappings, LoadBalancerAttributes: lbAttributes, + MinimumLoadBalancerCapacity: lbMinimumCapacity, Tags: tags, } @@ -484,6 +489,33 @@ func (t *defaultModelBuildTask) buildLoadBalancerAttributes(_ context.Context) ( return makeAttributesSliceFromMap(mergedAttributes), nil } +func (t *defaultModelBuildTask) buildLoadBalancerMinimumCapacity(_ context.Context) (*elbv2model.MinimumLoadBalancerCapacity, error) { + if !t.featureGates.Enabled(config.LBCapacityReservation) { + return nil, nil + } + // Parse the annotation + var loadBalancerMinimumCapacityMap map[string]string + if _, err := t.annotationParser.ParseStringMapAnnotation(annotations.SvcLBSuffixLoadBalancerCapacityReservation, &loadBalancerMinimumCapacityMap, t.service.Annotations); err != nil { + return nil, err + } + if loadBalancerMinimumCapacityMap == nil { + return nil, nil + } + // Transform annotation to minimumLoadBalancerCapacity object + var minimumLoadBalancerCapacity *elbv2model.MinimumLoadBalancerCapacity + var capacityUnits int64 + for key, value := range loadBalancerMinimumCapacityMap { + if key != elbv2model.CapacityUnits { + return nil, errors.Errorf("invalid key to set the capacity: %v, Expected key: %v", key, elbv2model.CapacityUnits) + } + capacityUnits, _ = strconv.ParseInt(value, 10, 64) + minimumLoadBalancerCapacity = &elbv2model.MinimumLoadBalancerCapacity{ + CapacityUnits: int32(capacityUnits), + } + } + return minimumLoadBalancerCapacity, nil +} + func makeAttributesSliceFromMap(loadBalancerAttributesMap map[string]string) []elbv2model.LoadBalancerAttribute { attributes := make([]elbv2model.LoadBalancerAttribute, 0, len(loadBalancerAttributesMap)) for attrKey, attrValue := range loadBalancerAttributesMap { diff --git a/pkg/service/model_build_load_balancer_test.go b/pkg/service/model_build_load_balancer_test.go index 8b3370e746..4e681c6db3 100644 --- a/pkg/service/model_build_load_balancer_test.go +++ b/pkg/service/model_build_load_balancer_test.go @@ -1802,3 +1802,80 @@ func Test_defaultModelBuildTask_buildLoadBalancerName(t *testing.T) { }) } } + +func Test_defaultModelBuilderTask_buildLbCapacity(t *testing.T) { + tests := []struct { + testName string + svc *corev1.Service + wantError bool + wantValue *elbv2.MinimumLoadBalancerCapacity + }{ + { + testName: "Default value", + svc: &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + "service.beta.kubernetes.io/aws-load-balancer-type": "nlb-ip", + }, + }, + }, + wantError: false, + wantValue: nil, + }, + { + testName: "Annotation specified", + svc: &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + "service.beta.kubernetes.io/aws-load-balancer-type": "nlb-ip", + "service.beta.kubernetes.io/aws-load-balancer-minimum-load-balancer-capacity": "CapacityUnits=3000", + }, + }, + }, + wantError: false, + wantValue: &elbv2.MinimumLoadBalancerCapacity{ + CapacityUnits: int32(3000), + }, + }, + { + testName: "Annotation invalid", + svc: &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + "service.beta.kubernetes.io/aws-load-balancer-type": "nlb-ip", + "service.beta.kubernetes.io/aws-load-balancer-minimum-load-balancer-capacity": "InvalidUnits=3000", + }, + }, + }, + wantError: true, + }, + } + for _, tt := range tests { + t.Run(tt.testName, func(t *testing.T) { + parser := annotations.NewSuffixAnnotationParser("service.beta.kubernetes.io") + featureGates := config.NewFeatureGates() + builder := &defaultModelBuildTask{ + service: tt.svc, + annotationParser: parser, + defaultAccessLogsS3Bucket: "", + defaultAccessLogsS3Prefix: "", + defaultLoadBalancingCrossZoneEnabled: false, + defaultProxyProtocolV2Enabled: false, + defaultHealthCheckProtocol: elbv2.ProtocolTCP, + defaultHealthCheckPort: healthCheckPortTrafficPort, + defaultHealthCheckPath: "/", + defaultHealthCheckInterval: 10, + defaultHealthCheckTimeout: 10, + defaultHealthCheckHealthyThreshold: 3, + defaultHealthCheckUnhealthyThreshold: 3, + featureGates: featureGates, + } + lbMinimumCapacity, err := builder.buildLoadBalancerMinimumCapacity(context.Background()) + if tt.wantError { + assert.Error(t, err) + } else { + assert.Equal(t, tt.wantValue, lbMinimumCapacity) + } + }) + } +} diff --git a/scripts/aws_sdk_model_override/cleanup.sh b/scripts/aws_sdk_model_override/cleanup.sh index d73ddd3a52..e13562a8ba 100755 --- a/scripts/aws_sdk_model_override/cleanup.sh +++ b/scripts/aws_sdk_model_override/cleanup.sh @@ -1,5 +1,5 @@ #!/bin/bash -SDK_VENDOR_PATH="./scripts/aws_sdk_model_override/aws-sdk-go" +SDK_VENDOR_PATH="./scripts/aws_sdk_model_override/awsSdkGoV2" rm -rf "${SDK_VENDOR_PATH}" -go mod edit -dropreplace github.com/aws/aws-sdk-go +go mod edit -dropreplace github.com/aws/aws-sdk-go-v2 diff --git a/scripts/aws_sdk_model_override/setup.sh b/scripts/aws_sdk_model_override/setup.sh index 6692146e6a..337c8542b7 100755 --- a/scripts/aws_sdk_model_override/setup.sh +++ b/scripts/aws_sdk_model_override/setup.sh @@ -2,21 +2,8 @@ set -e -SDK_VENDOR_PATH="./scripts/aws_sdk_model_override/aws-sdk-go" -SDK_MODEL_OVERRIDE_DST_PATH="${SDK_VENDOR_PATH}/models" -SDK_MODEL_OVERRIDE_SRC_PATH="./scripts/aws_sdk_model_override/models" - -# Clone the SDK to the vendor path (removing an old one if necessary) -rm -rf "${SDK_VENDOR_PATH}" -git clone --depth 1 https://github.com/aws/aws-sdk-go.git "${SDK_VENDOR_PATH}" - -# Override the SDK models -cp -r "${SDK_MODEL_OVERRIDE_SRC_PATH}"/* "${SDK_MODEL_OVERRIDE_DST_PATH}"/. - -# Generate the SDK -pushd "${SDK_VENDOR_PATH}" -make generate -popd +SDK_VENDOR_PATH="./scripts/aws_sdk_model_override" +SDK_MODEL_OVERRIDE_DST_PATH="${SDK_VENDOR_PATH}/awsSdkGoV2/service/elasticloadbalancingv2" # Use the vendored version of aws-sdk-go -go mod edit -replace github.com/aws/aws-sdk-go="${SDK_VENDOR_PATH}" +go mod edit -replace github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2="${SDK_MODEL_OVERRIDE_DST_PATH}"