From 0d5f45dd7af26eb9fd6375bd653254425fa0fc55 Mon Sep 17 00:00:00 2001 From: oliviassss Date: Mon, 5 Feb 2024 16:00:52 -0800 Subject: [PATCH 1/4] update helm chart for ServiceTargetENISGTags and README --- helm/aws-load-balancer-controller/Chart.yaml | 2 +- helm/aws-load-balancer-controller/README.md | 2 ++ .../templates/deployment.yaml | 3 +++ helm/aws-load-balancer-controller/values.yaml | 8 ++++++++ 4 files changed, 14 insertions(+), 1 deletion(-) diff --git a/helm/aws-load-balancer-controller/Chart.yaml b/helm/aws-load-balancer-controller/Chart.yaml index e16be6bf6..9a0e2c217 100644 --- a/helm/aws-load-balancer-controller/Chart.yaml +++ b/helm/aws-load-balancer-controller/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: aws-load-balancer-controller description: AWS Load Balancer Controller Helm chart for Kubernetes -version: 1.7.0 +version: 1.7.1 appVersion: v2.7.0 home: https://github.com/aws/eks-charts icon: https://raw.githubusercontent.com/aws/eks-charts/master/docs/logo/aws.png diff --git a/helm/aws-load-balancer-controller/README.md b/helm/aws-load-balancer-controller/README.md index dbb5aaf04..32fc7042f 100644 --- a/helm/aws-load-balancer-controller/README.md +++ b/helm/aws-load-balancer-controller/README.md @@ -264,3 +264,5 @@ The default values set by the application itself can be confirmed [here](https:/ | `controllerConfig.featureGates` | set of `key: value` pairs that describe AWS load balance controller features | `{}` | | `ingressClassConfig.default` | If `true`, the ingressclass will be the default class of the cluster. | `false` | | `enableServiceMutatorWebhook` | If `false`, disable the Service Mutator webhook which makes all new services of type LoadBalancer reconciled by the lb controller | `true` | +| `autoscaling` | If `autoscaling.enabled=true`, enable the HPA on the controller mainly to survive load induced failure by the calls to the `aws-load-balancer-webhook-service`. Please keep in mind that the controller pods have `priorityClassName: system-cluster-critical`, enabling HPA may lead to the eviction of other low-priority pods in the node | `false` | +| `serviceTargetENISGTags` | set of `key=value` pairs of AWS tags in addition to cluster name for finding the target ENI security group to which to add inbound rules from NLBs | None | diff --git a/helm/aws-load-balancer-controller/templates/deployment.yaml b/helm/aws-load-balancer-controller/templates/deployment.yaml index 3984bf450..c1bed7b86 100644 --- a/helm/aws-load-balancer-controller/templates/deployment.yaml +++ b/helm/aws-load-balancer-controller/templates/deployment.yaml @@ -156,6 +156,9 @@ spec: {{- if ne .Values.defaultTargetType "instance" }} - --default-target-type={{ .Values.defaultTargetType }} {{- end }} + {{- if .Values.serviceTargetENISGTags }} + - --service-target-eni-security-group-tags={{ .Values.serviceTargetENISGTags }} + {{- end }} {{- if or .Values.env .Values.envSecretName }} env: {{- if .Values.env}} diff --git a/helm/aws-load-balancer-controller/values.yaml b/helm/aws-load-balancer-controller/values.yaml index 4145743b7..03d45e7b9 100644 --- a/helm/aws-load-balancer-controller/values.yaml +++ b/helm/aws-load-balancer-controller/values.yaml @@ -15,6 +15,11 @@ imagePullSecrets: [] nameOverride: "" fullnameOverride: "" +# AWS LBC only has 1 main working pod, other pods are just standby +# the purpose of enable hpa is to survive load induced failure by the calls to the aws-load-balancer-webhook-service +# since the calls from kube-apiserver are sent round-robin to all replicas, and the failure policy on those webhooks is Fail +# if the pods become overloaded and do not respond within the timeout that could block the creation of pods, targetgroupbindings or ingresses +# Please keep in mind that the controller pods have `priorityClassName: system-cluster-critical`, enabling HPA may lead to the eviction of other low-priority pods in the node autoscaling: enabled: false minReplicas: 1 @@ -380,3 +385,6 @@ ingressClassConfig: # enableServiceMutatorWebhook allows you enable the webhook which makes this controller the default for all new services of type LoadBalancer enableServiceMutatorWebhook: true + +# serviceTargetENISGTags specifies AWS tags, in addition to the cluster tags, for finding the target ENI SG to which to add inbound rules from NLBs. +serviceTargetENISGTags: From 32a3339f7d2a48897a5548424a2859c583b0428b Mon Sep 17 00:00:00 2001 From: Shraddha Bang <18206078+shraddhabang@users.noreply.github.com> Date: Tue, 6 Feb 2024 20:19:28 -0800 Subject: [PATCH 2/4] Add a note to recommend to use compatible chart and image versions (#3559) --- helm/aws-load-balancer-controller/README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/helm/aws-load-balancer-controller/README.md b/helm/aws-load-balancer-controller/README.md index 32fc7042f..5dd580324 100644 --- a/helm/aws-load-balancer-controller/README.md +++ b/helm/aws-load-balancer-controller/README.md @@ -96,8 +96,11 @@ If you are setting `serviceMonitor.enabled: true` you need to have installed the ## Installing the Chart **Note**: You need to uninstall aws-alb-ingress-controller. Please refer to the [upgrade](#Upgrade) section below before you proceed. + **Note**: Starting chart version 1.4.1, you need to explicitly set `clusterSecretsPermissions.allowAllSecrets` to true to grant the controller permission to access all secrets for OIDC feature. We recommend configuring access to individual secrets resource separately [[link](https://kubernetes-sigs.github.io/aws-load-balancer-controller/v2.4/examples/secrets_access/)]. +**Note**: To ensure compatibility, we recommend installing the AWS Load Balancer controller image version with its compatible Helm chart version. Use the ```helm search repo eks/aws-load-balancer-controller --versions``` command to find the compatible versions. + Add the EKS repository to Helm: ```shell script helm repo add eks https://aws.github.io/eks-charts From e5fdc89492933faf78c07c36130b9e08f3b00294 Mon Sep 17 00:00:00 2001 From: Olivia Song Date: Thu, 8 Feb 2024 11:17:24 -0800 Subject: [PATCH 3/4] log enhancement for enabling RGT API (#3564) * log enhancement * change to debug level --- pkg/deploy/elbv2/listener_manager.go | 2 +- pkg/deploy/elbv2/tagging_manager.go | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/deploy/elbv2/listener_manager.go b/pkg/deploy/elbv2/listener_manager.go index 0fff98231..1f2e450be 100644 --- a/pkg/deploy/elbv2/listener_manager.go +++ b/pkg/deploy/elbv2/listener_manager.go @@ -161,7 +161,7 @@ func (m *defaultListenerManager) updateSDKListenerWithExtraCertificates(ctx cont sdkLS ListenerWithTags, isNewSDKListener bool) error { // if TLS is not supported, we shouldn't update if resLS.Spec.SSLPolicy == nil && sdkLS.Listener.SslPolicy == nil { - m.logger.V(1).Info("Res and Sdk Listener don't have SSL Policy set, we skip updating extra certs for non-TLS listener.") + m.logger.V(2).Info("Res and Sdk Listener don't have SSL Policy set, we skip updating extra certs for non-TLS listener.") return nil } diff --git a/pkg/deploy/elbv2/tagging_manager.go b/pkg/deploy/elbv2/tagging_manager.go index bea61659a..8e661425a 100644 --- a/pkg/deploy/elbv2/tagging_manager.go +++ b/pkg/deploy/elbv2/tagging_manager.go @@ -259,6 +259,7 @@ func (m *defaultTaggingManager) ListListenerRules(ctx context.Context, lsARN str // TODO: we can refactor this by store provisioned LB's ARN as annotations on Ingress/Service, thus avoid this heavy lookup calls when RGT is not available. func (m *defaultTaggingManager) ListLoadBalancers(ctx context.Context, tagFilters ...tracking.TagFilter) ([]LoadBalancerWithTags, error) { if m.featureGates.Enabled(config.EnableRGTAPI) { + m.logger.V(2).Info("ResourceGroupTagging enabled, list the load balancers via RGT API") return m.listLoadBalancersRGT(ctx, tagFilters) } return m.listLoadBalancersNative(ctx, tagFilters) @@ -266,6 +267,7 @@ func (m *defaultTaggingManager) ListLoadBalancers(ctx context.Context, tagFilter func (m *defaultTaggingManager) ListTargetGroups(ctx context.Context, tagFilters ...tracking.TagFilter) ([]TargetGroupWithTags, error) { if m.featureGates.Enabled(config.EnableRGTAPI) { + m.logger.V(2).Info("ResourceGroupTagging enabled, list the target groups via RGT API") return m.listTargetGroupsRGT(ctx, tagFilters) } return m.listTargetGroupsNative(ctx, tagFilters) From 95e362a647d407a83398dd087a0c5156d05535b8 Mon Sep 17 00:00:00 2001 From: Olivia Song Date: Mon, 12 Feb 2024 12:13:17 -0800 Subject: [PATCH 4/4] fix log level in listener manager and tagging manager (#3573) --- pkg/deploy/elbv2/listener_manager.go | 2 +- pkg/deploy/elbv2/tagging_manager.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/deploy/elbv2/listener_manager.go b/pkg/deploy/elbv2/listener_manager.go index 1f2e450be..22400da9d 100644 --- a/pkg/deploy/elbv2/listener_manager.go +++ b/pkg/deploy/elbv2/listener_manager.go @@ -161,7 +161,7 @@ func (m *defaultListenerManager) updateSDKListenerWithExtraCertificates(ctx cont sdkLS ListenerWithTags, isNewSDKListener bool) error { // if TLS is not supported, we shouldn't update if resLS.Spec.SSLPolicy == nil && sdkLS.Listener.SslPolicy == nil { - m.logger.V(2).Info("Res and Sdk Listener don't have SSL Policy set, we skip updating extra certs for non-TLS listener.") + m.logger.V(1).Info("Res and Sdk Listener don't have SSL Policy set, skip updating extra certs for non-TLS listener.") return nil } diff --git a/pkg/deploy/elbv2/tagging_manager.go b/pkg/deploy/elbv2/tagging_manager.go index 8e661425a..33caef61e 100644 --- a/pkg/deploy/elbv2/tagging_manager.go +++ b/pkg/deploy/elbv2/tagging_manager.go @@ -259,7 +259,7 @@ func (m *defaultTaggingManager) ListListenerRules(ctx context.Context, lsARN str // TODO: we can refactor this by store provisioned LB's ARN as annotations on Ingress/Service, thus avoid this heavy lookup calls when RGT is not available. func (m *defaultTaggingManager) ListLoadBalancers(ctx context.Context, tagFilters ...tracking.TagFilter) ([]LoadBalancerWithTags, error) { if m.featureGates.Enabled(config.EnableRGTAPI) { - m.logger.V(2).Info("ResourceGroupTagging enabled, list the load balancers via RGT API") + m.logger.V(1).Info("ResourceGroupTagging enabled, list the load balancers via RGT API") return m.listLoadBalancersRGT(ctx, tagFilters) } return m.listLoadBalancersNative(ctx, tagFilters) @@ -267,7 +267,7 @@ func (m *defaultTaggingManager) ListLoadBalancers(ctx context.Context, tagFilter func (m *defaultTaggingManager) ListTargetGroups(ctx context.Context, tagFilters ...tracking.TagFilter) ([]TargetGroupWithTags, error) { if m.featureGates.Enabled(config.EnableRGTAPI) { - m.logger.V(2).Info("ResourceGroupTagging enabled, list the target groups via RGT API") + m.logger.V(1).Info("ResourceGroupTagging enabled, list the target groups via RGT API") return m.listTargetGroupsRGT(ctx, tagFilters) } return m.listTargetGroupsNative(ctx, tagFilters)