From cbe94e09c6c1807a4864eb346fdba4df5d37a3d4 Mon Sep 17 00:00:00 2001 From: Weiwei Li Date: Tue, 24 Sep 2024 11:14:23 -0700 Subject: [PATCH] Support listener attributes --- .../elbv2/v1beta1/ingressclassparams_types.go | 20 ++ apis/elbv2/v1beta1/zz_generated.deepcopy.go | 27 +++ .../elbv2.k8s.aws_ingressclassparams.yaml | 30 +++ docs/install/iam_policy.json | 6 +- docs/install/iam_policy_cn.json | 6 +- docs/install/iam_policy_us-gov.json | 6 +- .../crds/crds.yaml | 30 +++ pkg/annotations/constants.go | 2 + pkg/aws/services/elbv2.go | 13 +- pkg/aws/services/elbv2_mocks.go | 30 +++ .../elbv2/listener_attributes_reconciler.go | 94 +++++++++ pkg/deploy/elbv2/listener_manager.go | 27 ++- pkg/ingress/model_build_listener.go | 86 +++++++- pkg/ingress/model_build_listener_test.go | 191 +++++++++++++++++- pkg/model/elbv2/listener.go | 10 + pkg/service/model_build_listener.go | 38 +++- pkg/service/model_build_listener_test.go | 119 +++++++++++ test/e2e/service/aws_resource_verifier.go | 28 ++- test/e2e/service/nlb_instance_target_test.go | 14 ++ test/framework/resources/aws/load_balancer.go | 12 ++ 20 files changed, 768 insertions(+), 21 deletions(-) create mode 100644 pkg/deploy/elbv2/listener_attributes_reconciler.go diff --git a/apis/elbv2/v1beta1/ingressclassparams_types.go b/apis/elbv2/v1beta1/ingressclassparams_types.go index 6fdfb3acb7..23479ff8f1 100644 --- a/apis/elbv2/v1beta1/ingressclassparams_types.go +++ b/apis/elbv2/v1beta1/ingressclassparams_types.go @@ -85,6 +85,22 @@ type Attribute struct { Value string `json:"value"` } +type ListenerProtocol string + +const ( + ListenerProtocolHTTP ListenerProtocol = "HTTP" + ListenerProtocolHTTPS ListenerProtocol = "HTTPS" +) + +type Listener struct { + // The protocol of the listener + Protocol ListenerProtocol `json:"protocol,omitempty"` + // The port of the listener + Port int32 `json:"port,omitempty"` + // The attributes of the listener + ListenerAttributes []Attribute `json:"listenerAttributes,omitempty"` +} + // IngressClassParamsSpec defines the desired state of IngressClassParams type IngressClassParamsSpec struct { // CertificateArn specifies the ARN of the certificates for all Ingresses that belong to IngressClass with this IngressClassParams. @@ -126,6 +142,10 @@ type IngressClassParamsSpec struct { // LoadBalancerAttributes define the custom attributes to LoadBalancers for all Ingress that that belong to IngressClass with this IngressClassParams. // +optional LoadBalancerAttributes []Attribute `json:"loadBalancerAttributes,omitempty"` + + // Listeners define a list of listeners with their protocol, port and attributes. + // +optional + Listeners []Listener `json:"listeners,omitempty"` } // +kubebuilder:object:root=true diff --git a/apis/elbv2/v1beta1/zz_generated.deepcopy.go b/apis/elbv2/v1beta1/zz_generated.deepcopy.go index 5cbf8a21a9..d4630526ab 100644 --- a/apis/elbv2/v1beta1/zz_generated.deepcopy.go +++ b/apis/elbv2/v1beta1/zz_generated.deepcopy.go @@ -162,6 +162,13 @@ func (in *IngressClassParamsSpec) DeepCopyInto(out *IngressClassParamsSpec) { *out = make([]Attribute, len(*in)) copy(*out, *in) } + if in.Listeners != nil { + in, out := &in.Listeners, &out.Listeners + *out = make([]Listener, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new IngressClassParamsSpec. @@ -189,6 +196,26 @@ func (in *IngressGroup) DeepCopy() *IngressGroup { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Listener) DeepCopyInto(out *Listener) { + *out = *in + if in.ListenerAttributes != nil { + in, out := &in.ListenerAttributes, &out.ListenerAttributes + *out = make([]Attribute, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Listener. +func (in *Listener) DeepCopy() *Listener { + if in == nil { + return nil + } + out := new(Listener) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *NetworkingIngressRule) DeepCopyInto(out *NetworkingIngressRule) { *out = *in diff --git a/config/crd/bases/elbv2.k8s.aws_ingressclassparams.yaml b/config/crd/bases/elbv2.k8s.aws_ingressclassparams.yaml index ba9ae4f623..26b39f0e8c 100644 --- a/config/crd/bases/elbv2.k8s.aws_ingressclassparams.yaml +++ b/config/crd/bases/elbv2.k8s.aws_ingressclassparams.yaml @@ -85,6 +85,36 @@ spec: - dualstack - dualstack-without-public-ipv4 type: string + listeners: + description: Listeners define a list of listeners with their protocol, + port and attributes. + items: + properties: + listenerAttributes: + description: The attributes of the listener + items: + description: Attributes defines custom attributes on resources. + properties: + key: + description: The key of the attribute. + type: string + value: + description: The value of the attribute. + type: string + required: + - key + - value + type: object + type: array + port: + description: The port of the listener + format: int32 + type: integer + protocol: + description: The protocol of the listener + type: string + type: object + type: array loadBalancerAttributes: description: LoadBalancerAttributes define the custom attributes to LoadBalancers for all Ingress that that belong to IngressClass with diff --git a/docs/install/iam_policy.json b/docs/install/iam_policy.json index e8a05f8e64..0480ec2db3 100644 --- a/docs/install/iam_policy.json +++ b/docs/install/iam_policy.json @@ -39,7 +39,8 @@ "elasticloadbalancing:DescribeTargetGroupAttributes", "elasticloadbalancing:DescribeTargetHealth", "elasticloadbalancing:DescribeTags", - "elasticloadbalancing:DescribeTrustStores" + "elasticloadbalancing:DescribeTrustStores", + "elasticloadbalancing:DescribeListenerAttributes" ], "Resource": "*" }, @@ -188,7 +189,8 @@ "elasticloadbalancing:DeleteLoadBalancer", "elasticloadbalancing:ModifyTargetGroup", "elasticloadbalancing:ModifyTargetGroupAttributes", - "elasticloadbalancing:DeleteTargetGroup" + "elasticloadbalancing:DeleteTargetGroup", + "elasticloadbalancing:ModifyListenerAttributes" ], "Resource": "*", "Condition": { diff --git a/docs/install/iam_policy_cn.json b/docs/install/iam_policy_cn.json index cb8bc040e3..b009bd74d8 100644 --- a/docs/install/iam_policy_cn.json +++ b/docs/install/iam_policy_cn.json @@ -39,7 +39,8 @@ "elasticloadbalancing:DescribeTargetGroupAttributes", "elasticloadbalancing:DescribeTargetHealth", "elasticloadbalancing:DescribeTags", - "elasticloadbalancing:DescribeTrustStores" + "elasticloadbalancing:DescribeTrustStores", + "elasticloadbalancing:DescribeListenerAttributes" ], "Resource": "*" }, @@ -210,7 +211,8 @@ "elasticloadbalancing:DeleteLoadBalancer", "elasticloadbalancing:ModifyTargetGroup", "elasticloadbalancing:ModifyTargetGroupAttributes", - "elasticloadbalancing:DeleteTargetGroup" + "elasticloadbalancing:DeleteTargetGroup", + "elasticloadbalancing:ModifyListenerAttributes" ], "Resource": "*", "Condition": { diff --git a/docs/install/iam_policy_us-gov.json b/docs/install/iam_policy_us-gov.json index 97ccf2ebbf..a50e4656c4 100644 --- a/docs/install/iam_policy_us-gov.json +++ b/docs/install/iam_policy_us-gov.json @@ -39,7 +39,8 @@ "elasticloadbalancing:DescribeTargetGroupAttributes", "elasticloadbalancing:DescribeTargetHealth", "elasticloadbalancing:DescribeTags", - "elasticloadbalancing:DescribeTrustStores" + "elasticloadbalancing:DescribeTrustStores", + "elasticloadbalancing:DescribeListenerAttributes" ], "Resource": "*" }, @@ -210,7 +211,8 @@ "elasticloadbalancing:DeleteLoadBalancer", "elasticloadbalancing:ModifyTargetGroup", "elasticloadbalancing:ModifyTargetGroupAttributes", - "elasticloadbalancing:DeleteTargetGroup" + "elasticloadbalancing:DeleteTargetGroup", + "elasticloadbalancing:ModifyListenerAttributes" ], "Resource": "*", "Condition": { diff --git a/helm/aws-load-balancer-controller/crds/crds.yaml b/helm/aws-load-balancer-controller/crds/crds.yaml index 3fcd14ea4c..e2d92380ba 100644 --- a/helm/aws-load-balancer-controller/crds/crds.yaml +++ b/helm/aws-load-balancer-controller/crds/crds.yaml @@ -84,6 +84,36 @@ spec: - dualstack - dualstack-without-public-ipv4 type: string + listeners: + description: Listeners define a list of listeners with their protocol, + port and attributes. + items: + properties: + listenerAttributes: + description: The attributes of the listener + items: + description: Attributes defines custom attributes on resources. + properties: + key: + description: The key of the attribute. + type: string + value: + description: The value of the attribute. + type: string + required: + - key + - value + type: object + type: array + port: + description: The port of the listener + format: int32 + type: integer + protocol: + description: The protocol of the listener + type: string + type: object + type: array loadBalancerAttributes: description: LoadBalancerAttributes define the custom attributes to LoadBalancers for all Ingress that that belong to IngressClass with diff --git a/pkg/annotations/constants.go b/pkg/annotations/constants.go index 493e0427ef..2c738a2e64 100644 --- a/pkg/annotations/constants.go +++ b/pkg/annotations/constants.go @@ -48,6 +48,7 @@ const ( IngressSuffixManageSecurityGroupRules = "manage-backend-security-group-rules" IngressSuffixMutualAuthentication = "mutual-authentication" IngressSuffixSecurityGroupPrefixLists = "security-group-prefix-lists" + IngressSuffixlsAttsAnnotationPrefix = "listener-attributes" // NLB annotation suffixes // prefixes service.beta.kubernetes.io, service.kubernetes.io @@ -88,4 +89,5 @@ const ( SvcLBSuffixManageSGRules = "aws-load-balancer-manage-backend-security-group-rules" SvcLBSuffixEnforceSGInboundRulesOnPrivateLinkTraffic = "aws-load-balancer-inbound-sg-rules-on-private-link-traffic" SvcLBSuffixSecurityGroupPrefixLists = "aws-load-balancer-security-group-prefix-lists" + SvcLBSuffixlsAttsAnnotationPrefix = "aws-load-balancer-listener-attributes" ) diff --git a/pkg/aws/services/elbv2.go b/pkg/aws/services/elbv2.go index 731d1e5ee2..0ff0e7d187 100644 --- a/pkg/aws/services/elbv2.go +++ b/pkg/aws/services/elbv2.go @@ -2,11 +2,12 @@ package services import ( "context" + "time" + "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2" "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2/types" "sigs.k8s.io/aws-load-balancer-controller/pkg/aws/endpoints" - "time" ) type ELBV2 interface { @@ -57,6 +58,8 @@ type ELBV2 interface { DescribeTrustStoresWithContext(ctx context.Context, input *elasticloadbalancingv2.DescribeTrustStoresInput) (*elasticloadbalancingv2.DescribeTrustStoresOutput, error) RemoveListenerCertificatesWithContext(ctx context.Context, input *elasticloadbalancingv2.RemoveListenerCertificatesInput) (*elasticloadbalancingv2.RemoveListenerCertificatesOutput, error) AddListenerCertificatesWithContext(ctx context.Context, input *elasticloadbalancingv2.AddListenerCertificatesInput) (*elasticloadbalancingv2.AddListenerCertificatesOutput, error) + DescribeListenerAttributesWithContext(ctx context.Context, input *elasticloadbalancingv2.DescribeListenerAttributesInput) (*elasticloadbalancingv2.DescribeListenerAttributesOutput, error) + ModifyListenerAttributesWithContext(ctx context.Context, input *elasticloadbalancingv2.ModifyListenerAttributesInput) (*elasticloadbalancingv2.ModifyListenerAttributesOutput, error) } func NewELBV2(cfg aws.Config, endpointsResolver *endpoints.Resolver) ELBV2 { @@ -268,3 +271,11 @@ func (c *elbv2Client) DescribeRulesAsList(ctx context.Context, input *elasticloa } return result, nil } + +func (c *elbv2Client) DescribeListenerAttributesWithContext(ctx context.Context, input *elasticloadbalancingv2.DescribeListenerAttributesInput) (*elasticloadbalancingv2.DescribeListenerAttributesOutput, error) { + return c.elbv2Client.DescribeListenerAttributes(ctx, input) +} + +func (c *elbv2Client) ModifyListenerAttributesWithContext(ctx context.Context, input *elasticloadbalancingv2.ModifyListenerAttributesInput) (*elasticloadbalancingv2.ModifyListenerAttributesOutput, error) { + return c.elbv2Client.ModifyListenerAttributes(ctx, input) +} diff --git a/pkg/aws/services/elbv2_mocks.go b/pkg/aws/services/elbv2_mocks.go index c6bb0a55c9..42c82464c6 100644 --- a/pkg/aws/services/elbv2_mocks.go +++ b/pkg/aws/services/elbv2_mocks.go @@ -201,6 +201,21 @@ func (mr *MockELBV2MockRecorder) DeregisterTargetsWithContext(arg0, arg1 interfa return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeregisterTargetsWithContext", reflect.TypeOf((*MockELBV2)(nil).DeregisterTargetsWithContext), arg0, arg1) } +// DescribeListenerAttributesWithContext mocks base method. +func (m *MockELBV2) DescribeListenerAttributesWithContext(arg0 context.Context, arg1 *elasticloadbalancingv2.DescribeListenerAttributesInput) (*elasticloadbalancingv2.DescribeListenerAttributesOutput, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DescribeListenerAttributesWithContext", arg0, arg1) + ret0, _ := ret[0].(*elasticloadbalancingv2.DescribeListenerAttributesOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// DescribeListenerAttributesWithContext indicates an expected call of DescribeListenerAttributesWithContext. +func (mr *MockELBV2MockRecorder) DescribeListenerAttributesWithContext(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DescribeListenerAttributesWithContext", reflect.TypeOf((*MockELBV2)(nil).DescribeListenerAttributesWithContext), arg0, arg1) +} + // DescribeListenerCertificatesAsList mocks base method. func (m *MockELBV2) DescribeListenerCertificatesAsList(arg0 context.Context, arg1 *elasticloadbalancingv2.DescribeListenerCertificatesInput) ([]types.Certificate, error) { m.ctrl.T.Helper() @@ -411,6 +426,21 @@ func (mr *MockELBV2MockRecorder) DescribeTrustStoresWithContext(arg0, arg1 inter return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DescribeTrustStoresWithContext", reflect.TypeOf((*MockELBV2)(nil).DescribeTrustStoresWithContext), arg0, arg1) } +// ModifyListenerAttributesWithContext mocks base method. +func (m *MockELBV2) ModifyListenerAttributesWithContext(arg0 context.Context, arg1 *elasticloadbalancingv2.ModifyListenerAttributesInput) (*elasticloadbalancingv2.ModifyListenerAttributesOutput, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ModifyListenerAttributesWithContext", arg0, arg1) + ret0, _ := ret[0].(*elasticloadbalancingv2.ModifyListenerAttributesOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ModifyListenerAttributesWithContext indicates an expected call of ModifyListenerAttributesWithContext. +func (mr *MockELBV2MockRecorder) ModifyListenerAttributesWithContext(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ModifyListenerAttributesWithContext", reflect.TypeOf((*MockELBV2)(nil).ModifyListenerAttributesWithContext), arg0, arg1) +} + // ModifyListenerWithContext mocks base method. func (m *MockELBV2) ModifyListenerWithContext(arg0 context.Context, arg1 *elasticloadbalancingv2.ModifyListenerInput) (*elasticloadbalancingv2.ModifyListenerOutput, error) { m.ctrl.T.Helper() diff --git a/pkg/deploy/elbv2/listener_attributes_reconciler.go b/pkg/deploy/elbv2/listener_attributes_reconciler.go new file mode 100644 index 0000000000..d909bbec74 --- /dev/null +++ b/pkg/deploy/elbv2/listener_attributes_reconciler.go @@ -0,0 +1,94 @@ +package elbv2 + +import ( + "context" + + awssdk "github.com/aws/aws-sdk-go-v2/aws" + elbv2sdk "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2" + elbv2types "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2/types" + "github.com/go-logr/logr" + "k8s.io/apimachinery/pkg/util/sets" + "sigs.k8s.io/aws-load-balancer-controller/pkg/algorithm" + "sigs.k8s.io/aws-load-balancer-controller/pkg/aws/services" + elbv2model "sigs.k8s.io/aws-load-balancer-controller/pkg/model/elbv2" +) + +// Reconciler for Listener attributes +type ListenerAttributesReconciler interface { + Reconcile(ctx context.Context, resLS *elbv2model.Listener, sdkLS ListenerWithTags) error +} + +// NewListenerAttributesReconciler constructs new ListenerAttributesReconciler. +func NewDefaultListenerAttributesReconciler(elbv2Client services.ELBV2, logger logr.Logger) *defaultListenerAttributesReconciler { + return &defaultListenerAttributesReconciler{ + elbv2Client: elbv2Client, + logger: logger, + } +} + +var _ ListenerAttributesReconciler = &defaultListenerAttributesReconciler{} + +// default implementation for ListenerAttributeReconciler +type defaultListenerAttributesReconciler struct { + elbv2Client services.ELBV2 + logger logr.Logger +} + +func (r *defaultListenerAttributesReconciler) Reconcile(ctx context.Context, resLS *elbv2model.Listener, sdkLS ListenerWithTags) error { + desiredAttrs := r.getDesiredListenerAttributes(ctx, resLS) + currentAttrs, err := r.getCurrentListenerAttributes(ctx, sdkLS) + if err != nil { + return err + } + attributesToUpdate, _ := algorithm.DiffStringMap(desiredAttrs, currentAttrs) + if len(attributesToUpdate) > 0 { + req := &elbv2sdk.ModifyListenerAttributesInput{ + ListenerArn: sdkLS.Listener.ListenerArn, + Attributes: nil, + } + for _, attrKey := range sets.StringKeySet(attributesToUpdate).List() { + req.Attributes = append(req.Attributes, elbv2types.ListenerAttribute{ + Key: awssdk.String(attrKey), + Value: awssdk.String(attributesToUpdate[attrKey]), + }) + } + r.logger.Info("modifying listener attributes", + "stackID", resLS.Stack().StackID(), + "resourceID", resLS.ID(), + "arn", awssdk.ToString(sdkLS.Listener.ListenerArn), + "change", attributesToUpdate) + if _, err := r.elbv2Client.ModifyListenerAttributesWithContext(ctx, req); err != nil { + return err + } + r.logger.Info("modified listener attribute", + "stackID", resLS.Stack().StackID(), + "resourceID", resLS.ID(), + "arn", awssdk.ToString(sdkLS.Listener.ListenerArn)) + + } + return nil + +} + +func (r *defaultListenerAttributesReconciler) getDesiredListenerAttributes(ctx context.Context, resLS *elbv2model.Listener) map[string]string { + lsAttributes := make(map[string]string, len(resLS.Spec.ListenerAttributes)) + for _, attr := range resLS.Spec.ListenerAttributes { + lsAttributes[attr.Key] = attr.Value + } + return lsAttributes +} + +func (r *defaultListenerAttributesReconciler) getCurrentListenerAttributes(ctx context.Context, sdkLS ListenerWithTags) (map[string]string, error) { + req := &elbv2sdk.DescribeListenerAttributesInput{ + ListenerArn: sdkLS.Listener.ListenerArn, + } + resp, err := r.elbv2Client.DescribeListenerAttributesWithContext(ctx, req) + if err != nil { + return nil, err + } + lsAttributes := make(map[string]string, len(resp.Attributes)) + for _, attr := range resp.Attributes { + lsAttributes[awssdk.ToString(attr.Key)] = awssdk.ToString(attr.Value) + } + return lsAttributes, nil +} diff --git a/pkg/deploy/elbv2/listener_manager.go b/pkg/deploy/elbv2/listener_manager.go index 2f635e5357..4cc44e1ec3 100644 --- a/pkg/deploy/elbv2/listener_manager.go +++ b/pkg/deploy/elbv2/listener_manager.go @@ -2,12 +2,12 @@ package elbv2 import ( "context" - elbv2types "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2/types" "reflect" "time" awssdk "github.com/aws/aws-sdk-go-v2/aws" elbv2sdk "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2" + elbv2types "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2/types" "github.com/go-logr/logr" "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" @@ -41,6 +41,7 @@ func NewDefaultListenerManager(elbv2Client services.ELBV2, trackingProvider trac logger: logger, waitLSExistencePollInterval: defaultWaitLSExistencePollInterval, waitLSExistenceTimeout: defaultWaitLSExistenceTimeout, + attributesReconciler: NewDefaultListenerAttributesReconciler(elbv2Client, logger), } } @@ -57,6 +58,7 @@ type defaultListenerManager struct { waitLSExistencePollInterval time.Duration waitLSExistenceTimeout time.Duration + attributesReconciler ListenerAttributesReconciler } func (m *defaultListenerManager) Create(ctx context.Context, resLS *elbv2model.Listener) (elbv2model.ListenerStatus, error) { @@ -91,6 +93,11 @@ func (m *defaultListenerManager) Create(ctx context.Context, resLS *elbv2model.L }); err != nil { return elbv2model.ListenerStatus{}, errors.Wrap(err, "failed to update extra certificates on listener") } + if areListenerAttributesSupported(resLS.Spec.Protocol) { + if err := m.attributesReconciler.Reconcile(ctx, resLS, sdkLS); err != nil { + return elbv2model.ListenerStatus{}, err + } + } return buildResListenerStatus(sdkLS), nil } @@ -106,6 +113,11 @@ func (m *defaultListenerManager) Update(ctx context.Context, resLS *elbv2model.L if err := m.updateSDKListenerWithExtraCertificates(ctx, resLS, sdkLS, false); err != nil { return elbv2model.ListenerStatus{}, err } + if areListenerAttributesSupported(resLS.Spec.Protocol) { + if err := m.attributesReconciler.Reconcile(ctx, resLS, sdkLS); err != nil { + return elbv2model.ListenerStatus{}, err + } + } return buildResListenerStatus(sdkLS), nil } @@ -354,3 +366,16 @@ func buildResListenerStatus(sdkLS ListenerWithTags) elbv2model.ListenerStatus { ListenerARN: awssdk.ToString(sdkLS.Listener.ListenerArn), } } + +var PROTOCOLS_WITH_ATTRIBUTES = map[elbv2model.Protocol]bool{ + elbv2model.ProtocolHTTP: false, + elbv2model.ProtocolHTTPS: false, + elbv2model.ProtocolTCP: true, + elbv2model.ProtocolUDP: false, + elbv2model.ProtocolTLS: false, +} + +func areListenerAttributesSupported(protocol elbv2model.Protocol) bool { + supported, exists := PROTOCOLS_WITH_ATTRIBUTES[protocol] + return exists && supported +} diff --git a/pkg/ingress/model_build_listener.go b/pkg/ingress/model_build_listener.go index 73605aef56..31757773e5 100644 --- a/pkg/ingress/model_build_listener.go +++ b/pkg/ingress/model_build_listener.go @@ -4,11 +4,12 @@ import ( "context" "encoding/json" "fmt" - elbv2sdk "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2" - "k8s.io/utils/strings/slices" "net" "strings" + elbv2sdk "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2" + "k8s.io/utils/strings/slices" + awssdk "github.com/aws/aws-sdk-go-v2/aws" "github.com/pkg/errors" networking "k8s.io/api/networking/v1" @@ -46,6 +47,10 @@ func (t *defaultModelBuildTask) buildListenerSpec(ctx context.Context, lbARN cor CertificateARN: awssdk.String(certARN), }) } + lsAttributes, attributesErr := t.buildListenerAttributes(ctx, ingList, port, config.protocol) + if attributesErr != nil { + return elbv2model.ListenerSpec{}, attributesErr + } return elbv2model.ListenerSpec{ LoadBalancerARN: lbARN, Port: port, @@ -55,6 +60,7 @@ func (t *defaultModelBuildTask) buildListenerSpec(ctx context.Context, lbARN cor SSLPolicy: config.sslPolicy, MutualAuthentication: config.mutualAuthentication, Tags: tags, + ListenerAttributes: lsAttributes, }, nil } @@ -98,6 +104,14 @@ func (t *defaultModelBuildTask) buildListenerTags(_ context.Context, ingList []C return algorithm.MergeStringMap(t.defaultTags, ingGroupTags), nil } +func (t *defaultModelBuildTask) buildListenerAttributes(ctx context.Context, ingList []ClassifiedIngress, port int32, listenerProtocol elbv2model.Protocol) ([]elbv2model.ListenerAttribute, error) { + ingGroupListenerAttributes, err := t.buildIngressGroupListenerAttributes(ctx, ingList, listenerProtocol, port) + if err != nil { + return nil, err + } + return ingGroupListenerAttributes, nil +} + // the listen port config for specific listener port. type listenPortConfig struct { protocol elbv2model.Protocol @@ -409,3 +423,71 @@ func (t *defaultModelBuildTask) fetchTrustStoreArnFromName(ctx context.Context, } return tsNameAndArnMap, nil } + +func (t *defaultModelBuildTask) buildIngressGroupListenerAttributes(ctx context.Context, ingList []ClassifiedIngress, listenerProtocol elbv2model.Protocol, port int32) ([]elbv2model.ListenerAttribute, error) { + rawIngGrouplistenerAttributes := make(map[string]string) + for _, ing := range ingList { + ingAttributes, err := t.buildIngressListenerAttributes(ctx, ing.Ing.Annotations, port, listenerProtocol) + if err != nil { + return nil, err + } + for _, attribute := range ingAttributes { + attributeKey := attribute.Key + attributeValue := attribute.Value + if existingAttributeValue, exists := rawIngGrouplistenerAttributes[attributeKey]; exists && existingAttributeValue != attributeValue { + return nil, errors.Errorf("conflicting attributes %v: %v | %v", attributeKey, existingAttributeValue, attributeValue) + } + rawIngGrouplistenerAttributes[attributeKey] = attributeValue + } + } + if len(ingList) > 0 { + ingClassAttributes, err := t.buildIngressClassListenerAttributes(ingList[0].IngClassConfig, listenerProtocol, port) + if err != nil { + return nil, err + } + rawIngGrouplistenerAttributes = algorithm.MergeStringMap(ingClassAttributes, rawIngGrouplistenerAttributes) + } + attributes := make([]elbv2model.ListenerAttribute, 0, len(rawIngGrouplistenerAttributes)) + for attrKey, attrValue := range rawIngGrouplistenerAttributes { + attributes = append(attributes, elbv2model.ListenerAttribute{ + Key: attrKey, + Value: attrValue, + }) + } + return attributes, nil +} + +// buildIngressClassLoadBalancerAttributes builds the LB attributes for an IngressClass. +func (t *defaultModelBuildTask) buildIngressClassListenerAttributes(ingClassConfig ClassConfiguration, listenerProtocol elbv2model.Protocol, port int32) (map[string]string, error) { + if ingClassConfig.IngClassParams == nil || len(ingClassConfig.IngClassParams.Spec.Listeners) == 0 { + return nil, nil + } + listeners := ingClassConfig.IngClassParams.Spec.Listeners + ingressClassListenerAttributes := make(map[string]string) + for _, listenerConfig := range listeners { + if string(listenerConfig.Protocol) == string(listenerProtocol) && listenerConfig.Port == port { + for _, attr := range listenerConfig.ListenerAttributes { + ingressClassListenerAttributes[attr.Key] = attr.Value + } + return ingressClassListenerAttributes, nil + } + } + return nil, nil +} + +// Build attributes for listener +func (t *defaultModelBuildTask) buildIngressListenerAttributes(ctx context.Context, ingressAnnotations map[string]string, port int32, listenerProtocol elbv2model.Protocol) ([]elbv2model.ListenerAttribute, error) { + var rawAttributes map[string]string + annotationKey := fmt.Sprintf("%v.%v-%v", annotations.IngressSuffixlsAttsAnnotationPrefix, listenerProtocol, port) + if _, err := t.annotationParser.ParseStringMapAnnotation(annotationKey, &rawAttributes, ingressAnnotations); err != nil { + return nil, err + } + attributes := make([]elbv2model.ListenerAttribute, 0, len(rawAttributes)) + for attrKey, attrValue := range rawAttributes { + attributes = append(attributes, elbv2model.ListenerAttribute{ + Key: attrKey, + Value: attrValue, + }) + } + return attributes, nil +} diff --git a/pkg/ingress/model_build_listener_test.go b/pkg/ingress/model_build_listener_test.go index d52b4e9b24..ca856b6d9a 100644 --- a/pkg/ingress/model_build_listener_test.go +++ b/pkg/ingress/model_build_listener_test.go @@ -2,12 +2,14 @@ package ingress import ( "context" + "testing" + "github.com/stretchr/testify/assert" networking "k8s.io/api/networking/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/aws-load-balancer-controller/pkg/annotations" "sigs.k8s.io/aws-load-balancer-controller/pkg/model/elbv2" - "testing" + elbv2model "sigs.k8s.io/aws-load-balancer-controller/pkg/model/elbv2" ) func Test_computeIngressListenPortConfigByPort_MutualAuthentication(t *testing.T) { @@ -103,3 +105,190 @@ func Test_computeIngressListenPortConfigByPort_MutualAuthentication(t *testing.T }) } } +func Test_buildListenerAttributes(t *testing.T) { + type fields struct { + ingGroup Group + } + + tests := []struct { + name string + fields fields + + wantErr bool + wantValue []elbv2model.ListenerAttribute + }{ + { + name: "Listener attribute annotation value is not stringMap", + fields: fields{ + ingGroup: Group{ + ID: GroupID{Name: "explicit-group"}, + Members: []ClassifiedIngress{ + { + Ing: &networking.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "awesome-ns", + Name: "ing-1", + Annotations: map[string]string{ + "alb.ingress.kubernetes.io/listen-ports": `[{"HTTP": 80}]`, + "alb.ingress.kubernetes.io/listener-attributes.HTTP-80": "attrKey", + }, + }, + }, + }, + }, + }, + }, + wantErr: true, + }, + { + name: "Listener attribute annotation is not specified", + fields: fields{ + ingGroup: Group{ + ID: GroupID{Name: "explicit-group"}, + Members: []ClassifiedIngress{ + { + Ing: &networking.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "awesome-ns", + Name: "ing-2", + Annotations: map[string]string{ + "alb.ingress.kubernetes.io/listen-ports": `[{"HTTP": 80}]`, + }, + }, + }, + }, + }, + }, + }, + wantErr: false, + wantValue: []elbv2model.ListenerAttribute{}, + }, + { + name: "Listener attribute annotation is specified", + fields: fields{ + ingGroup: Group{ + ID: GroupID{Name: "explicit-group"}, + Members: []ClassifiedIngress{ + { + Ing: &networking.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "awesome-ns", + Name: "ing-3", + Annotations: map[string]string{ + "alb.ingress.kubernetes.io/listen-ports": `[{"HTTP": 80}]`, + "alb.ingress.kubernetes.io/listener-attributes.HTTP-80": "routing.http.response.server.enabled=false", + }, + }, + }, + }, + }, + }, + }, + wantErr: false, + wantValue: []elbv2model.ListenerAttribute{ + { + Key: "routing.http.response.server.enabled", + Value: "false", + }, + }, + }, + { + name: "Listener attribute conflict", + fields: fields{ + ingGroup: Group{ + ID: GroupID{Name: "explicit-group"}, + Members: []ClassifiedIngress{ + { + Ing: &networking.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "awesome-ns", + Name: "ing-4", + Annotations: map[string]string{ + "alb.ingress.kubernetes.io/listen-ports": `[{"HTTP": 80}]`, + "alb.ingress.kubernetes.io/listener-attributes.HTTP-80": "routing.http.response.server.enabled=false", + }, + }, + }, + }, + { + Ing: &networking.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "awesome-ns", + Name: "ing-5", + Annotations: map[string]string{ + "alb.ingress.kubernetes.io/listen-ports": `[{"HTTP": 80}]`, + "alb.ingress.kubernetes.io/listener-attributes.HTTP-80": "routing.http.response.server.enabled=true", + }, + }, + }, + }, + }, + }, + }, + wantErr: true, + }, + { + name: "merge Listener attributes", + fields: fields{ + ingGroup: Group{ + ID: GroupID{Name: "explicit-group"}, + Members: []ClassifiedIngress{ + { + Ing: &networking.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "awesome-ns", + Name: "ing-4", + Annotations: map[string]string{ + "alb.ingress.kubernetes.io/listen-ports": `[{"HTTP": 80}]`, + "alb.ingress.kubernetes.io/listener-attributes.HTTP-80": "attrKey1=attrValue1", + }, + }, + }, + }, + { + Ing: &networking.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "awesome-ns", + Name: "ing-5", + Annotations: map[string]string{ + "alb.ingress.kubernetes.io/listen-ports": `[{"HTTP": 80}]`, + "alb.ingress.kubernetes.io/listener-attributes.HTTP-80": "attrKey2=attrValue2", + }, + }, + }, + }, + }, + }, + }, + wantErr: false, + wantValue: []elbv2model.ListenerAttribute{ + { + Key: "attrKey1", + Value: "attrValue1", + }, + { + Key: "attrKey2", + Value: "attrValue2", + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + task := &defaultModelBuildTask{ + ingGroup: tt.fields.ingGroup, + annotationParser: annotations.NewSuffixAnnotationParser("alb.ingress.kubernetes.io"), + } + + listenerAttributes, err := task.buildListenerAttributes(context.Background(), tt.fields.ingGroup.Members, 80, "HTTP") + t.Log(listenerAttributes) + + if tt.wantErr { + assert.Error(t, err) + } else { + assert.ElementsMatch(t, tt.wantValue, listenerAttributes) + } + + }) + } +} diff --git a/pkg/model/elbv2/listener.go b/pkg/model/elbv2/listener.go index e7cec3e621..19aeef5e16 100644 --- a/pkg/model/elbv2/listener.go +++ b/pkg/model/elbv2/listener.go @@ -3,6 +3,7 @@ package elbv2 import ( "context" "encoding/json" + "github.com/pkg/errors" "sigs.k8s.io/aws-load-balancer-controller/pkg/model/core" ) @@ -357,6 +358,10 @@ type ListenerSpec struct { // The tags. // +optional Tags map[string]string `json:"tags,omitempty"` + + // Listener attributes + // +optional + ListenerAttributes []ListenerAttribute `json:"listenerAttributes,omitempty"` } // ListenerStatus defines the observed state of Listener @@ -364,3 +369,8 @@ type ListenerStatus struct { // The Amazon Resource Name (ARN) of the listener. ListenerARN string `json:"listenerARN"` } + +type ListenerAttribute struct { + Key string `type:"string"` + Value string `type:"string"` +} diff --git a/pkg/service/model_build_listener.go b/pkg/service/model_build_listener.go index 41f33404f2..36aafb697d 100644 --- a/pkg/service/model_build_listener.go +++ b/pkg/service/model_build_listener.go @@ -73,15 +73,20 @@ func (t *defaultModelBuildTask) buildListenerSpec(ctx context.Context, port core } defaultActions := t.buildListenerDefaultActions(ctx, targetGroup) + lsAttributes, attributesErr := t.buildListenerAttributes(ctx, t.service.Annotations, port.Port, listenerProtocol) + if attributesErr != nil { + return elbv2model.ListenerSpec{}, attributesErr + } return elbv2model.ListenerSpec{ - LoadBalancerARN: t.loadBalancer.LoadBalancerARN(), - Port: port.Port, - Protocol: listenerProtocol, - Certificates: certificates, - SSLPolicy: sslPolicy, - ALPNPolicy: alpnPolicy, - DefaultActions: defaultActions, - Tags: tags, + LoadBalancerARN: t.loadBalancer.LoadBalancerARN(), + Port: port.Port, + Protocol: listenerProtocol, + Certificates: certificates, + SSLPolicy: sslPolicy, + ALPNPolicy: alpnPolicy, + DefaultActions: defaultActions, + Tags: tags, + ListenerAttributes: lsAttributes, }, nil } @@ -212,3 +217,20 @@ func (t *defaultModelBuildTask) buildListenerConfig(ctx context.Context) (*liste func (t *defaultModelBuildTask) buildListenerTags(ctx context.Context) (map[string]string, error) { return t.buildAdditionalResourceTags(ctx) } + +// Build attributes for listener +func (t *defaultModelBuildTask) buildListenerAttributes(ctx context.Context, svcAnnotations map[string]string, port int32, listenerProtocol elbv2model.Protocol) ([]elbv2model.ListenerAttribute, error) { + var rawAttributes map[string]string + annotationKey := fmt.Sprintf("%v.%v-%v", annotations.SvcLBSuffixlsAttsAnnotationPrefix, listenerProtocol, port) + if _, err := t.annotationParser.ParseStringMapAnnotation(annotationKey, &rawAttributes, svcAnnotations); err != nil { + return nil, err + } + attributes := make([]elbv2model.ListenerAttribute, 0, len(rawAttributes)) + for attrKey, attrValue := range rawAttributes { + attributes = append(attributes, elbv2model.ListenerAttribute{ + Key: attrKey, + Value: attrValue, + }) + } + return attributes, nil +} diff --git a/pkg/service/model_build_listener_test.go b/pkg/service/model_build_listener_test.go index 25d0ac76cb..9fd455afa0 100644 --- a/pkg/service/model_build_listener_test.go +++ b/pkg/service/model_build_listener_test.go @@ -182,3 +182,122 @@ func Test_defaultModelBuilderTask_buildListenerConfig(t *testing.T) { }) } } + +const tcpIdleTimeoutSeconds = "tcp.idle_timeout.seconds" + +func Test_defaultModelBuilderTask_buildListenerAttributes(t *testing.T) { + tests := []struct { + testName string + svc *corev1.Service + wantError bool + wantValue [][]elbv2model.ListenerAttribute + }{ + { + testName: "Listener attribute annotation value is not stringMap", + svc: &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + "service.beta.kubernetes.io/aws-load-balancer-type": "instance", + "service.beta.kubernetes.io/aws-load-balancer-listener-attributes.TCP-80": "tcp.idle_timeout.seconds", + }, + }, + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{ + { + Name: "http", + Port: 80, + TargetPort: intstr.FromInt(8080), + Protocol: corev1.ProtocolTCP, + NodePort: 38888, + }, + }, + }, + }, + wantError: true, + }, + { + testName: "Listener attribute annotation is not specified", + svc: &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + "service.beta.kubernetes.io/aws-load-balancer-type": "instance", + }, + }, + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{ + { + Name: "http", + Port: 80, + TargetPort: intstr.FromInt(8080), + Protocol: corev1.ProtocolTCP, + NodePort: 38888, + }, + }, + }, + }, + wantError: false, + wantValue: [][]elbv2model.ListenerAttribute{ + {}, + }, + }, + { + testName: "Listener attribute annotation is specified", + svc: &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + "service.beta.kubernetes.io/aws-load-balancer-type": "ip", + "service.beta.kubernetes.io/aws-load-balancer-listener-attributes.TCP-80": "tcp.idle_timeout.seconds=400", + }, + }, + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{ + { + Name: "test1", + Port: 80, + TargetPort: intstr.FromInt(8080), + Protocol: corev1.ProtocolTCP, + NodePort: 38888, + }, + { + Name: "test2", + Port: 80, + TargetPort: intstr.FromInt(8080), + Protocol: corev1.ProtocolUDP, + NodePort: 38888, + }, + }, + }, + }, + wantError: false, + wantValue: [][]elbv2model.ListenerAttribute{ + { + { + Key: tcpIdleTimeoutSeconds, + Value: "400", + }, + }, + {}, + }, + }, + } + for _, tt := range tests { + t.Run(tt.testName, func(t *testing.T) { + parser := annotations.NewSuffixAnnotationParser("service.beta.kubernetes.io") + builder := &defaultModelBuildTask{ + service: tt.svc, + annotationParser: parser, + } + + for index, port := range tt.svc.Spec.Ports { + listenerAttributes, err := builder.buildListenerAttributes(context.Background(), tt.svc.Annotations, port.Port, elbv2model.Protocol(port.Protocol)) + + if tt.wantError { + assert.Error(t, err) + } else { + assert.ElementsMatch(t, tt.wantValue[index], listenerAttributes) + } + } + + }) + } +} diff --git a/test/e2e/service/aws_resource_verifier.go b/test/e2e/service/aws_resource_verifier.go index 990f5953f3..19e52c261e 100644 --- a/test/e2e/service/aws_resource_verifier.go +++ b/test/e2e/service/aws_resource_verifier.go @@ -2,14 +2,15 @@ package service import ( "context" + "sort" + "strconv" + awssdk "github.com/aws/aws-sdk-go-v2/aws" elbv2types "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2/types" . "github.com/onsi/gomega" "github.com/pkg/errors" "sigs.k8s.io/aws-load-balancer-controller/test/framework" "sigs.k8s.io/aws-load-balancer-controller/test/framework/utils" - "sort" - "strconv" ) type TargetGroupHC struct { @@ -247,3 +248,26 @@ func verifyTargetGroupAttributes(ctx context.Context, f *framework.Framework, lb } return matchedAttrs == len(expectedAttributes) } + +func verifyListenerAttributes(ctx context.Context, f *framework.Framework, lsARN string, expectedAttrs map[string]string) error { + lsAttrs, err := f.LBManager.GetListenerAttributes(ctx, lsARN) + Expect(err).NotTo(HaveOccurred()) + for _, attr := range lsAttrs { + if val, ok := expectedAttrs[awssdk.ToString(attr.Key)]; ok && val != awssdk.ToString(attr.Value) { + return errors.Errorf("Attribute %v, expected %v, actual %v", awssdk.ToString(attr.Key), val, awssdk.ToString(attr.Value)) + } + } + return nil +} + +func getLoadBalancerListenerARN(ctx context.Context, f *framework.Framework, lbARN string, port string) string { + lsARN := "" + listeners, err := f.LBManager.GetLoadBalancerListeners(ctx, lbARN) + Expect(err).ToNot(HaveOccurred()) + for _, ls := range listeners { + if strconv.Itoa(int(awssdk.ToInt32(ls.Port))) == port { + lsARN = awssdk.ToString(ls.ListenerArn) + } + } + return lsARN +} diff --git a/test/e2e/service/nlb_instance_target_test.go b/test/e2e/service/nlb_instance_target_test.go index 29de0da564..dccb21d531 100644 --- a/test/e2e/service/nlb_instance_target_test.go +++ b/test/e2e/service/nlb_instance_target_test.go @@ -161,6 +161,20 @@ var _ = Describe("test k8s service reconciled by the aws load balancer controlle Expect(err).NotTo(HaveOccurred()) }) }) + By("modifying listener attributes", func() { + err := stack.UpdateServiceAnnotations(ctx, tf, map[string]string{ + "service.beta.kubernetes.io/aws-load-balancer-listener-attributes.TCP-80": "tcp.idle_timeout.seconds=400", + }) + Expect(err).NotTo(HaveOccurred()) + + lsARN := getLoadBalancerListenerARN(ctx, tf, lbARN, "80") + + Eventually(func() bool { + return verifyListenerAttributes(ctx, tf, lsARN, map[string]string{ + "tcp.idle_timeout.seconds": "400", + }) == nil + }, utils.PollTimeoutShort, utils.PollIntervalMedium).Should(BeTrue()) + }) It("should provision internal load-balancer resources", func() { By("deploying stack", func() { annotation["service.beta.kubernetes.io/aws-load-balancer-scheme"] = "internal" diff --git a/test/framework/resources/aws/load_balancer.go b/test/framework/resources/aws/load_balancer.go index e598332f38..6f5780e898 100644 --- a/test/framework/resources/aws/load_balancer.go +++ b/test/framework/resources/aws/load_balancer.go @@ -2,6 +2,7 @@ package aws import ( "context" + awssdk "github.com/aws/aws-sdk-go-v2/aws" elbv2sdk "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2" elbv2types "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2/types" @@ -20,6 +21,7 @@ type LoadBalancerManager interface { GetLoadBalancerAttributes(ctx context.Context, lbARN string) ([]elbv2types.LoadBalancerAttribute, error) GetLoadBalancerResourceTags(ctx context.Context, resARN string) ([]elbv2types.Tag, error) GetLoadBalancerListenerRules(ctx context.Context, lsARN string) ([]elbv2types.Rule, error) + GetListenerAttributes(ctx context.Context, lsARN string) ([]elbv2types.ListenerAttribute, error) } // NewDefaultLoadBalancerManager constructs new defaultLoadBalancerManager. @@ -118,3 +120,13 @@ func (m *defaultLoadBalancerManager) GetLoadBalancerListenerRules(ctx context.Co } return listenersRules.Rules, nil } + +func (m *defaultLoadBalancerManager) GetListenerAttributes(ctx context.Context, lsARN string) ([]elbv2types.ListenerAttribute, error) { + resp, err := m.elbv2Client.DescribeListenerAttributesWithContext(ctx, &elbv2sdk.DescribeListenerAttributesInput{ + ListenerArn: awssdk.String(lsARN), + }) + if err != nil { + return nil, err + } + return resp.Attributes, nil +}