From 21588bb619f49998b6868d7d1ba38a7b64cced74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A5rten=20Svantesson?= Date: Mon, 12 Feb 2024 19:53:42 +0100 Subject: [PATCH] feat: support externalname services --- pkg/backend/endpoint_resolver.go | 70 ++++-- pkg/backend/endpoint_resolver_test.go | 211 ++++++++++--------- pkg/backend/endpoint_types.go | 6 +- pkg/ingress/model_build_target_group.go | 8 +- pkg/targetgroupbinding/networking_manager.go | 10 +- pkg/targetgroupbinding/resource_manager.go | 74 ++++--- 6 files changed, 227 insertions(+), 152 deletions(-) diff --git a/pkg/backend/endpoint_resolver.go b/pkg/backend/endpoint_resolver.go index 054e7abbb8..d21e689642 100644 --- a/pkg/backend/endpoint_resolver.go +++ b/pkg/backend/endpoint_resolver.go @@ -3,6 +3,7 @@ package backend import ( "context" "fmt" + "net" awssdk "github.com/aws/aws-sdk-go/aws" "github.com/go-logr/logr" @@ -27,12 +28,17 @@ var ErrNotFound = errors.New("backend not found") type EndpointResolver interface { // ResolvePodEndpoints will resolve endpoints backed by pods directly. // returns resolved podEndpoints and whether there are unready endpoints that can potentially turn ready in future reconciles. - ResolvePodEndpoints(ctx context.Context, svcKey types.NamespacedName, port intstr.IntOrString, - opts ...EndpointResolveOption) ([]PodEndpoint, bool, error) + ResolvePodEndpoints(ctx context.Context, svckey types.NamespacedName, svc *corev1.Service, port intstr.IntOrString, opts ...EndpointResolveOption) ([]IpEndpoint, bool, error) // ResolveNodePortEndpoints will resolve endpoints backed by nodePort. ResolveNodePortEndpoints(ctx context.Context, svcKey types.NamespacedName, port intstr.IntOrString, opts ...EndpointResolveOption) ([]NodePortEndpoint, error) + + // FindService finds a k8s service + FindService(ctx context.Context, svcKey types.NamespacedName) (*corev1.Service, error) + + // ResolveExternalNameEndpoints will resolve external name using dns + ResolveExternalNameEndpoints(ctx context.Context, svc *corev1.Service, port intstr.IntOrString) ([]IpEndpoint, error) } // NewDefaultEndpointResolver constructs new defaultEndpointResolver @@ -42,6 +48,7 @@ func NewDefaultEndpointResolver(k8sClient client.Client, podInfoRepo k8s.PodInfo podInfoRepo: podInfoRepo, failOpenEnabled: failOpenEnabled, endpointSliceEnabled: endpointSliceEnabled, + dnsResolver: net.DefaultResolver, logger: logger, } } @@ -58,13 +65,34 @@ type defaultEndpointResolver struct { // [Pod Endpoint] whether to use endpointSlice instead of endpoints endpointSliceEnabled bool logger logr.Logger + // dnsResolver to use for resolving external names + dnsResolver dnsResolver +} + +type dnsResolver interface { + LookupHost(ctx context.Context, host string) (addrs []string, err error) +} + +func (r *defaultEndpointResolver) ResolveExternalNameEndpoints(ctx context.Context, svc *corev1.Service, port intstr.IntOrString) ([]IpEndpoint, error) { + if port.Type == intstr.String { + return nil, fmt.Errorf("port of target group must be numeric for external name") + } + addrs, err := r.dnsResolver.LookupHost(ctx, svc.Spec.ExternalName) + if err != nil { + return nil, err + } + endpoints := make([]IpEndpoint, len(addrs)) + for i, ip := range addrs { + endpoints[i] = IpEndpoint{IP: ip, Port: int64(port.IntVal)} + } + return endpoints, nil } -func (r *defaultEndpointResolver) ResolvePodEndpoints(ctx context.Context, svcKey types.NamespacedName, port intstr.IntOrString, opts ...EndpointResolveOption) ([]PodEndpoint, bool, error) { +func (r *defaultEndpointResolver) ResolvePodEndpoints(ctx context.Context, svcKey types.NamespacedName, svc *corev1.Service, port intstr.IntOrString, opts ...EndpointResolveOption) ([]IpEndpoint, bool, error) { resolveOpts := defaultEndpointResolveOptions() resolveOpts.ApplyOptions(opts) - _, svcPort, err := r.findServiceAndServicePort(ctx, svcKey, port) + _, svcPort, err := r.findServicePort(svc, port) if err != nil { return nil, false, err } @@ -140,9 +168,9 @@ func (r *defaultEndpointResolver) computeServiceEndpointsData(ctx context.Contex return endpointsDataList, nil } -func (r *defaultEndpointResolver) resolvePodEndpointsWithEndpointsData(ctx context.Context, svcKey types.NamespacedName, svcPort corev1.ServicePort, endpointsDataList []EndpointsData, podReadinessGates []corev1.PodConditionType) ([]PodEndpoint, bool, error) { - var readyPodEndpoints []PodEndpoint - var unknownPodEndpoints []PodEndpoint +func (r *defaultEndpointResolver) resolvePodEndpointsWithEndpointsData(ctx context.Context, svcKey types.NamespacedName, svcPort corev1.ServicePort, endpointsDataList []EndpointsData, podReadinessGates []corev1.PodConditionType) ([]IpEndpoint, bool, error) { + var readyPodEndpoints []IpEndpoint + var unknownPodEndpoints []IpEndpoint containsPotentialReadyEndpoints := false for _, epsData := range endpointsDataList { @@ -170,7 +198,7 @@ func (r *defaultEndpointResolver) resolvePodEndpointsWithEndpointsData(ctx conte containsPotentialReadyEndpoints = true continue } - podEndpoint := buildPodEndpoint(pod, epAddr, epPort) + podEndpoint := buildPodEndpoint(&pod, epAddr, epPort) if ep.Conditions.Ready != nil && *ep.Conditions.Ready { readyPodEndpoints = append(readyPodEndpoints, podEndpoint) continue @@ -214,13 +242,14 @@ func (r *defaultEndpointResolver) resolvePodEndpointsWithEndpointsData(ctx conte } func (r *defaultEndpointResolver) findServiceAndServicePort(ctx context.Context, svcKey types.NamespacedName, port intstr.IntOrString) (*corev1.Service, corev1.ServicePort, error) { - svc := &corev1.Service{} - if err := r.k8sClient.Get(ctx, svcKey, svc); err != nil { - if apierrors.IsNotFound(err) { - return nil, corev1.ServicePort{}, fmt.Errorf("%w: %v", ErrNotFound, err.Error()) - } + svc, err := r.FindService(ctx, svcKey) + if err != nil { return nil, corev1.ServicePort{}, err } + return r.findServicePort(svc, port) +} + +func (r *defaultEndpointResolver) findServicePort(svc *corev1.Service, port intstr.IntOrString) (*corev1.Service, corev1.ServicePort, error) { svcPort, err := k8s.LookupServicePort(svc, port) if err != nil { return nil, corev1.ServicePort{}, fmt.Errorf("%w: %v", ErrNotFound, err.Error()) @@ -229,6 +258,17 @@ func (r *defaultEndpointResolver) findServiceAndServicePort(ctx context.Context, return svc, svcPort, nil } +func (r *defaultEndpointResolver) FindService(ctx context.Context, svcKey types.NamespacedName) (*corev1.Service, error) { + svc := &corev1.Service{} + if err := r.k8sClient.Get(ctx, svcKey, svc); err != nil { + if apierrors.IsNotFound(err) { + return nil, fmt.Errorf("%w: %v", ErrNotFound, err.Error()) + } + return nil, err + } + return svc, nil +} + // filterNodesByReadyConditionStatus will filter out nodes that matches specified ready condition status func filterNodesByReadyConditionStatus(nodes []*corev1.Node, readyCondStatus corev1.ConditionStatus) []*corev1.Node { var nodesWithMatchingReadyStatus []*corev1.Node @@ -281,8 +321,8 @@ func buildEndpointsDataFromEndpointSliceList(epsList *discovery.EndpointSliceLis return endpointsDataList } -func buildPodEndpoint(pod k8s.PodInfo, epAddr string, port int32) PodEndpoint { - return PodEndpoint{ +func buildPodEndpoint(pod *k8s.PodInfo, epAddr string, port int32) IpEndpoint { + return IpEndpoint{ IP: epAddr, Port: int64(port), Pod: pod, diff --git a/pkg/backend/endpoint_resolver_test.go b/pkg/backend/endpoint_resolver_test.go index 27a01e7cce..3785521344 100644 --- a/pkg/backend/endpoint_resolver_test.go +++ b/pkg/backend/endpoint_resolver_test.go @@ -689,7 +689,6 @@ func Test_defaultEndpointResolver_ResolvePodEndpoints(t *testing.T) { } type env struct { nodes []*corev1.Node - services []*corev1.Service endpointsList []*corev1.Endpoints endpointSlices []*discovery.EndpointSlice } @@ -699,16 +698,16 @@ func Test_defaultEndpointResolver_ResolvePodEndpoints(t *testing.T) { endpointSliceEnabled bool } type args struct { - svcKey types.NamespacedName - port intstr.IntOrString - opts []EndpointResolveOption + svc *corev1.Service + port intstr.IntOrString + opts []EndpointResolveOption } tests := []struct { name string env env fields fields args args - want []PodEndpoint + want []IpEndpoint wantContainsPotentialReadyEndpoints bool wantErr error }{ @@ -716,7 +715,6 @@ func Test_defaultEndpointResolver_ResolvePodEndpoints(t *testing.T) { name: "[with endpoints][with failOpen] choose every ready pod only when there are ready pods", env: env{ nodes: []*corev1.Node{nodeA, nodeB, nodeC}, - services: []*corev1.Service{svc1}, endpointsList: []*corev1.Endpoints{ep1}, }, fields: fields{ @@ -761,20 +759,20 @@ func Test_defaultEndpointResolver_ResolvePodEndpoints(t *testing.T) { }, }, args: args{ - svcKey: k8s.NamespacedName(svc1), - port: intstr.FromString("http"), - opts: nil, + svc: svc1, + port: intstr.FromString("http"), + opts: nil, }, - want: []PodEndpoint{ + want: []IpEndpoint{ { IP: "192.168.1.1", Port: 8080, - Pod: pod1, + Pod: &pod1, }, { IP: "192.168.1.4", Port: 8080, - Pod: pod4, + Pod: &pod4, }, }, wantContainsPotentialReadyEndpoints: false, @@ -783,7 +781,6 @@ func Test_defaultEndpointResolver_ResolvePodEndpoints(t *testing.T) { name: "[with endpointSlices][with failOpen] choose every ready pod only when there are ready pods", env: env{ nodes: []*corev1.Node{nodeA, nodeB, nodeC}, - services: []*corev1.Service{svc1}, endpointSlices: []*discovery.EndpointSlice{eps1}, }, fields: fields{ @@ -833,20 +830,20 @@ func Test_defaultEndpointResolver_ResolvePodEndpoints(t *testing.T) { }, }, args: args{ - svcKey: k8s.NamespacedName(svc1), - port: intstr.FromString("http"), - opts: nil, + svc: svc1, + port: intstr.FromString("http"), + opts: nil, }, - want: []PodEndpoint{ + want: []IpEndpoint{ { IP: "192.168.1.1", Port: 8080, - Pod: pod1, + Pod: &pod1, }, { IP: "192.168.1.4", Port: 8080, - Pod: pod4, + Pod: &pod4, }, }, wantContainsPotentialReadyEndpoints: false, @@ -855,7 +852,6 @@ func Test_defaultEndpointResolver_ResolvePodEndpoints(t *testing.T) { name: "[with endpointSlices][without failOpen] choose every ready pod only when there are ready pods", env: env{ nodes: []*corev1.Node{nodeA, nodeB, nodeC}, - services: []*corev1.Service{svc1}, endpointSlices: []*discovery.EndpointSlice{eps1}, }, fields: fields{ @@ -905,20 +901,20 @@ func Test_defaultEndpointResolver_ResolvePodEndpoints(t *testing.T) { }, }, args: args{ - svcKey: k8s.NamespacedName(svc1), - port: intstr.FromString("http"), - opts: nil, + svc: svc1, + port: intstr.FromString("http"), + opts: nil, }, - want: []PodEndpoint{ + want: []IpEndpoint{ { IP: "192.168.1.1", Port: 8080, - Pod: pod1, + Pod: &pod1, }, { IP: "192.168.1.4", Port: 8080, - Pod: pod4, + Pod: &pod4, }, }, wantContainsPotentialReadyEndpoints: false, @@ -927,7 +923,6 @@ func Test_defaultEndpointResolver_ResolvePodEndpoints(t *testing.T) { name: "[with endpointSlices][with failOpen] choose every unknown pod when there are no ready pods", env: env{ nodes: []*corev1.Node{nodeA, nodeB, nodeC}, - services: []*corev1.Service{svc1}, endpointSlices: []*discovery.EndpointSlice{eps2}, }, fields: fields{ @@ -967,25 +962,25 @@ func Test_defaultEndpointResolver_ResolvePodEndpoints(t *testing.T) { }, }, args: args{ - svcKey: k8s.NamespacedName(svc1), - port: intstr.FromString("http"), - opts: nil, + svc: svc1, + port: intstr.FromString("http"), + opts: nil, }, - want: []PodEndpoint{ + want: []IpEndpoint{ { IP: "192.168.1.2", Port: 8080, - Pod: pod2, + Pod: &pod2, }, { IP: "192.168.1.5", Port: 8080, - Pod: pod5, + Pod: &pod5, }, { IP: "192.168.1.8", Port: 8080, - Pod: pod8, + Pod: &pod8, }, }, wantContainsPotentialReadyEndpoints: false, @@ -994,7 +989,6 @@ func Test_defaultEndpointResolver_ResolvePodEndpoints(t *testing.T) { name: "[with endpointSlices][without failOpen] don't choose unknown pod when there are no ready pods", env: env{ nodes: []*corev1.Node{nodeA, nodeB, nodeC}, - services: []*corev1.Service{svc1}, endpointSlices: []*discovery.EndpointSlice{eps2}, }, fields: fields{ @@ -1034,9 +1028,9 @@ func Test_defaultEndpointResolver_ResolvePodEndpoints(t *testing.T) { }, }, args: args{ - svcKey: k8s.NamespacedName(svc1), - port: intstr.FromString("http"), - opts: nil, + svc: svc1, + port: intstr.FromString("http"), + opts: nil, }, want: nil, wantContainsPotentialReadyEndpoints: false, @@ -1045,7 +1039,6 @@ func Test_defaultEndpointResolver_ResolvePodEndpoints(t *testing.T) { name: "[with endpointSlices][with failOpen] choose every ready pod only when there are ready pods - some pod have readinessGate", env: env{ nodes: []*corev1.Node{nodeA, nodeB, nodeC}, - services: []*corev1.Service{svc1}, endpointSlices: []*discovery.EndpointSlice{eps1}, }, fields: fields{ @@ -1095,20 +1088,20 @@ func Test_defaultEndpointResolver_ResolvePodEndpoints(t *testing.T) { }, }, args: args{ - svcKey: k8s.NamespacedName(svc1), - port: intstr.FromString("http"), - opts: []EndpointResolveOption{WithPodReadinessGate("custom-condition")}, + svc: svc1, + port: intstr.FromString("http"), + opts: []EndpointResolveOption{WithPodReadinessGate("custom-condition")}, }, - want: []PodEndpoint{ + want: []IpEndpoint{ { IP: "192.168.1.1", Port: 8080, - Pod: pod1, + Pod: &pod1, }, { IP: "192.168.1.4", Port: 8080, - Pod: pod4, + Pod: &pod4, }, }, wantContainsPotentialReadyEndpoints: true, @@ -1117,7 +1110,6 @@ func Test_defaultEndpointResolver_ResolvePodEndpoints(t *testing.T) { name: "[with endpointSlices][with failOpen] choose every ready pod only when there are ready pods - no pod have readinessGate", env: env{ nodes: []*corev1.Node{nodeA, nodeB, nodeC}, - services: []*corev1.Service{svc1}, endpointSlices: []*discovery.EndpointSlice{eps3}, }, fields: fields{ @@ -1162,20 +1154,20 @@ func Test_defaultEndpointResolver_ResolvePodEndpoints(t *testing.T) { }, }, args: args{ - svcKey: k8s.NamespacedName(svc1), - port: intstr.FromString("http"), - opts: []EndpointResolveOption{WithPodReadinessGate("custom-condition")}, + svc: svc1, + port: intstr.FromString("http"), + opts: []EndpointResolveOption{WithPodReadinessGate("custom-condition")}, }, - want: []PodEndpoint{ + want: []IpEndpoint{ { IP: "192.168.1.1", Port: 8080, - Pod: pod1, + Pod: &pod1, }, { IP: "192.168.1.4", Port: 8080, - Pod: pod4, + Pod: &pod4, }, }, wantContainsPotentialReadyEndpoints: false, @@ -1184,7 +1176,6 @@ func Test_defaultEndpointResolver_ResolvePodEndpoints(t *testing.T) { name: "[with endpoints][with failOpen] choose every ready pod only when there are ready pods - ignore pods don't exists", env: env{ nodes: []*corev1.Node{nodeA, nodeB, nodeC}, - services: []*corev1.Service{svc1}, endpointsList: []*corev1.Endpoints{ep1}, }, fields: fields{ @@ -1229,66 +1220,46 @@ func Test_defaultEndpointResolver_ResolvePodEndpoints(t *testing.T) { }, }, args: args{ - svcKey: k8s.NamespacedName(svc1), - port: intstr.FromString("http"), - opts: nil, + svc: svc1, + port: intstr.FromString("http"), + opts: nil, }, - want: []PodEndpoint{ + want: []IpEndpoint{ { IP: "192.168.1.4", Port: 8080, - Pod: pod4, + Pod: &pod4, }, }, wantContainsPotentialReadyEndpoints: true, }, - { - name: "service not found", - env: env{ - services: []*corev1.Service{}, - endpointsList: []*corev1.Endpoints{}, - }, - fields: fields{ - podInfoRepoGetCalls: []podInfoRepoGetCall{}, - }, - args: args{ - svcKey: k8s.NamespacedName(svc1), - port: intstr.FromString("http"), - opts: nil, - }, - want: []PodEndpoint{}, - wantContainsPotentialReadyEndpoints: false, - wantErr: fmt.Errorf("%w: %v", ErrNotFound, "services \"svc-1\" not found"), - }, { name: "service port not found", env: env{ - services: []*corev1.Service{svc1WithoutHTTPPort}, endpointsList: []*corev1.Endpoints{}, }, fields: fields{ podInfoRepoGetCalls: []podInfoRepoGetCall{}, }, args: args{ - svcKey: k8s.NamespacedName(svc1), - port: intstr.FromString("http"), - opts: nil, + svc: svc1WithoutHTTPPort, + port: intstr.FromString("http"), + opts: nil, }, wantErr: fmt.Errorf("%w: %v", ErrNotFound, "unable to find port http on service test-ns/svc-1"), }, { name: "endpoints not found", env: env{ - services: []*corev1.Service{svc1}, endpointsList: []*corev1.Endpoints{}, }, fields: fields{ podInfoRepoGetCalls: []podInfoRepoGetCall{}, }, args: args{ - svcKey: k8s.NamespacedName(svc1), - port: intstr.FromString("http"), - opts: nil, + svc: svc1, + port: intstr.FromString("http"), + opts: nil, }, wantErr: fmt.Errorf("%w: %v", ErrNotFound, "endpoints \"svc-1\" not found"), }, @@ -1311,9 +1282,6 @@ func Test_defaultEndpointResolver_ResolvePodEndpoints(t *testing.T) { for _, node := range tt.env.nodes { assert.NoError(t, k8sClient.Create(ctx, node.DeepCopy())) } - for _, svc := range tt.env.services { - assert.NoError(t, k8sClient.Create(ctx, svc.DeepCopy())) - } for _, endpoints := range tt.env.endpointsList { assert.NoError(t, k8sClient.Create(ctx, endpoints.DeepCopy())) } @@ -1328,14 +1296,14 @@ func Test_defaultEndpointResolver_ResolvePodEndpoints(t *testing.T) { endpointSliceEnabled: tt.fields.endpointSliceEnabled, logger: logr.New(&log.NullLogSink{}), } - got, gotContainsPotentialReadyEndpoints, err := r.ResolvePodEndpoints(ctx, tt.args.svcKey, tt.args.port, tt.args.opts...) + got, gotContainsPotentialReadyEndpoints, err := r.ResolvePodEndpoints(ctx, k8s.NamespacedName(tt.args.svc), tt.args.svc, tt.args.port, tt.args.opts...) if tt.wantErr != nil { assert.EqualError(t, err, tt.wantErr.Error()) } else { assert.NoError(t, err) opt := cmp.Options{ equality.IgnoreFakeClientPopulatedFields(), - cmpopts.SortSlices(func(lhs PodEndpoint, rhs PodEndpoint) bool { + cmpopts.SortSlices(func(lhs IpEndpoint, rhs IpEndpoint) bool { return lhs.IP < rhs.IP }), } @@ -1347,6 +1315,61 @@ func Test_defaultEndpointResolver_ResolvePodEndpoints(t *testing.T) { } } +type mockDns map[string][]string + +func (m mockDns) LookupHost(_ context.Context, host string) (addrs []string, err error) { + var ok bool + addrs, ok = m[host] + if !ok { + err = fmt.Errorf("No ips are registered for %s", host) + } + return +} + +func Test_defaultEndpointResolver_ResolveExternalNameEndpoints(t *testing.T) { + svcExternalName := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "testNS", + Name: "svc-1", + }, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeExternalName, + ExternalName: "service.in.my.vpc", + }, + } + ctx := context.Background() + + want := []IpEndpoint{ + { + IP: "10.10.10.10", + Port: 80, + }, + { + IP: "10.10.11.10", + Port: 80, + }, + } + opt := cmp.Options{ + equality.IgnoreFakeClientPopulatedFields(), + cmpopts.SortSlices(func(lhs IpEndpoint, rhs IpEndpoint) bool { + return lhs.IP < rhs.IP + }), + } + + r := &defaultEndpointResolver{ + logger: logr.New(&log.NullLogSink{}), + dnsResolver: mockDns{"service.in.my.vpc": []string{"10.10.10.10", "10.10.11.10"}}, + } + got, err := r.ResolveExternalNameEndpoints(ctx, svcExternalName, intstr.FromInt(80)) + assert.NoError(t, err) + assert.True(t, cmp.Equal(want, got, opt), + "diff: %v", cmp.Diff(want, got, opt)) + + r.dnsResolver = mockDns{} + got, err = r.ResolveExternalNameEndpoints(ctx, svcExternalName, intstr.FromInt(80)) + assert.EqualError(t, err, fmt.Errorf("No ips are registered for %s", svcExternalName.Spec.ExternalName).Error()) +} + func Test_defaultEndpointResolver_ResolveNodePortEndpoints(t *testing.T) { testNS := "test-ns" node1 := &corev1.Node{ @@ -2079,7 +2102,7 @@ func Test_defaultEndpointResolver_findServiceAndServicePort(t *testing.T) { } else { opt := cmp.Options{ equality.IgnoreFakeClientPopulatedFields(), - cmpopts.SortSlices(func(lhs PodEndpoint, rhs PodEndpoint) bool { + cmpopts.SortSlices(func(lhs IpEndpoint, rhs IpEndpoint) bool { return lhs.IP < rhs.IP }), } @@ -2596,7 +2619,7 @@ func Test_buildPodEndpoint(t *testing.T) { tests := []struct { name string args args - want PodEndpoint + want IpEndpoint }{ { name: "base case", @@ -2607,10 +2630,10 @@ func Test_buildPodEndpoint(t *testing.T) { epAddr: "192.168.1.1", port: 80, }, - want: PodEndpoint{ + want: IpEndpoint{ IP: "192.168.1.1", Port: 80, - Pod: k8s.PodInfo{ + Pod: &k8s.PodInfo{ Key: types.NamespacedName{Name: "sample-node"}, }, }, @@ -2618,7 +2641,7 @@ func Test_buildPodEndpoint(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got := buildPodEndpoint(tt.args.pod, tt.args.epAddr, tt.args.port) + got := buildPodEndpoint(&tt.args.pod, tt.args.epAddr, tt.args.port) assert.Equal(t, tt.want, got) }) } diff --git a/pkg/backend/endpoint_types.go b/pkg/backend/endpoint_types.go index 76d017bd5f..88844b6bed 100644 --- a/pkg/backend/endpoint_types.go +++ b/pkg/backend/endpoint_types.go @@ -7,14 +7,14 @@ import ( "sigs.k8s.io/aws-load-balancer-controller/pkg/k8s" ) -// An endpoint provided by pod directly. -type PodEndpoint struct { +// IpEndpoint is an endpoint for an ip address +type IpEndpoint struct { // Pod's IP. IP string // Pod's container port. Port int64 // Pod that provides this endpoint. - Pod k8s.PodInfo + Pod *k8s.PodInfo } // An endpoint provided by nodePort as traffic proxy. diff --git a/pkg/ingress/model_build_target_group.go b/pkg/ingress/model_build_target_group.go index b855efb291..591a27c7bd 100644 --- a/pkg/ingress/model_build_target_group.go +++ b/pkg/ingress/model_build_target_group.go @@ -140,7 +140,7 @@ func (t *defaultModelBuildTask) buildTargetGroupBindingNetworking(ctx context.Co func (t *defaultModelBuildTask) buildTargetGroupSpec(ctx context.Context, ing ClassifiedIngress, svc *corev1.Service, port intstr.IntOrString, svcPort corev1.ServicePort) (elbv2model.TargetGroupSpec, error) { svcAndIngAnnotations := algorithm.MergeStringMap(svc.Annotations, ing.Ing.Annotations) - targetType, err := t.buildTargetGroupTargetType(ctx, svcAndIngAnnotations) + targetType, err := t.buildTargetGroupTargetType(ctx, svcAndIngAnnotations, svc) if err != nil { return elbv2model.TargetGroupSpec{}, err } @@ -207,9 +207,13 @@ func (t *defaultModelBuildTask) buildTargetGroupName(_ context.Context, return fmt.Sprintf("k8s-%.8s-%.8s-%.10s", sanitizedNamespace, sanitizedName, uuid) } -func (t *defaultModelBuildTask) buildTargetGroupTargetType(_ context.Context, svcAndIngAnnotations map[string]string) (elbv2model.TargetType, error) { +func (t *defaultModelBuildTask) buildTargetGroupTargetType(_ context.Context, svcAndIngAnnotations map[string]string, svc *corev1.Service) (elbv2model.TargetType, error) { rawTargetType := string(t.defaultTargetType) _ = t.annotationParser.ParseStringAnnotation(annotations.IngressSuffixTargetType, &rawTargetType, svcAndIngAnnotations) + if svc.Spec.Type == corev1.ServiceTypeExternalName && rawTargetType != string(elbv2model.TargetTypeIP) { + t.logger.Info("Target type will be ip for since service is an ExternalName", "service", k8s.NamespacedName(svc)) + rawTargetType = string(elbv2model.TargetTypeIP) + } switch rawTargetType { case string(elbv2model.TargetTypeInstance): return elbv2model.TargetTypeInstance, nil diff --git a/pkg/targetgroupbinding/networking_manager.go b/pkg/targetgroupbinding/networking_manager.go index f8199026de..8287a9c8d9 100644 --- a/pkg/targetgroupbinding/networking_manager.go +++ b/pkg/targetgroupbinding/networking_manager.go @@ -35,7 +35,7 @@ const ( // NetworkingManager manages the networking for targetGroupBindings. type NetworkingManager interface { // ReconcileForPodEndpoints reconcile network settings for TargetGroupBindings with podEndpoints. - ReconcileForPodEndpoints(ctx context.Context, tgb *elbv2api.TargetGroupBinding, endpoints []backend.PodEndpoint) error + ReconcileForPodEndpoints(ctx context.Context, tgb *elbv2api.TargetGroupBinding, endpoints []backend.IpEndpoint) error // ReconcileForNodePortEndpoints reconcile network settings for TargetGroupBindings with nodePortEndpoints. ReconcileForNodePortEndpoints(ctx context.Context, tgb *elbv2api.TargetGroupBinding, endpoints []backend.NodePortEndpoint) error @@ -95,7 +95,7 @@ type defaultNetworkingManager struct { disableRestrictedSGRules bool } -func (m *defaultNetworkingManager) ReconcileForPodEndpoints(ctx context.Context, tgb *elbv2api.TargetGroupBinding, endpoints []backend.PodEndpoint) error { +func (m *defaultNetworkingManager) ReconcileForPodEndpoints(ctx context.Context, tgb *elbv2api.TargetGroupBinding, endpoints []backend.IpEndpoint) error { var ingressPermissionsPerSG map[string][]networking.IPPermissionInfo if tgb.Spec.Networking != nil { var err error @@ -123,12 +123,12 @@ func (m *defaultNetworkingManager) Cleanup(ctx context.Context, tgb *elbv2api.Ta return m.reconcileWithIngressPermissionsPerSG(ctx, tgb, nil) } -func (m *defaultNetworkingManager) computeIngressPermissionsPerSGWithPodEndpoints(ctx context.Context, tgbNetworking elbv2api.TargetGroupBindingNetworking, endpoints []backend.PodEndpoint) (map[string][]networking.IPPermissionInfo, error) { +func (m *defaultNetworkingManager) computeIngressPermissionsPerSGWithPodEndpoints(ctx context.Context, tgbNetworking elbv2api.TargetGroupBindingNetworking, endpoints []backend.IpEndpoint) (map[string][]networking.IPPermissionInfo, error) { pods := make([]k8s.PodInfo, 0, len(endpoints)) podByPodKey := make(map[types.NamespacedName]k8s.PodInfo, len(endpoints)) for _, endpoint := range endpoints { - pods = append(pods, endpoint.Pod) - podByPodKey[endpoint.Pod.Key] = endpoint.Pod + pods = append(pods, *endpoint.Pod) + podByPodKey[endpoint.Pod.Key] = *endpoint.Pod } eniInfoByPodKey, err := m.podENIResolver.Resolve(ctx, pods) if err != nil { diff --git a/pkg/targetgroupbinding/resource_manager.go b/pkg/targetgroupbinding/resource_manager.go index 8b439bda09..e55acdacc8 100644 --- a/pkg/targetgroupbinding/resource_manager.go +++ b/pkg/targetgroupbinding/resource_manager.go @@ -112,20 +112,26 @@ func (m *defaultResourceManager) reconcileWithIPTargetType(ctx context.Context, backend.WithPodReadinessGate(targetHealthCondType), } - var endpoints []backend.PodEndpoint + var endpoints []backend.IpEndpoint var containsPotentialReadyEndpoints bool - var err error + svc, err := m.endpointResolver.FindService(ctx, svcKey) + if svc.Spec.Type == corev1.ServiceTypeExternalName { + endpoints, err = m.endpointResolver.ResolveExternalNameEndpoints(ctx, svc, tgb.Spec.ServiceRef.Port) + if err != nil { + return err + } + } else { - endpoints, containsPotentialReadyEndpoints, err = m.endpointResolver.ResolvePodEndpoints(ctx, svcKey, tgb.Spec.ServiceRef.Port, resolveOpts...) + endpoints, containsPotentialReadyEndpoints, err = m.endpointResolver.ResolvePodEndpoints(ctx, svcKey, svc, tgb.Spec.ServiceRef.Port, resolveOpts...) - if err != nil { - if errors.Is(err, backend.ErrNotFound) { - m.eventRecorder.Event(tgb, corev1.EventTypeWarning, k8s.TargetGroupBindingEventReasonBackendNotFound, err.Error()) - return m.Cleanup(ctx, tgb) + if err != nil { + if errors.Is(err, backend.ErrNotFound) { + m.eventRecorder.Event(tgb, corev1.EventTypeWarning, k8s.TargetGroupBindingEventReasonBackendNotFound, err.Error()) + return m.Cleanup(ctx, tgb) + } + return err } - return err } - tgARN := tgb.Spec.TargetGroupARN targets, err := m.targetsManager.ListTargets(ctx, tgARN) if err != nil { @@ -135,9 +141,11 @@ func (m *defaultResourceManager) reconcileWithIPTargetType(ctx context.Context, matchedEndpointAndTargets, unmatchedEndpoints, unmatchedTargets := matchPodEndpointWithTargets(endpoints, notDrainingTargets) needNetworkingRequeue := false - if err := m.networkingManager.ReconcileForPodEndpoints(ctx, tgb, endpoints); err != nil { - m.eventRecorder.Event(tgb, corev1.EventTypeWarning, k8s.TargetGroupBindingEventReasonFailedNetworkReconcile, err.Error()) - needNetworkingRequeue = true + if svc.Spec.Type != corev1.ServiceTypeExternalName { + if err := m.networkingManager.ReconcileForPodEndpoints(ctx, tgb, endpoints); err != nil { + m.eventRecorder.Event(tgb, corev1.EventTypeWarning, k8s.TargetGroupBindingEventReasonFailedNetworkReconcile, err.Error()) + needNetworkingRequeue = true + } } if len(unmatchedTargets) > 0 { if err := m.deregisterTargets(ctx, tgARN, unmatchedTargets); err != nil { @@ -149,23 +157,23 @@ func (m *defaultResourceManager) reconcileWithIPTargetType(ctx context.Context, return err } } + if svc.Spec.Type != corev1.ServiceTypeExternalName { + anyPodNeedFurtherProbe, err := m.updateTargetHealthPodCondition(ctx, targetHealthCondType, matchedEndpointAndTargets, unmatchedEndpoints) + if err != nil { + return err + } - anyPodNeedFurtherProbe, err := m.updateTargetHealthPodCondition(ctx, targetHealthCondType, matchedEndpointAndTargets, unmatchedEndpoints) - if err != nil { - return err - } - - if anyPodNeedFurtherProbe { - if containsTargetsInInitialState(matchedEndpointAndTargets) || len(unmatchedEndpoints) != 0 { - return runtime.NewRequeueNeededAfter("monitor targetHealth", m.targetHealthRequeueDuration) + if anyPodNeedFurtherProbe { + if containsTargetsInInitialState(matchedEndpointAndTargets) || len(unmatchedEndpoints) != 0 { + return runtime.NewRequeueNeededAfter("monitor targetHealth", m.targetHealthRequeueDuration) + } + return runtime.NewRequeueNeeded("monitor targetHealth") } - return runtime.NewRequeueNeeded("monitor targetHealth") - } - if containsPotentialReadyEndpoints { - return runtime.NewRequeueNeeded("monitor potential ready endpoints") + if containsPotentialReadyEndpoints { + return runtime.NewRequeueNeeded("monitor potential ready endpoints") + } } - _ = drainingTargets if needNetworkingRequeue { @@ -239,13 +247,13 @@ func (m *defaultResourceManager) cleanupTargets(ctx context.Context, tgb *elbv2a // updateTargetHealthPodCondition will updates pod's targetHealth condition for matchedEndpointAndTargets and unmatchedEndpoints. // returns whether further probe is needed or not func (m *defaultResourceManager) updateTargetHealthPodCondition(ctx context.Context, targetHealthCondType corev1.PodConditionType, - matchedEndpointAndTargets []podEndpointAndTargetPair, unmatchedEndpoints []backend.PodEndpoint) (bool, error) { + matchedEndpointAndTargets []podEndpointAndTargetPair, unmatchedEndpoints []backend.IpEndpoint) (bool, error) { anyPodNeedFurtherProbe := false for _, endpointAndTarget := range matchedEndpointAndTargets { pod := endpointAndTarget.endpoint.Pod targetHealth := endpointAndTarget.target.TargetHealth - needFurtherProbe, err := m.updateTargetHealthPodConditionForPod(ctx, pod, targetHealth, targetHealthCondType) + needFurtherProbe, err := m.updateTargetHealthPodConditionForPod(ctx, *pod, targetHealth, targetHealthCondType) if err != nil { return false, err } @@ -261,7 +269,7 @@ func (m *defaultResourceManager) updateTargetHealthPodCondition(ctx context.Cont Reason: awssdk.String(elbv2sdk.TargetHealthReasonEnumElbRegistrationInProgress), Description: awssdk.String("Target registration is in progress"), } - needFurtherProbe, err := m.updateTargetHealthPodConditionForPod(ctx, pod, targetHealth, targetHealthCondType) + needFurtherProbe, err := m.updateTargetHealthPodConditionForPod(ctx, *pod, targetHealth, targetHealthCondType) if err != nil { return false, err } @@ -382,7 +390,7 @@ func (m *defaultResourceManager) deregisterTargets(ctx context.Context, tgARN st return m.targetsManager.DeregisterTargets(ctx, tgARN, sdkTargets) } -func (m *defaultResourceManager) registerPodEndpoints(ctx context.Context, tgARN string, endpoints []backend.PodEndpoint) error { +func (m *defaultResourceManager) registerPodEndpoints(ctx context.Context, tgARN string, endpoints []backend.IpEndpoint) error { vpcInfo, err := m.vpcInfoProvider.FetchVPCInfo(ctx, m.vpcID) if err != nil { return err @@ -425,7 +433,7 @@ func (m *defaultResourceManager) registerNodePortEndpoints(ctx context.Context, } type podEndpointAndTargetPair struct { - endpoint backend.PodEndpoint + endpoint backend.IpEndpoint target TargetInfo } @@ -451,12 +459,12 @@ func containsTargetsInInitialState(matchedEndpointAndTargets []podEndpointAndTar return false } -func matchPodEndpointWithTargets(endpoints []backend.PodEndpoint, targets []TargetInfo) ([]podEndpointAndTargetPair, []backend.PodEndpoint, []TargetInfo) { +func matchPodEndpointWithTargets(endpoints []backend.IpEndpoint, targets []TargetInfo) ([]podEndpointAndTargetPair, []backend.IpEndpoint, []TargetInfo) { var matchedEndpointAndTargets []podEndpointAndTargetPair - var unmatchedEndpoints []backend.PodEndpoint + var unmatchedEndpoints []backend.IpEndpoint var unmatchedTargets []TargetInfo - endpointsByUID := make(map[string]backend.PodEndpoint, len(endpoints)) + endpointsByUID := make(map[string]backend.IpEndpoint, len(endpoints)) for _, endpoint := range endpoints { endpointUID := fmt.Sprintf("%v:%v", endpoint.IP, endpoint.Port) endpointsByUID[endpointUID] = endpoint