From 6ba997d20b32555e02906a635ef00ce980cbb52b Mon Sep 17 00:00:00 2001 From: bobz965 Date: Fri, 4 Aug 2023 12:17:58 +0800 Subject: [PATCH] =?UTF-8?q?=E7=AE=80=E5=8C=96=20ovn=20eip=20=E7=B1=BB?= =?UTF-8?q?=E5=9E=8B=20(#3107)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 简化 ovn eip 类型 * support nat reuse lrp ip * fix const value * fix ecmp bfd static route * fix cleanup --- dist/images/cleanup.sh | 20 ++++++-- pkg/apis/kubeovn/v1/types.go | 8 ++-- pkg/controller/external-gw.go | 2 +- pkg/controller/ovn_dnat.go | 24 ++++------ pkg/controller/ovn_eip.go | 14 +++--- pkg/controller/ovn_fip.go | 18 ++++++-- pkg/controller/ovn_snat.go | 12 +++-- pkg/controller/subnet.go | 32 ++++++------- pkg/controller/vpc.go | 23 ++++------ pkg/daemon/ovs_linux.go | 4 +- pkg/ovs/ovn-nb-logical_router_route.go | 13 ++++-- pkg/util/const.go | 21 +++++---- test/e2e/ovn-vpc-nat-gw/e2e_test.go | 63 ++++++++++++-------------- 13 files changed, 132 insertions(+), 122 deletions(-) diff --git a/dist/images/cleanup.sh b/dist/images/cleanup.sh index d5b02586d91..c7a2a149900 100644 --- a/dist/images/cleanup.sh +++ b/dist/images/cleanup.sh @@ -18,6 +18,10 @@ for vd in $(kubectl get vpc-dns -o name); do kubectl delete --ignore-not-found $vd done +for ip in $(kubectl get ip -o name); do + kubectl delete --ignore-not-found $ip +done + for vip in $(kubectl get vip -o name); do kubectl delete --ignore-not-found $vip done @@ -38,6 +42,10 @@ for eip in $(kubectl get eip -o name); do kubectl delete --ignore-not-found $eip done +for odnat in $(kubectl get odnat -o name); do + kubectl delete --ignore-not-found $odnat +done + for osnat in $(kubectl get osnat -o name); do kubectl delete --ignore-not-found $osnat done @@ -63,6 +71,8 @@ for subnet in $(kubectl get subnet -o name); do kubectl patch "$subnet" --type='json' -p '[{"op": "replace", "path": "/metadata/finalizers", "value": []}]' kubectl delete --ignore-not-found "$subnet" done +# subnet join will recreate, so delete subnet crd right now +kubectl delete --ignore-not-found crd subnets.kubeovn.io set -e for vpc in $(kubectl get vpc -o name); do @@ -120,21 +130,21 @@ kubectl delete --ignore-not-found crd \ security-groups.kubeovn.io \ ips.kubeovn.io \ ippools.kubeovn.io \ - subnets.kubeovn.io \ vpc-nat-gateways.kubeovn.io \ vpcs.kubeovn.io \ vlans.kubeovn.io \ provider-networks.kubeovn.io \ iptables-dnat-rules.kubeovn.io \ - iptables-eips.kubeovn.io \ - iptables-fip-rules.kubeovn.io \ iptables-snat-rules.kubeovn.io \ + iptables-fip-rules.kubeovn.io \ + iptables-eips.kubeovn.io \ vips.kubeovn.io \ switch-lb-rules.kubeovn.io \ vpc-dnses.kubeovn.io \ - ovn-eips.kubeovn.io ovn-fips.kubeovn.io \ - ovn-snat-rules.kubeovn.io \ ovn-dnat-rules.kubeovn.io \ + ovn-snat-rules.kubeovn.io \ + ovn-fips.kubeovn.io \ + ovn-eips.kubeovn.io \ qos-policies.kubeovn.io # Remove annotations/labels in namespaces and nodes diff --git a/pkg/apis/kubeovn/v1/types.go b/pkg/apis/kubeovn/v1/types.go index 151f41f2255..bd4fb4150e5 100644 --- a/pkg/apis/kubeovn/v1/types.go +++ b/pkg/apis/kubeovn/v1/types.go @@ -949,10 +949,10 @@ type OvnEipSpec struct { V6Ip string `json:"v6Ip"` MacAddress string `json:"macAddress"` Type string `json:"type"` - // usage type: eip, lrp, node external gw - // eip: only used by nat, fip, snat, dnat, all the nat type will record int the eip status - // lrp: logical router port - // node external gw: is lsp, in the case of bfd session between lrp and lsp, the lsp is on the node as external gateway + // usage type: lrp, lsp, nat + // nat: used by nat: dnat, snat, fip + // lrp: lrp created by vpc enable external, and also could be used by nat + // lsp: in the case of bfd session between lrp and lsp, the lsp is on the node as ecmp nexthop } // OvnEipCondition describes the state of an object at a certain point. diff --git a/pkg/controller/external-gw.go b/pkg/controller/external-gw.go index 031e3f47a0a..5f5f66bcc24 100644 --- a/pkg/controller/external-gw.go +++ b/pkg/controller/external-gw.go @@ -200,7 +200,7 @@ func (c *Controller) createDefaultVpcLrpEip(config map[string]string) (string, s klog.Errorf("failed to acquire ip address for default vpc lrp %s, %v", lrpEipName, err) return "", "", err } - if err := c.createOrUpdateCrdOvnEip(lrpEipName, c.config.ExternalGatewaySwitch, v4ip, v6ip, mac, util.LrpUsingEip); err != nil { + if err := c.createOrUpdateCrdOvnEip(lrpEipName, c.config.ExternalGatewaySwitch, v4ip, v6ip, mac, util.Lrp); err != nil { klog.Errorf("failed to create ovn eip cr for lrp %s, %v", lrpEipName, err) return "", "", err } diff --git a/pkg/controller/ovn_dnat.go b/pkg/controller/ovn_dnat.go index a2e05789fe1..53e89eee1bd 100644 --- a/pkg/controller/ovn_dnat.go +++ b/pkg/controller/ovn_dnat.go @@ -215,7 +215,7 @@ func (c *Controller) handleAddOvnDnatRule(key string) error { klog.V(3).Infof("handle add dnat %s", key) var internalV4Ip, mac, subnetName string - if cachedDnat.Spec.IpType == util.NatUsingVip { + if cachedDnat.Spec.IpType == util.Vip { internalVip, err := c.virtualIpsLister.Get(cachedDnat.Spec.IpName) if err != nil { klog.Errorf("failed to get vip %s, %v", cachedDnat.Spec.IpName, err) @@ -248,8 +248,10 @@ func (c *Controller) handleAddOvnDnatRule(key string) error { return err } - if cachedEip.Status.Type != "" && cachedEip.Status.Type != util.NatUsingEip { - err = fmt.Errorf("ovn eip %s type is not %s, can not use", cachedEip.Name, util.NatUsingEip) + if cachedEip.Spec.Type == util.Lsp { + // eip is using by ecmp nexthop lsp, nat can not use + err = fmt.Errorf("ovn nat %s can not use type %s eip %s", key, util.Lsp, eipName) + klog.Error(err) return err } @@ -354,7 +356,7 @@ func (c *Controller) handleUpdateOvnDnatRule(key string) error { klog.V(3).Infof("handle update dnat %s", key) var internalV4Ip, mac, subnetName string - if cachedDnat.Spec.IpType == util.NatUsingVip { + if cachedDnat.Spec.IpType == util.Vip { internalVip, err := c.virtualIpsLister.Get(cachedDnat.Spec.IpName) if err != nil { klog.Errorf("failed to get vip %s, %v", cachedDnat.Spec.IpName, err) @@ -410,16 +412,10 @@ func (c *Controller) handleUpdateOvnDnatRule(key string) error { return err } - if cachedEip.Spec.Type != "" && cachedEip.Spec.Type != util.DnatUsingEip { - // eip is in use by other nat - err = fmt.Errorf("failed to update dnat %s, eip '%s' is using by %s", key, eipName, cachedEip.Spec.Type) - return err - } - - if cachedEip.Spec.Type == util.DnatUsingEip && - cachedEip.Annotations[util.VpcNatAnnotation] != "" && - cachedEip.Annotations[util.VpcNatAnnotation] != cachedDnat.Name { - err = fmt.Errorf("failed to update dnat %s, eip '%s' is using by other dnat %s", key, eipName, cachedEip.Annotations[util.VpcNatAnnotation]) + if cachedEip.Spec.Type == util.Lsp { + // eip is using by ecmp nexthop lsp, nat can not use + err = fmt.Errorf("ovn nat %s can not use type %s eip %s", key, util.Lsp, eipName) + klog.Error(err) return err } diff --git a/pkg/controller/ovn_eip.go b/pkg/controller/ovn_eip.go index b47a70dde67..d1ce97555a1 100644 --- a/pkg/controller/ovn_eip.go +++ b/pkg/controller/ovn_eip.go @@ -247,7 +247,7 @@ func (c *Controller) handleAddOvnEip(key string) error { return err } - if cachedEip.Spec.Type == util.NodeExtGwUsingEip { + if cachedEip.Spec.Type == util.Lsp { mergedIp := util.GetStringIP(v4ip, v6ip) if err := c.ovnClient.CreateBareLogicalSwitchPort(subnet.Name, portName, mergedIp, mac); err != nil { klog.Error("failed to create lsp for ovn eip %s, %v", key, err) @@ -262,8 +262,8 @@ func (c *Controller) handleAddOvnEip(key string) error { klog.Errorf("failed to create or update ovn eip '%s', %v", cachedEip.Name, err) return err } - if cachedEip.Spec.Type != util.NodeExtGwUsingEip { - // node ext gw eip has a nic on node, so left node to make it ready + if cachedEip.Spec.Type != util.Lsp { + // node ext gw use lsp eip, has a nic on gw node, so left node to make it ready if err = c.patchOvnEipStatus(key, true); err != nil { klog.Errorf("failed to patch ovn eip %s: %v", key, err) return err @@ -302,8 +302,8 @@ func (c *Controller) handleUpdateOvnEip(key string) error { } return nil } - if cachedEip.Spec.Type != util.NodeExtGwUsingEip { - // node ext gw eip has a nic on node, so left node to make it ready + if cachedEip.Spec.Type != util.Lsp { + // node ext gw use lsp eip, has a nic on gw node, so left node to make it ready if err = c.patchOvnEipStatus(key, true); err != nil { klog.Errorf("failed to patch ovn eip %s: %v", key, err) return err @@ -345,14 +345,14 @@ func (c *Controller) handleDelOvnEip(eip *kubeovnv1.OvnEip) error { return err } - if eip.Spec.Type == util.NodeExtGwUsingEip { + if eip.Spec.Type == util.Lsp { if err := c.ovnClient.DeleteLogicalSwitchPort(eip.Name); err != nil { klog.Errorf("failed to delete lsp %s, %v", eip.Name, err) return err } } - if eip.Spec.Type == util.LrpUsingEip { + if eip.Spec.Type == util.Lrp { if err := c.ovnClient.DeleteLogicalRouterPort(eip.Name); err != nil { klog.Errorf("failed to delete lrp %s, %v", eip.Name, err) return err diff --git a/pkg/controller/ovn_fip.go b/pkg/controller/ovn_fip.go index bf9eafbd4bb..0da5c12f6c1 100644 --- a/pkg/controller/ovn_fip.go +++ b/pkg/controller/ovn_fip.go @@ -204,7 +204,7 @@ func (c *Controller) handleAddOvnFip(key string) error { } klog.V(3).Infof("handle add fip %s", key) var internalV4Ip, mac, subnetName string - if cachedFip.Spec.IpType == util.NatUsingVip { + if cachedFip.Spec.IpType == util.Vip { internalVip, err := c.virtualIpsLister.Get(cachedFip.Spec.IpName) if err != nil { klog.Errorf("failed to get vip %s, %v", cachedFip.Spec.IpName, err) @@ -237,6 +237,13 @@ func (c *Controller) handleAddOvnFip(key string) error { return err } + if cachedEip.Spec.Type == util.Lsp { + // eip is using by ecmp nexthop lsp, nat can not use + err = fmt.Errorf("ovn nat %s can not use type %s eip %s", key, util.Lsp, eipName) + klog.Error(err) + return err + } + if err = c.ovnFipTryUseEip(key, cachedEip.Spec.V4Ip); err != nil { err = fmt.Errorf("failed to add fip %s, %v", key, err) klog.Error(err) @@ -258,6 +265,7 @@ func (c *Controller) handleAddOvnFip(key string) error { err = fmt.Errorf("ovn eip %s type is not %s, can not use", cachedEip.Name, util.NatUsingEip) return err } + if err = c.ovnFipTryUseEip(key, cachedEip.Spec.V4Ip); err != nil { err = fmt.Errorf("failed to update fip %s, %v", key, err) klog.Error(err) @@ -310,7 +318,7 @@ func (c *Controller) handleUpdateOvnFip(key string) error { } klog.V(3).Infof("handle update fip %s", key) var internalV4Ip, mac, subnetName string - if cachedFip.Spec.IpType == util.NatUsingVip { + if cachedFip.Spec.IpType == util.Vip { internalVip, err := c.virtualIpsLister.Get(cachedFip.Spec.IpName) if err != nil { klog.Errorf("failed to get vip %s, %v", cachedFip.Spec.IpName, err) @@ -341,8 +349,10 @@ func (c *Controller) handleUpdateOvnFip(key string) error { klog.Errorf("failed to get eip, %v", err) return err } - if cachedEip.Status.Type != "" && cachedEip.Status.Type != util.NatUsingEip { - err = fmt.Errorf("ovn eip %s type is not %s, can not use", cachedEip.Name, util.NatUsingEip) + if cachedEip.Spec.Type == util.Lsp { + // eip is using by ecmp nexthop lsp, nat can not use + err = fmt.Errorf("ovn nat %s can not use type %s eip %s", key, util.Lsp, eipName) + klog.Error(err) return err } if err = c.ovnFipTryUseEip(key, cachedEip.Spec.V4Ip); err != nil { diff --git a/pkg/controller/ovn_snat.go b/pkg/controller/ovn_snat.go index 918b9c46cb5..47faa0a55ed 100644 --- a/pkg/controller/ovn_snat.go +++ b/pkg/controller/ovn_snat.go @@ -197,8 +197,10 @@ func (c *Controller) handleAddOvnSnatRule(key string) error { return err } - if cachedEip.Status.Type != "" && cachedEip.Status.Type != util.NatUsingEip { - err = fmt.Errorf("ovn eip %s type is not %s, can not use", cachedEip.Name, util.NatUsingEip) + if cachedEip.Spec.Type == util.Lsp { + // eip is using by ecmp nexthop lsp, nat can not use + err = fmt.Errorf("ovn nat %s can not use type %s eip %s", key, util.Lsp, eipName) + klog.Error(err) return err } @@ -303,8 +305,10 @@ func (c *Controller) handleUpdateOvnSnatRule(key string) error { return nil } - if cachedEip.Status.Type != "" && cachedEip.Status.Type != util.NatUsingEip { - err = fmt.Errorf("ovn eip %s type is not %s, can not use", cachedEip.Name, util.NatUsingEip) + if cachedEip.Spec.Type == util.Lsp { + // eip is using by ecmp nexthop lsp, nat can not use + err = fmt.Errorf("ovn nat %s can not use type %s eip %s", key, util.Lsp, eipName) + klog.Error(err) return err } diff --git a/pkg/controller/subnet.go b/pkg/controller/subnet.go index 33621aed713..cc8e28d8871 100644 --- a/pkg/controller/subnet.go +++ b/pkg/controller/subnet.go @@ -1190,13 +1190,13 @@ func (c *Controller) reconcileNamespaces(subnet *kubeovnv1.Subnet) error { func (c *Controller) reconcileCustomVpcBfdStaticRoute(vpcName, subnetName string) error { // vpc enable bfd and subnet enable ecmp // use static ecmp route with bfd - ovnEips, err := c.ovnEipsLister.List(labels.SelectorFromSet(labels.Set{util.OvnEipTypeLabel: util.NodeExtGwUsingEip})) + ovnEips, err := c.ovnEipsLister.List(labels.SelectorFromSet(labels.Set{util.OvnEipTypeLabel: util.Lsp})) if err != nil { klog.Errorf("failed to list node external ovn eip, %v", err) return err } if len(ovnEips) < 2 { - err := fmt.Errorf("ecmp route with bfd for HA, which need two %s type eips at least, has %d", util.NodeExtGwUsingEip, len(ovnEips)) + err := fmt.Errorf("ecmp route with bfd for HA, which need two %s type eips at least, has %d", util.Lsp, len(ovnEips)) klog.Error(err) return err } @@ -1266,17 +1266,15 @@ func (c *Controller) reconcileCustomVpcBfdStaticRoute(vpcName, subnetName string } } if needUpdate { - if _, err = c.config.KubeOvnClient.KubeovnV1().Vpcs().Update(context.Background(), vpc, metav1.UpdateOptions{}); err != nil { + if vpc, err = c.config.KubeOvnClient.KubeovnV1().Vpcs().Update(context.Background(), vpc, metav1.UpdateOptions{}); err != nil { klog.Errorf("failed to update vpc spec static route %s, %v", vpc.Name, err) return err } + if err = c.patchVpcBfdStatus(vpc.Name); err != nil { + klog.Errorf("failed to patch vpc %s, %v", vpc.Name, err) + return err + } } - - if err = c.patchVpcBfdStatus(vpc.Name); err != nil { - klog.Errorf("failed to patch vpc %s, %v", vpc.Name, err) - return err - } - return nil } @@ -1292,7 +1290,7 @@ func (c *Controller) reconcileCustomVpcAddNormalStaticRoute(vpcName string) erro return err } gatewayV4, gatewayV6 := util.SplitStringIP(defualtExternalSubnet.Spec.Gateway) - vpc, err := c.vpcsLister.Get(vpcName) + cachedVpc, err := c.vpcsLister.Get(vpcName) if err != nil { if k8serrors.IsNotFound(err) { return nil @@ -1300,7 +1298,7 @@ func (c *Controller) reconcileCustomVpcAddNormalStaticRoute(vpcName string) erro klog.Errorf("failed to get vpc %s, %v", vpcName, err) return err } - + vpc := cachedVpc.DeepCopy() rtbs := c.getRouteTablesByVpc(vpc) routeTotal := len(vpc.Spec.StaticRoutes) + len(rtbs)*2 routes := make([]*kubeovnv1.StaticRoute, 0, routeTotal) @@ -1351,17 +1349,15 @@ func (c *Controller) reconcileCustomVpcAddNormalStaticRoute(vpcName string) erro if needUpdate { vpc.Spec.StaticRoutes = routes - if _, err = c.config.KubeOvnClient.KubeovnV1().Vpcs().Update(context.Background(), vpc, metav1.UpdateOptions{}); err != nil { + if vpc, err = c.config.KubeOvnClient.KubeovnV1().Vpcs().Update(context.Background(), vpc, metav1.UpdateOptions{}); err != nil { klog.Errorf("failed to update vpc spec static route %s, %v", vpc.Name, err) return err } + if err = c.patchVpcBfdStatus(vpc.Name); err != nil { + klog.Errorf("failed to patch vpc %s, %v", vpc.Name, err) + return err + } } - - if err = c.patchVpcBfdStatus(vpc.Name); err != nil { - klog.Errorf("failed to patch vpc %s, %v", vpc.Name, err) - return err - } - return nil } diff --git a/pkg/controller/vpc.go b/pkg/controller/vpc.go index 6e4d47801ac..1ae09be459c 100644 --- a/pkg/controller/vpc.go +++ b/pkg/controller/vpc.go @@ -46,20 +46,15 @@ func (c *Controller) enqueueUpdateVpc(old, new interface{}) { utilruntime.HandleError(err) return } - - _, oldOk := oldVpc.Labels[util.VpcExternalLabel] - _, newOk := newVpc.Labels[util.VpcExternalLabel] - if oldOk || newOk { - return - } - - if !newVpc.DeletionTimestamp.IsZero() || + if newVpc.DeletionTimestamp.IsZero() || !reflect.DeepEqual(oldVpc.Spec.Namespaces, newVpc.Spec.Namespaces) || !reflect.DeepEqual(oldVpc.Spec.StaticRoutes, newVpc.Spec.StaticRoutes) || !reflect.DeepEqual(oldVpc.Spec.PolicyRoutes, newVpc.Spec.PolicyRoutes) || !reflect.DeepEqual(oldVpc.Spec.VpcPeerings, newVpc.Spec.VpcPeerings) || - !reflect.DeepEqual(oldVpc.Annotations, newVpc.Annotations) { - klog.V(3).Infof("enqueue update vpc %s", key) + !reflect.DeepEqual(oldVpc.Annotations, newVpc.Annotations) || + oldVpc.Labels[util.VpcExternalLabel] != newVpc.Labels[util.VpcExternalLabel] { + // TODO:// label VpcExternalLabel replace with spec enable external + klog.Infof("enqueue update vpc %s", key) c.addOrUpdateVpcQueue.Add(key) } } @@ -281,9 +276,9 @@ func (c *Controller) handleAddOrUpdateVpc(key string) error { klog.Errorf("failed to get vpc %s static route list, %v", vpc.Name, err) return err } - rtbs := c.getRouteTablesByVpc(vpc) targetRoutes := vpc.Spec.StaticRoutes + klog.Infof("vpc %s spec static routes: %v, exist route:", key, targetRoutes, existRoute) if vpc.Name == c.config.ClusterRouter { if _, ok := rtbs[util.MainRouteTable]; !ok { rtbs[util.MainRouteTable] = nil @@ -380,8 +375,8 @@ func (c *Controller) handleAddOrUpdateVpc(key string) error { } for _, item := range routeNeedAdd { - klog.Infof("vpc %s add static route: %+v", vpc.Name, item) if item.BfdId != "" { + klog.Infof("vpc %s add static ecmp route: %+v", vpc.Name, item) if err = c.ovnClient.AddLogicalRouterStaticRoute( vpc.Name, item.RouteTable, convertPolicy(item.Policy), item.CIDR, &item.BfdId, item.NextHopIP, ); err != nil { @@ -389,6 +384,7 @@ func (c *Controller) handleAddOrUpdateVpc(key string) error { return err } } else { + klog.Infof("vpc %s add static route: %+v", vpc.Name, item) if err = c.ovnClient.AddLogicalRouterStaticRoute( vpc.Name, item.RouteTable, convertPolicy(item.Policy), item.CIDR, nil, item.NextHopIP, ); err != nil { @@ -812,7 +808,7 @@ func (c *Controller) handleAddVpcExternal(key string) error { klog.Errorf("failed to acquire ip address for lrp eip %s, %v", lrpEipName, err) return err } - if err := c.createOrUpdateCrdOvnEip(lrpEipName, c.config.ExternalGatewaySwitch, v4ip, v6ip, mac, util.LrpUsingEip); err != nil { + if err := c.createOrUpdateCrdOvnEip(lrpEipName, c.config.ExternalGatewaySwitch, v4ip, v6ip, mac, util.Lrp); err != nil { klog.Errorf("failed to create ovn eip for lrp %s: %v", lrpEipName, err) return err } @@ -972,6 +968,7 @@ func (c *Controller) patchVpcBfdStatus(key string) error { } vpc := cachedVpc.DeepCopy() if vpc.Status.EnableBfd != vpc.Spec.EnableBfd { + vpc.Status.EnableExternal = cachedVpc.Spec.EnableExternal vpc.Status.EnableBfd = cachedVpc.Spec.EnableBfd bytes, err := vpc.Status.Bytes() if err != nil { diff --git a/pkg/daemon/ovs_linux.go b/pkg/daemon/ovs_linux.go index 2f730ae145c..10fb3b3d7b6 100644 --- a/pkg/daemon/ovs_linux.go +++ b/pkg/daemon/ovs_linux.go @@ -443,14 +443,14 @@ func (c *Controller) checkNodeGwNicInNs(nodeExtIp, ip, gw string, gwNS ns.NetNS) klog.Error(err) return err } - filters := labels.Set{util.OvnEipTypeLabel: util.LrpUsingEip} + filters := labels.Set{util.OvnEipTypeLabel: util.Lrp} ovnEips, err := c.ovnEipsLister.List(labels.SelectorFromSet(filters)) if err != nil { klog.Errorf("failed to list ovn eip, %v", err) return err } if len(ovnEips) == 0 { - klog.Errorf("failed to get type %s ovn eip, %v", util.LrpUsingEip, err) + klog.Errorf("failed to get type %s ovn eip, %v", util.Lrp, err) // node ext gw eip need lrp eip to establish bfd session return nil } diff --git a/pkg/ovs/ovn-nb-logical_router_route.go b/pkg/ovs/ovn-nb-logical_router_route.go index 99f4dbe7611..385adcdf180 100644 --- a/pkg/ovs/ovn-nb-logical_router_route.go +++ b/pkg/ovs/ovn-nb-logical_router_route.go @@ -77,6 +77,9 @@ func (c *ovnClient) AddLogicalRouterStaticRoute(lrName, routeTable, policy, ipPr if util.ContainsString(nexthops, route.Nexthop) { existing.Add(route.Nexthop) } else { + if route.BFD != nil && bfdId != nil && *route.BFD != *bfdId { + continue + } toDel = append(toDel, route.UUID) } } @@ -91,10 +94,7 @@ func (c *ovnClient) AddLogicalRouterStaticRoute(lrName, routeTable, policy, ipPr toAdd = append(toAdd, route) } } - - if err = c.CreateLogicalRouterStaticRoutes(lrName, toAdd...); err != nil { - return fmt.Errorf("add static routes to logical router %s: %v", lrName, err) - } + klog.Infof("logical router %s del static routes: %v", lrName, toDel) ops, err := c.LogicalRouterUpdateStaticRouteOp(lrName, toDel, ovsdb.MutateOperationDelete) if err != nil { klog.Error(err) @@ -104,7 +104,10 @@ func (c *ovnClient) AddLogicalRouterStaticRoute(lrName, routeTable, policy, ipPr klog.Error(err) return fmt.Errorf("failed to delete static routes from logical router %s: %v", lrName, err) } - + klog.Infof("logical router %s add static routes: %v", lrName, toAdd) + if err = c.CreateLogicalRouterStaticRoutes(lrName, toAdd...); err != nil { + return fmt.Errorf("failed to add static routes to logical router %s: %v", lrName, err) + } return nil } diff --git a/pkg/util/const.go b/pkg/util/const.go index 43b6c1d76a3..83ad8a71a28 100644 --- a/pkg/util/const.go +++ b/pkg/util/const.go @@ -172,18 +172,19 @@ const ( DefaultVpc = "ovn-cluster" DefaultSubnet = "ovn-default" - EcmpRouteType = "ecmp" - NormalRouteType = "normal" - - LrpUsingEip = "lrp" - FipUsingEip = "fip" - NatUsingVip = "vip" - NatUsingEip = "eip" - SnatUsingEip = "snat" - DnatUsingEip = "dnat" - NodeExtGwUsingEip = "node-ext-gw" + NormalRouteType = "normal" + EcmpRouteType = "ecmp" StaticRouteBfdEcmp = "ecmp_symmetric_reply" + Vip = "vip" + Lrp = "lrp" + Lsp = "lsp" + + NatUsingEip = "nat" + FipUsingEip = "fip" + SnatUsingEip = "snat" + DnatUsingEip = "dnat" + OvnFip = "ovn" IptablesFip = "iptables" diff --git a/test/e2e/ovn-vpc-nat-gw/e2e_test.go b/test/e2e/ovn-vpc-nat-gw/e2e_test.go index 64c4cb39245..bb807c0800c 100644 --- a/test/e2e/ovn-vpc-nat-gw/e2e_test.go +++ b/test/e2e/ovn-vpc-nat-gw/e2e_test.go @@ -301,16 +301,14 @@ var _ = framework.Describe("[group:ovn-vpc-nat-gw]", func() { ginkgo.By("Deleting ovn snat " + snatName) ovnSnatRuleClient.DeleteSync(snatName) - ginkgo.By("Deleting ovn eip " + fipEipName) - ovnEipClient.DeleteSync(fipEipName) + ginkgo.By("Deleting ovn fip " + fipEipName) + ovnFipClient.DeleteSync(fipEipName) ginkgo.By("Deleting ovn eip " + dnatEipName) ovnEipClient.DeleteSync(dnatEipName) ginkgo.By("Deleting ovn eip " + snatEipName) ovnEipClient.DeleteSync(snatEipName) ginkgo.By("Deleting ovn share eip " + sharedEipName) ovnEipClient.DeleteSync(sharedEipName) - ginkgo.By("Deleting ovn share vip " + sharedEipName) - vipClient.DeleteSync(sharedVipName) ginkgo.By("Deleting ovn vip " + arpProxyVip1Name) vipClient.DeleteSync(arpProxyVip1Name) @@ -320,6 +318,8 @@ var _ = framework.Describe("[group:ovn-vpc-nat-gw]", func() { vipClient.DeleteSync(dnatVipName) ginkgo.By("Deleting ovn vip " + fipVipName) vipClient.DeleteSync(fipVipName) + ginkgo.By("Deleting ovn share vip " + sharedVipName) + vipClient.DeleteSync(sharedVipName) ginkgo.By("Deleting subnet " + noBfdSubnetName) subnetClient.DeleteSync(noBfdSubnetName) @@ -445,13 +445,13 @@ var _ = framework.Describe("[group:ovn-vpc-nat-gw]", func() { shareEip := framework.MakeOvnEip(sharedEipName, underlaySubnetName, "", "", "", "") _ = ovnEipClient.CreateSync(shareEip) ginkgo.By("Creating the first ovn fip with share eip vip should be ok") - shareFipShouldOk := framework.MakeOvnFip(sharedEipFipShoudOkName, sharedEipName, util.NatUsingVip, sharedVipName) + shareFipShouldOk := framework.MakeOvnFip(sharedEipFipShoudOkName, sharedEipName, util.Vip, sharedVipName) _ = ovnFipClient.CreateSync(shareFipShouldOk) ginkgo.By("Creating the second ovn fip with share eip vip should be failed") - shareFipShouldFail := framework.MakeOvnFip(sharedEipFipShoudFailName, sharedEipName, util.NatUsingVip, sharedVipName) + shareFipShouldFail := framework.MakeOvnFip(sharedEipFipShoudFailName, sharedEipName, util.Vip, sharedVipName) _ = ovnFipClient.Create(shareFipShouldFail) ginkgo.By("Creating ovn dnat for dnat with share eip vip") - shareDnat := framework.MakeOvnDnatRule(sharedEipDnatName, sharedEipName, util.NatUsingVip, sharedVipName, "80", "8080", "tcp") + shareDnat := framework.MakeOvnDnatRule(sharedEipDnatName, sharedEipName, util.Vip, sharedVipName, "80", "8080", "tcp") _ = ovnDnatRuleClient.CreateSync(shareDnat) ginkgo.By("Creating ovn snat with share eip vip") shareSnat := framework.MakeOvnSnatRule(sharedEipSnatName, sharedEipName, noBfdSubnetName, "") @@ -502,7 +502,7 @@ var _ = framework.Describe("[group:ovn-vpc-nat-gw]", func() { ginkgo.By("2. Creating custom vpc enable external and bfd") for _, nodeName := range nodeNames { ginkgo.By("Creating ovn node-ext-gw type eip on node " + nodeName) - eip := makeOvnEip(nodeName, underlaySubnetName, "", "", "", util.NodeExtGwUsingEip) + eip := makeOvnEip(nodeName, underlaySubnetName, "", "", "", util.Lsp) _ = ovnEipClient.CreateSync(eip) } bfdSubnetV4Cidr := "192.168.1.0/24" @@ -538,7 +538,7 @@ var _ = framework.Describe("[group:ovn-vpc-nat-gw]", func() { eip := makeOvnEip(fipEipName, underlaySubnetName, "", "", "", "") _ = ovnEipClient.CreateSync(eip) ginkgo.By("Creating ovn fip " + fipName) - fip := makeOvnFip(fipName, fipEipName, util.NatUsingVip, fipVipName) + fip := makeOvnFip(fipName, fipEipName, util.Vip, fipVipName) _ = ovnFipClient.CreateSync(fip) ginkgo.By("Creating ovn eip " + snatEipName) @@ -555,7 +555,7 @@ var _ = framework.Describe("[group:ovn-vpc-nat-gw]", func() { dnatEip := makeOvnEip(dnatEipName, underlaySubnetName, "", "", "", "") _ = ovnEipClient.CreateSync(dnatEip) ginkgo.By("Creating ovn dnat " + dnatName) - dnat := makeOvnDnat(dnatName, dnatEipName, util.NatUsingVip, dnatVipName, "80", "8080", "tcp") + dnat := makeOvnDnat(dnatName, dnatEipName, util.Vip, dnatVipName, "80", "8080", "tcp") _ = ovnDnatRuleClient.CreateSync(dnat) k8sNodes, err := e2enode.GetReadySchedulableNodes(context.Background(), cs) @@ -572,7 +572,6 @@ var _ = framework.Describe("[group:ovn-vpc-nat-gw]", func() { framework.ExpectEqual(route.Policy, kubeovnv1.PolicySrc) framework.ExpectNotEmpty(route.CIDR) } - k8sNodes, err = e2enode.GetReadySchedulableNodes(context.Background(), cs) framework.ExpectNoError(err) for _, node := range k8sNodes.Items { @@ -605,43 +604,38 @@ var _ = framework.Describe("[group:ovn-vpc-nat-gw]", func() { framework.ExpectNotEmpty(nodes) ginkgo.By("Creating crd in distributed case") for _, node := range nodeNames { - podName := fmt.Sprintf("fip-%s", node) - ginkgo.By("Creating pod " + podName + " with subnet " + bfdSubnetName) + podOnNodeName := fmt.Sprintf("on-node-%s", node) + eipOnNodeName := fmt.Sprintf("eip-on-node-%s", node) + fipOnNodeName := fmt.Sprintf("fip-on-node-%s", node) + ginkgo.By("Creating pod " + podOnNodeName + " with subnet " + bfdSubnetName) annotations := map[string]string{util.LogicalSwitchAnnotation: bfdSubnetName} cmd := []string{"sh", "-c", "sleep infinity"} - pod := framework.MakePod(namespaceName, podName, nil, annotations, image, cmd, nil) + pod := framework.MakePod(namespaceName, podOnNodeName, nil, annotations, image, cmd, nil) pod.Spec.NodeName = node _ = podClient.CreateSync(pod) // create fip in distributed case // for now, vip has no lsp, so not support in distributed case - ipName := ovs.PodNameToPortName(podName, namespaceName, bfdSubnet.Spec.Provider) + ipName := ovs.PodNameToPortName(podOnNodeName, namespaceName, bfdSubnet.Spec.Provider) ginkgo.By("Get pod ip" + ipName) ip := ipClient.Get(ipName) - fipEipName = fmt.Sprintf("fip-%s", node) - ginkgo.By("Creating ovn eip " + fipEipName) - eip = makeOvnEip(fipEipName, underlaySubnetName, "", "", "", "") + ginkgo.By("Creating ovn eip " + eipOnNodeName) + eip = makeOvnEip(eipOnNodeName, underlaySubnetName, "", "", "", "") _ = ovnEipClient.CreateSync(eip) - fipName = fmt.Sprintf("fip-%s", node) - ginkgo.By("Creating ovn fip " + fipName) - fip := makeOvnFip(fipName, fipEipName, "", ip.Name) + ginkgo.By("Creating ovn fip " + fipOnNodeName) + fip := makeOvnFip(fipOnNodeName, eipOnNodeName, "", ip.Name) _ = ovnFipClient.CreateSync(fip) - // clean fip eip in distributed case - ginkgo.By("Deleting ovn fip " + fipName) - ovnFipClient.DeleteSync(fipName) - ginkgo.By("Deleting ovn eip " + fipEipName) - ovnEipClient.DeleteSync(fipEipName) } ginkgo.By("Deleting crd in distributed case") for _, node := range nodeNames { - fipEipName = fmt.Sprintf("fip-%s", node) - fipName = fmt.Sprintf("fip-%s", node) - ginkgo.By("Deleting ovn fip " + fipName) - ovnFipClient.DeleteSync(fipName) - ginkgo.By("Deleting ovn eip " + fipEipName) - ovnEipClient.DeleteSync(fipEipName) - podName := fmt.Sprintf("fip-%s", node) - ipName := ovs.PodNameToPortName(podName, namespaceName, bfdSubnet.Spec.Provider) + podOnNodeName := fmt.Sprintf("on-node-%s", node) + eipOnNodeName := fmt.Sprintf("eip-on-node-%s", node) + fipOnNodeName := fmt.Sprintf("fip-on-node-%s", node) + ginkgo.By("Deleting node ovn fip " + fipOnNodeName) + ovnFipClient.DeleteSync(fipOnNodeName) + ginkgo.By("Deleting node ovn eip " + eipOnNodeName) + ovnEipClient.DeleteSync(eipOnNodeName) + ipName := ovs.PodNameToPortName(podOnNodeName, namespaceName, bfdSubnet.Spec.Provider) ginkgo.By("Deleting pod ip" + ipName) ipClient.DeleteSync(ipName) } @@ -665,7 +659,6 @@ var _ = framework.Describe("[group:ovn-vpc-nat-gw]", func() { // label should be false after remove node external gw framework.ExpectHaveKeyWithValue(node.Labels, util.NodeExtGwLabel, "false") } - }) })