From 55dbfbb8acbef4b3b7992ac97b8d1251559c2f42 Mon Sep 17 00:00:00 2001 From: bobz965 Date: Wed, 29 May 2024 16:10:40 +0800 Subject: [PATCH] Fix change subnet (#4088) * couting vip ovn eip add and del --------- Signed-off-by: bobz965 --- charts/kube-ovn/templates/kube-ovn-crd.yaml | 3 + dist/images/install.sh | 3 + pkg/controller/ip.go | 72 +++++++++------ pkg/controller/pod.go | 39 --------- pkg/controller/subnet.go | 97 ++++++++++++++++----- pkg/webhook/ip.go | 16 ++++ test/e2e/framework/image.go | 2 +- test/e2e/kube-ovn/underlay/underlay.go | 48 +++++++++- test/e2e/ovn-vpc-nat-gw/e2e_test.go | 45 ++++++++-- test/e2e/vip/e2e_test.go | 41 ++++++++- 10 files changed, 264 insertions(+), 102 deletions(-) diff --git a/charts/kube-ovn/templates/kube-ovn-crd.yaml b/charts/kube-ovn/templates/kube-ovn-crd.yaml index dd08755212c..fa994821913 100644 --- a/charts/kube-ovn/templates/kube-ovn-crd.yaml +++ b/charts/kube-ovn/templates/kube-ovn-crd.yaml @@ -1191,6 +1191,9 @@ spec: - jsonPath: .status.ready name: Ready type: boolean + - jsonPath: .spec.externalSubnet + name: ExternalSubnet + type: string schema: openAPIV3Schema: type: object diff --git a/dist/images/install.sh b/dist/images/install.sh index c698376c875..73d8d62e002 100755 --- a/dist/images/install.sh +++ b/dist/images/install.sh @@ -1427,6 +1427,9 @@ spec: - jsonPath: .status.ready name: Ready type: boolean + - jsonPath: .spec.externalSubnet + name: ExternalSubnet + type: string schema: openAPIV3Schema: type: object diff --git a/pkg/controller/ip.go b/pkg/controller/ip.go index 2f188a5072d..3816cf3d40c 100644 --- a/pkg/controller/ip.go +++ b/pkg/controller/ip.go @@ -64,6 +64,31 @@ func (c *Controller) enqueueUpdateIP(oldObj, newObj interface{}) { c.updateSubnetStatusQueue.Add(as) } } + // ip can not change these specs below + if oldIP.Spec.Subnet != "" && newIP.Spec.Subnet != oldIP.Spec.Subnet { + klog.Errorf("ip %s subnet can not change", newIP.Name) + } + if oldIP.Spec.Namespace != "" && newIP.Spec.Namespace != oldIP.Spec.Namespace { + klog.Errorf("ip %s namespace can not change", newIP.Name) + } + if oldIP.Spec.PodName != "" && newIP.Spec.PodName != oldIP.Spec.PodName { + klog.Errorf("ip %s podName can not change", newIP.Name) + } + if oldIP.Spec.PodType != "" && newIP.Spec.PodType != oldIP.Spec.PodType { + klog.Errorf("ip %s podType can not change", newIP.Name) + } + if oldIP.Spec.MacAddress != "" && newIP.Spec.MacAddress != oldIP.Spec.MacAddress { + klog.Errorf("ip %s macAddress can not change", newIP.Name) + } + if oldIP.Spec.NodeName != "" && newIP.Spec.NodeName != oldIP.Spec.NodeName { + klog.Errorf("ip %s nodeName can not change", newIP.Name) + } + if oldIP.Spec.V4IPAddress != "" && newIP.Spec.V4IPAddress != oldIP.Spec.V4IPAddress { + klog.Errorf("ip %s v4IPAddress can not change", newIP.Name) + } + if oldIP.Spec.V6IPAddress != "" && newIP.Spec.V6IPAddress != oldIP.Spec.V6IPAddress { + klog.Errorf("ip %s v6IPAddress can not change", newIP.Name) + } } func (c *Controller) enqueueDelIP(obj interface{}) { @@ -286,7 +311,6 @@ func (c *Controller) handleUpdateIP(key string) error { klog.Errorf("failed to get subnet %s: %v", cachedIP.Spec.Subnet, err) return err } - cleanIPAM := true if isOvnSubnet(subnet) { portName := cachedIP.Name port, err := c.OVNNbClient.GetLogicalSwitchPort(portName, true) @@ -294,41 +318,33 @@ func (c *Controller) handleUpdateIP(key string) error { klog.Errorf("failed to get logical switch port %s: %v", portName, err) return err } - if port != nil && len(port.Addresses) > 0 { - address := port.Addresses[0] - if strings.Contains(address, cachedIP.Spec.MacAddress) { - klog.Infof("delete ip cr lsp %s from switch %s", portName, subnet.Name) - if err := c.OVNNbClient.DeleteLogicalSwitchPort(portName); err != nil { - klog.Errorf("failed to delete ip cr lsp %s from switch %s: %v", portName, subnet.Name, err) - return err - } - klog.V(3).Infof("sync sg for deleted port %s", portName) - sgList, err := c.getPortSg(port) - if err != nil { - klog.Errorf("get port sg failed, %v", err) - return err - } - for _, sgName := range sgList { - if sgName != "" { - c.syncSgPortsQueue.Add(sgName) - } + if port != nil { + klog.Infof("delete ip cr lsp %s from switch %s", portName, subnet.Name) + if err := c.OVNNbClient.DeleteLogicalSwitchPort(portName); err != nil { + klog.Errorf("failed to delete ip cr lsp %s from switch %s: %v", portName, subnet.Name, err) + return err + } + klog.V(3).Infof("sync sg for deleted port %s", portName) + sgList, err := c.getPortSg(port) + if err != nil { + klog.Errorf("get port sg failed, %v", err) + return err + } + for _, sgName := range sgList { + if sgName != "" { + c.syncSgPortsQueue.Add(sgName) } - } else { - // ip subnet changed in pod handle add or update pod process - klog.Infof("lsp %s ip changed, only delete old ip cr %s", portName, key) - cleanIPAM = false } } } - if cleanIPAM { - podKey := fmt.Sprintf("%s/%s", cachedIP.Spec.Namespace, cachedIP.Spec.PodName) - klog.Infof("ip cr %s release ipam pod key %s from subnet %s", cachedIP.Name, podKey, cachedIP.Spec.Subnet) - c.ipam.ReleaseAddressByPod(podKey, cachedIP.Spec.Subnet) - } + podKey := fmt.Sprintf("%s/%s", cachedIP.Spec.Namespace, cachedIP.Spec.PodName) + klog.Infof("ip cr %s release ipam pod key %s from subnet %s", cachedIP.Name, podKey, cachedIP.Spec.Subnet) + c.ipam.ReleaseAddressByPod(podKey, cachedIP.Spec.Subnet) if err = c.handleDelIPFinalizer(cachedIP); err != nil { klog.Errorf("failed to handle del ip finalizer %v", err) return err } + c.updateSubnetStatusQueue.Add(cachedIP.Spec.Subnet) } return nil } diff --git a/pkg/controller/pod.go b/pkg/controller/pod.go index 96b3faa514c..531bb9a0351 100644 --- a/pkg/controller/pod.go +++ b/pkg/controller/pod.go @@ -534,39 +534,6 @@ func (c *Controller) getPodKubeovnNets(pod *v1.Pod) ([]*kubeovnNet, error) { return podNets, nil } -func (c *Controller) changeVMSubnet(vmName, namespace, providerName, subnetName string) error { - ipName := ovs.PodNameToPortName(vmName, namespace, providerName) - ipCR, err := c.ipsLister.Get(ipName) - if err != nil { - if k8serrors.IsNotFound(err) { - return nil - } - err := fmt.Errorf("failed to get ip CR %s: %v", ipName, err) - klog.Error(err) - return err - } - if ipCR.Spec.Subnet != subnetName { - key := fmt.Sprintf("%s/%s", namespace, vmName) - klog.Infof("release ipam for vm %s from old subnet %s", key, ipCR.Spec.Subnet) - c.ipam.ReleaseAddressByPod(key, ipCR.Spec.Subnet) - klog.Infof("gc logical switch port %s", key) - if err := c.OVNNbClient.DeleteLogicalSwitchPort(key); err != nil { - klog.Errorf("failed to delete lsp %s, %v", key, err) - return err - } - // old lsp has been deleted, delete old ip cr - if err := c.config.KubeOvnClient.KubeovnV1().IPs().Delete(context.Background(), ipName, metav1.DeleteOptions{}); err != nil { - if !k8serrors.IsNotFound(err) { - klog.Errorf("failed to delete ip %s, %v", ipName, err) - return err - } - } - c.updateSubnetStatusQueue.Add(ipCR.Spec.Subnet) - // handleAddOrUpdatePod will create new lsp and new ip cr - } - return nil -} - func (c *Controller) handleAddOrUpdatePod(key string) (err error) { namespace, name, err := cache.SplitMetaNamespaceKey(key) if err != nil { @@ -679,15 +646,9 @@ func (c *Controller) reconcileAllocateSubnets(cachedPod, pod *v1.Pod, needAlloca delete(pod.Annotations, fmt.Sprintf(util.PodNicAnnotationTemplate, podNet.ProviderName)) } pod.Annotations[fmt.Sprintf(util.AllocatedAnnotationTemplate, podNet.ProviderName)] = "true" - if vmKey != "" { pod.Annotations[fmt.Sprintf(util.VMAnnotationTemplate, podNet.ProviderName)] = vmName - if err := c.changeVMSubnet(vmName, namespace, podNet.ProviderName, subnet.Name); err != nil { - klog.Errorf("vm %s change subnet to %s failed: %v", vmKey, subnet.Name, err) - return nil, err - } } - if err := util.ValidateNetworkBroadcast(podNet.Subnet.Spec.CIDRBlock, ipStr); err != nil { klog.Errorf("validate pod %s/%s failed: %v", namespace, name, err) c.recorder.Eventf(pod, v1.EventTypeWarning, "ValidatePodNetworkFailed", err.Error()) diff --git a/pkg/controller/subnet.go b/pkg/controller/subnet.go index dbfdf208820..f98e9ad80fa 100644 --- a/pkg/controller/subnet.go +++ b/pkg/controller/subnet.go @@ -540,7 +540,7 @@ func (c *Controller) handleSubnetFinalizer(subnet *kubeovnv1.Subnet) (bool, erro return false, nil } -func (c Controller) patchSubnetStatus(subnet *kubeovnv1.Subnet, reason, errStr string) { +func (c Controller) patchSubnetStatus(subnet *kubeovnv1.Subnet, reason, errStr string) error { if errStr != "" { subnet.Status.SetError(reason, errStr) subnet.Status.NotValidated(reason, errStr) @@ -557,13 +557,16 @@ func (c Controller) patchSubnetStatus(subnet *kubeovnv1.Subnet, reason, errStr s } } - if bytes, err := subnet.Status.Bytes(); err != nil { + bytes, err := subnet.Status.Bytes() + if err != nil { klog.Error(err) - } else { - if _, err := c.config.KubeOvnClient.KubeovnV1().Subnets().Patch(context.Background(), subnet.Name, types.MergePatchType, bytes, metav1.PatchOptions{}, "status"); err != nil { - klog.Error("patch subnet status failed", err) - } + return err + } + if _, err := c.config.KubeOvnClient.KubeovnV1().Subnets().Patch(context.Background(), subnet.Name, types.MergePatchType, bytes, metav1.PatchOptions{}, "status"); err != nil { + klog.Errorf("failed to patch status for subnet %s, %v", subnet.Name, err) + return err } + return nil } func (c *Controller) validateVpcBySubnet(subnet *kubeovnv1.Subnet) (*kubeovnv1.Vpc, error) { @@ -621,7 +624,10 @@ func (c *Controller) checkSubnetConflict(subnet *kubeovnv1.Subnet) error { if util.CIDROverlap(sub.Spec.CIDRBlock, subnet.Spec.CIDRBlock) { err = fmt.Errorf("subnet %s cidr %s is conflict with subnet %s cidr %s", subnet.Name, subnet.Spec.CIDRBlock, sub.Name, sub.Spec.CIDRBlock) klog.Error(err) - c.patchSubnetStatus(subnet, "ValidateLogicalSwitchFailed", err.Error()) + if err = c.patchSubnetStatus(subnet, "ValidateLogicalSwitchFailed", err.Error()); err != nil { + klog.Error(err) + return err + } return err } @@ -629,7 +635,10 @@ func (c *Controller) checkSubnetConflict(subnet *kubeovnv1.Subnet) error { subnet.Spec.PolicyRoutingTableID == sub.Spec.PolicyRoutingTableID { err = fmt.Errorf("subnet %s policy routing table ID %d is conflict with subnet %s policy routing table ID %d", subnet.Name, subnet.Spec.PolicyRoutingTableID, sub.Name, sub.Spec.PolicyRoutingTableID) klog.Error(err) - c.patchSubnetStatus(subnet, "ValidateLogicalSwitchFailed", err.Error()) + if err = c.patchSubnetStatus(subnet, "ValidateLogicalSwitchFailed", err.Error()); err != nil { + klog.Error(err) + return err + } return err } } @@ -645,7 +654,10 @@ func (c *Controller) checkSubnetConflict(subnet *kubeovnv1.Subnet) error { if addr.Type == v1.NodeInternalIP && util.CIDRContainIP(subnet.Spec.CIDRBlock, addr.Address) { err = fmt.Errorf("subnet %s cidr %s conflict with node %s address %s", subnet.Name, subnet.Spec.CIDRBlock, node.Name, addr.Address) klog.Error(err) - c.patchSubnetStatus(subnet, "ValidateLogicalSwitchFailed", err.Error()) + if err = c.patchSubnetStatus(subnet, "ValidateLogicalSwitchFailed", err.Error()); err != nil { + klog.Error(err) + return err + } return err } } @@ -736,10 +748,16 @@ func (c *Controller) handleAddOrUpdateSubnet(key string) error { if err = util.ValidateSubnet(*subnet); err != nil { klog.Errorf("failed to validate subnet %s, %v", subnet.Name, err) - c.patchSubnetStatus(subnet, "ValidateLogicalSwitchFailed", err.Error()) + if err = c.patchSubnetStatus(subnet, "ValidateLogicalSwitchFailed", err.Error()); err != nil { + klog.Error(err) + return err + } + return err + } + if err = c.patchSubnetStatus(subnet, "ValidateLogicalSwitchSuccess", ""); err != nil { + klog.Error(err) return err } - c.patchSubnetStatus(subnet, "ValidateLogicalSwitchSuccess", "") if err := c.ipam.AddOrUpdateSubnet(subnet.Name, subnet.Spec.CIDRBlock, subnet.Spec.Gateway, subnet.Spec.ExcludeIps); err != nil { klog.Error(err) @@ -768,7 +786,10 @@ func (c *Controller) handleAddOrUpdateSubnet(key string) error { if !isOvnSubnet(subnet) { // subnet provider is not ovn, and vpc is empty, should not reconcile - c.patchSubnetStatus(subnet, "SetNonOvnSubnetSuccess", "") + if err = c.patchSubnetStatus(subnet, "SetNonOvnSubnetSuccess", ""); err != nil { + klog.Error(err) + return err + } subnet.Status.EnsureStandardConditions() klog.Infof("non ovn subnet %s is ready", subnet.Name) @@ -848,7 +869,10 @@ func (c *Controller) handleAddOrUpdateSubnet(key string) error { } if subnet.Spec.EnableLb != nil && *subnet.Spec.EnableLb { if err := c.OVNNbClient.LogicalSwitchUpdateLoadBalancers(subnet.Name, ovsdb.MutateOperationInsert, lbs...); err != nil { - c.patchSubnetStatus(subnet, "AddLbToLogicalSwitchFailed", err.Error()) + if err = c.patchSubnetStatus(subnet, "AddLbToLogicalSwitchFailed", err.Error()); err != nil { + klog.Error(err) + return err + } klog.Error(err) return err } @@ -877,25 +901,40 @@ func (c *Controller) handleAddOrUpdateSubnet(key string) error { if subnet.Spec.Private { if err := c.OVNNbClient.SetLogicalSwitchPrivate(subnet.Name, subnet.Spec.CIDRBlock, c.config.NodeSwitchCIDR, subnet.Spec.AllowSubnets); err != nil { - c.patchSubnetStatus(subnet, "SetPrivateLogicalSwitchFailed", err.Error()) + if err = c.patchSubnetStatus(subnet, "SetPrivateLogicalSwitchFailed", err.Error()); err != nil { + klog.Error(err) + return err + } klog.Error(err) return err } - c.patchSubnetStatus(subnet, "SetPrivateLogicalSwitchSuccess", "") + if err = c.patchSubnetStatus(subnet, "SetPrivateLogicalSwitchSuccess", ""); err != nil { + klog.Error(err) + return err + } } else { // clear acl when direction is "" if err = c.OVNNbClient.DeleteAcls(subnet.Name, logicalSwitchKey, "", nil); err != nil { - c.patchSubnetStatus(subnet, "ResetLogicalSwitchAclFailed", err.Error()) + if err = c.patchSubnetStatus(subnet, "ResetLogicalSwitchAclFailed", err.Error()); err != nil { + klog.Error(err) + return err + } klog.Error(err) return err } - c.patchSubnetStatus(subnet, "ResetLogicalSwitchAclSuccess", "") + if err = c.patchSubnetStatus(subnet, "ResetLogicalSwitchAclSuccess", ""); err != nil { + klog.Error(err) + return err + } } if err := c.OVNNbClient.UpdateLogicalSwitchACL(subnet.Name, subnet.Spec.CIDRBlock, subnet.Spec.Acls, subnet.Spec.AllowEWTraffic); err != nil { - c.patchSubnetStatus(subnet, "SetLogicalSwitchAclsFailed", err.Error()) + if err = c.patchSubnetStatus(subnet, "SetLogicalSwitchAclsFailed", err.Error()); err != nil { + klog.Error(err) + return err + } klog.Error(err) return err } @@ -1441,7 +1480,10 @@ func (c *Controller) reconcileDistributedSubnetRouteInDefaultVpc(subnet *kubeovn return err } subnet.Status.ActivateGateway = "" - c.patchSubnetStatus(subnet, "ChangeToDistributedGw", "") + if err := c.patchSubnetStatus(subnet, "ChangeToDistributedGw", ""); err != nil { + klog.Error(err) + return err + } } nodes, err := c.nodesLister.List(labels.Everything()) @@ -1592,7 +1634,10 @@ func (c *Controller) reconcileDefaultCentralizedSubnetRouteInDefaultVpc(subnet * if newActivateNode == "" { klog.Warningf("all gateways of subnet %s are not ready", subnet.Name) subnet.Status.ActivateGateway = newActivateNode - c.patchSubnetStatus(subnet, "NoActiveGatewayFound", fmt.Sprintf("subnet %s gws are not ready", subnet.Name)) + if err := c.patchSubnetStatus(subnet, "NoActiveGatewayFound", fmt.Sprintf("subnet %s gws are not ready", subnet.Name)); err != nil { + klog.Error(err) + return err + } return fmt.Errorf("subnet %s gws are not ready", subnet.Name) } @@ -1604,7 +1649,10 @@ func (c *Controller) reconcileDefaultCentralizedSubnetRouteInDefaultVpc(subnet * return err } subnet.Status.ActivateGateway = newActivateNode - c.patchSubnetStatus(subnet, "ReconcileCentralizedGatewaySuccess", "") + if err := c.patchSubnetStatus(subnet, "ReconcileCentralizedGatewaySuccess", ""); err != nil { + klog.Error(err) + return err + } klog.Infof("delete old distributed policy route for subnet %s", subnet.Name) if err := c.deletePolicyRouteByGatewayType(subnet, kubeovnv1.GWDistributedType, false); err != nil { @@ -1774,8 +1822,10 @@ func (c *Controller) reconcileOvnDefaultVpcRoute(subnet *kubeovnv1.Subnet) error // centralized subnet if subnet.Spec.GatewayNode == "" { subnet.Status.NotReady("NoReadyGateway", "") - c.patchSubnetStatus(subnet, "NoReadyGateway", "") - + if err := c.patchSubnetStatus(subnet, "NoReadyGateway", ""); err != nil { + klog.Error(err) + return err + } err := fmt.Errorf("subnet %s Spec.GatewayNode field must be specified for centralized gateway type", subnet.Name) klog.Error(err) return err @@ -1949,6 +1999,7 @@ func (c *Controller) reconcileU2OInterconnectionIP(subnet *kubeovnv1.Subnet) err klog.Infof("release underlay to overlay interconnection ip address %s for subnet %s", subnet.Status.U2OInterconnectionIP, subnet.Name) c.ipam.ReleaseAddressByPod(u2oInterconnName, subnet.Name) subnet.Status.U2OInterconnectionIP = "" + subnet.Status.U2OInterconnectionVPC = "" if err := c.config.KubeOvnClient.KubeovnV1().IPs().Delete(context.Background(), u2oInterconnName, metav1.DeleteOptions{}); err != nil { if !k8serrors.IsNotFound(err) { diff --git a/pkg/webhook/ip.go b/pkg/webhook/ip.go index c08d6a25656..82ed2d68928 100644 --- a/pkg/webhook/ip.go +++ b/pkg/webhook/ip.go @@ -61,6 +61,22 @@ func (v *ValidatingHook) IPUpdateHook(ctx context.Context, req admission.Request err := fmt.Errorf("ip %s podType can not change", ipNew.Name) return ctrlwebhook.Errored(http.StatusBadRequest, err) } + if ipOld.Spec.V4IPAddress != "" && ipNew.Spec.V4IPAddress != ipOld.Spec.V4IPAddress { + err := fmt.Errorf("ip %s v4IPAddress can not change", ipNew.Name) + return ctrlwebhook.Errored(http.StatusBadRequest, err) + } + if ipOld.Spec.V6IPAddress != "" && ipNew.Spec.V6IPAddress != ipOld.Spec.V6IPAddress { + err := fmt.Errorf("ip %s v6IPAddress can not change", ipNew.Name) + return ctrlwebhook.Errored(http.StatusBadRequest, err) + } + if ipOld.Spec.MacAddress != "" && ipNew.Spec.MacAddress != ipOld.Spec.MacAddress { + err := fmt.Errorf("ip %s macAddress can not change", ipNew.Name) + return ctrlwebhook.Errored(http.StatusBadRequest, err) + } + if ipOld.Spec.NodeName != "" && ipNew.Spec.NodeName != ipOld.Spec.NodeName { + err := fmt.Errorf("ip %s nodeName can not change", ipNew.Name) + return ctrlwebhook.Errored(http.StatusBadRequest, err) + } return ctrlwebhook.Allowed("by pass") } diff --git a/test/e2e/framework/image.go b/test/e2e/framework/image.go index 1089d909a5d..e2fd91f78a1 100644 --- a/test/e2e/framework/image.go +++ b/test/e2e/framework/image.go @@ -1,6 +1,6 @@ package framework const ( - PauseImage = "kubeovn/pause:3.7" + PauseImage = "kubeovn/pause:3.9" AgnhostImage = "kubeovn/agnhost:2.47" ) diff --git a/test/e2e/kube-ovn/underlay/underlay.go b/test/e2e/kube-ovn/underlay/underlay.go index a323091e6f5..8538b579b33 100644 --- a/test/e2e/kube-ovn/underlay/underlay.go +++ b/test/e2e/kube-ovn/underlay/underlay.go @@ -63,6 +63,29 @@ func waitSubnetStatusUpdate(subnetName string, subnetClient *framework.SubnetCli }, "") } +func waitSubnetU2OStatus(subnetName string, subnetClient *framework.SubnetClient, enableU2O bool) { + framework.WaitUntil(1*time.Second, 3*time.Second, func(_ context.Context) (bool, error) { + ginkgo.By("Waiting for U2OInterconnection status of subnet " + subnetName + " to be " + strconv.FormatBool(enableU2O)) + subnet := subnetClient.Get(subnetName) + if enableU2O { + if subnet.Status.U2OInterconnectionIP != "" && subnet.Status.U2OInterconnectionVPC != "" { + framework.Logf("current enable U2O subnet status: U2OInterconnectionIP = %s, U2OInterconnectionVPC = %s", + subnet.Status.U2OInterconnectionIP, subnet.Status.U2OInterconnectionVPC) + return true, nil + } + ginkgo.By("Keep waiting for U2O to be true: current enable U2O subnet status: U2OInterconnectionIP = " + subnet.Status.U2OInterconnectionIP + ", U2OInterconnectionVPC = " + subnet.Status.U2OInterconnectionVPC) + } else { + if subnet.Status.U2OInterconnectionIP == "" && subnet.Status.U2OInterconnectionVPC == "" { + framework.Logf("current disable U2O subnet status: U2OInterconnectionIP = %s, U2OInterconnectionVPC = %s", + subnet.Status.U2OInterconnectionIP, subnet.Status.U2OInterconnectionVPC) + return true, nil + } + ginkgo.By("Keep waiting for U2O to be false: current enable U2O subnet status: U2OInterconnectionIP = " + subnet.Status.U2OInterconnectionIP + ", U2OInterconnectionVPC = " + subnet.Status.U2OInterconnectionVPC) + } + return false, nil + }, "") +} + var _ = framework.SerialDescribe("[group:underlay]", func() { f := framework.NewDefaultFramework("underlay") @@ -542,6 +565,8 @@ var _ = framework.SerialDescribe("[group:underlay]", func() { ginkgo.By("step1: Enable u2o check") subnet = subnetClient.Get(subnetName) + ginkgo.By("1. waiting for U2OInterconnection status of subnet " + subnetName + " to be true") + waitSubnetU2OStatus(subnetName, subnetClient, true) checkU2OItems(f, subnet, underlayPod, overlayPod, false) ginkgo.By("step2: Disable u2o check") @@ -560,6 +585,8 @@ var _ = framework.SerialDescribe("[group:underlay]", func() { waitSubnetStatusUpdate(subnetName, subnetClient, 1) subnet = subnetClient.Get(subnetName) + ginkgo.By("2. waiting for U2OInterconnection status of subnet " + subnetName + " to be false") + waitSubnetU2OStatus(subnetName, subnetClient, false) checkU2OItems(f, subnet, underlayPod, overlayPod, false) ginkgo.By("step3: Recover enable u2o check") @@ -578,6 +605,8 @@ var _ = framework.SerialDescribe("[group:underlay]", func() { waitSubnetStatusUpdate(subnetName, subnetClient, 2) subnet = subnetClient.Get(subnetName) + ginkgo.By("3. waiting for U2OInterconnection status of subnet " + subnetName + " to be true") + waitSubnetU2OStatus(subnetName, subnetClient, true) checkU2OItems(f, subnet, underlayPod, overlayPod, false) ginkgo.By("step4: Check if kube-ovn-controller restart") @@ -588,6 +617,8 @@ var _ = framework.SerialDescribe("[group:underlay]", func() { deployClient.RestartSync(deploy) subnet = subnetClient.Get(subnetName) + ginkgo.By("4. waiting for U2OInterconnection status of subnet " + subnetName + " to be true") + waitSubnetU2OStatus(subnetName, subnetClient, true) checkU2OItems(f, subnet, underlayPod, overlayPod, false) ginkgo.By("step5: Disable u2o check after restart kube-controller") @@ -606,6 +637,8 @@ var _ = framework.SerialDescribe("[group:underlay]", func() { waitSubnetStatusUpdate(subnetName, subnetClient, 1) subnet = subnetClient.Get(subnetName) + ginkgo.By("5. waiting for U2OInterconnection status of subnet " + subnetName + " to be false") + waitSubnetU2OStatus(subnetName, subnetClient, false) checkU2OItems(f, subnet, underlayPod, overlayPod, false) ginkgo.By("step6: Recover enable u2o check after restart kube-ovn-controller") @@ -624,6 +657,8 @@ var _ = framework.SerialDescribe("[group:underlay]", func() { waitSubnetStatusUpdate(subnetName, subnetClient, 2) subnet = subnetClient.Get(subnetName) + ginkgo.By("6. waiting for U2OInterconnection status of subnet " + subnetName + " to be true") + waitSubnetU2OStatus(subnetName, subnetClient, true) checkU2OItems(f, subnet, underlayPod, overlayPod, false) if f.VersionPriorTo(1, 9) { @@ -666,6 +701,8 @@ var _ = framework.SerialDescribe("[group:underlay]", func() { waitSubnetStatusUpdate(subnetName, subnetClient, 2) subnet = subnetClient.Get(subnetName) + ginkgo.By("7. waiting for U2OInterconnection status of subnet " + subnetName + " to be true") + waitSubnetU2OStatus(subnetName, subnetClient, true) checkU2OItems(f, subnet, underlayPod, overlayPod, false) } @@ -707,6 +744,8 @@ var _ = framework.SerialDescribe("[group:underlay]", func() { waitSubnetStatusUpdate(subnetName, subnetClient, 2) subnet = subnetClient.Get(subnetName) + ginkgo.By("8. waiting for U2OInterconnection status of subnet " + subnetName + " to be true") + waitSubnetU2OStatus(subnetName, subnetClient, true) checkU2OItems(f, subnet, underlayPod, podOverlayCustomVPC, true) ginkgo.By("step9: Change underlay subnet interconnection to overlay subnet in default vpc") @@ -726,6 +765,8 @@ var _ = framework.SerialDescribe("[group:underlay]", func() { waitSubnetStatusUpdate(subnetName, subnetClient, 2) subnet = subnetClient.Get(subnetName) + ginkgo.By("9. waiting for U2OInterconnection status of subnet " + subnetName + " to be true") + waitSubnetU2OStatus(subnetName, subnetClient, true) checkU2OItems(f, subnet, underlayPod, overlayPod, false) ginkgo.By("step10: Disable u2o") @@ -744,12 +785,14 @@ var _ = framework.SerialDescribe("[group:underlay]", func() { waitSubnetStatusUpdate(subnetName, subnetClient, 1) subnet = subnetClient.Get(subnetName) + ginkgo.By("10. waiting for U2OInterconnection status of subnet " + subnetName + " to be false") + waitSubnetU2OStatus(subnetName, subnetClient, false) checkU2OItems(f, subnet, underlayPod, overlayPod, false) }) }) func checkU2OItems(f *framework.Framework, subnet *apiv1.Subnet, underlayPod, overlayPod *corev1.Pod, isU2OCustomVpc bool) { - ginkgo.By("checking underlay subnet's u2o interconnect ip") + ginkgo.By("checking subnet's u2o interconnect ip of underlay subnet " + subnet.Name) if subnet.Spec.U2OInterconnection { framework.ExpectTrue(subnet.Spec.U2OInterconnection) framework.ExpectIPInCIDR(subnet.Status.U2OInterconnectionIP, subnet.Spec.CIDRBlock) @@ -779,7 +822,7 @@ func checkU2OItems(f *framework.Framework, subnet *apiv1.Subnet, underlayPod, ov if util.CheckProtocol(cidr) == apiv1.ProtocolIPv4 { protocolStr = "ip4" gw = v4gw - ginkgo.By("checking underlay subnet's using ips") + ginkgo.By("checking subnet's using ips of underlay subnet " + subnet.Name + " " + protocolStr) if subnet.Spec.U2OInterconnection { framework.ExpectEqual(int(subnet.Status.V4UsingIPs), 2) } else { @@ -788,6 +831,7 @@ func checkU2OItems(f *framework.Framework, subnet *apiv1.Subnet, underlayPod, ov } else { protocolStr = "ip6" gw = v6gw + ginkgo.By("checking subnet's using ips of underlay subnet " + subnet.Name + " " + protocolStr) if subnet.Spec.U2OInterconnection { framework.ExpectEqual(int(subnet.Status.V6UsingIPs), 2) } else { diff --git a/test/e2e/ovn-vpc-nat-gw/e2e_test.go b/test/e2e/ovn-vpc-nat-gw/e2e_test.go index 896269d7e75..4e88b7dcd0d 100644 --- a/test/e2e/ovn-vpc-nat-gw/e2e_test.go +++ b/test/e2e/ovn-vpc-nat-gw/e2e_test.go @@ -94,8 +94,7 @@ var _ = framework.Describe("[group:ovn-vpc-nat-gw]", func() { var ovnSnatRuleClient *framework.OvnSnatRuleClient var ovnDnatRuleClient *framework.OvnDnatRuleClient var podClient *framework.PodClient - - var lrpEipSnatName, lrpExtraEipSnatName string + var countingEipName, lrpEipSnatName, lrpExtraEipSnatName string var fipName string var ipDnatVipName, ipDnatEipName, ipDnatName string var ipFipVipName, ipFipEipName, ipFipName string @@ -134,6 +133,7 @@ var _ = framework.Describe("[group:ovn-vpc-nat-gw]", func() { // nats use ip crd name or vip crd fipName = "fip-" + framework.RandomSuffix() + countingEipName = "counting-eip-" + framework.RandomSuffix() noBfdSubnetName = "no-bfd-subnet-" + framework.RandomSuffix() noBfdExtraSubnetName = "no-bfd-extra-subnet-" + framework.RandomSuffix() lrpEipSnatName = "lrp-eip-snat-" + framework.RandomSuffix() @@ -501,11 +501,42 @@ var _ = framework.Describe("[group:ovn-vpc-nat-gw]", func() { vlanSubnetCidr := strings.Join(cidr, ",") vlanSubnetGw := strings.Join(gateway, ",") underlaySubnet := framework.MakeSubnet(underlaySubnetName, vlanName, vlanSubnetCidr, vlanSubnetGw, "", "", excludeIPs, nil, nil) - _ = subnetClient.CreateSync(underlaySubnet) - vlanSubnet := subnetClient.Get(underlaySubnetName) - ginkgo.By("Checking underlay vlan " + vlanSubnet.Name) - framework.ExpectEqual(vlanSubnet.Spec.Vlan, vlanName) - framework.ExpectNotEqual(vlanSubnet.Spec.CIDRBlock, "") + oldUnderlayExternalSubnet := subnetClient.CreateSync(underlaySubnet) + countingEip := makeOvnEip(countingEipName, underlaySubnetName, "", "", "", "") + _ = ovnEipClient.CreateSync(countingEip) + ginkgo.By("Checking underlay vlan " + oldUnderlayExternalSubnet.Name) + framework.ExpectEqual(oldUnderlayExternalSubnet.Spec.Vlan, vlanName) + framework.ExpectNotEqual(oldUnderlayExternalSubnet.Spec.CIDRBlock, "") + time.Sleep(3 * time.Second) + newUnerlayExternalSubnet := subnetClient.Get(underlaySubnetName) + ginkgo.By("Check status using ovn eip for subnet " + underlaySubnetName) + if newUnerlayExternalSubnet.Spec.Protocol == kubeovnv1.ProtocolIPv4 { + framework.ExpectEqual(oldUnderlayExternalSubnet.Status.V4AvailableIPs-1, newUnerlayExternalSubnet.Status.V4AvailableIPs) + framework.ExpectEqual(oldUnderlayExternalSubnet.Status.V4UsingIPs+1, newUnerlayExternalSubnet.Status.V4UsingIPs) + framework.ExpectNotEqual(oldUnderlayExternalSubnet.Status.V4AvailableIPRange, newUnerlayExternalSubnet.Status.V4AvailableIPRange) + framework.ExpectNotEqual(oldUnderlayExternalSubnet.Status.V4UsingIPRange, newUnerlayExternalSubnet.Status.V4UsingIPRange) + } else { + framework.ExpectEqual(oldUnderlayExternalSubnet.Status.V6AvailableIPs-1, newUnerlayExternalSubnet.Status.V6AvailableIPs) + framework.ExpectEqual(oldUnderlayExternalSubnet.Status.V6UsingIPs+1, newUnerlayExternalSubnet.Status.V6UsingIPs) + framework.ExpectNotEqual(oldUnderlayExternalSubnet.Status.V6AvailableIPRange, newUnerlayExternalSubnet.Status.V6AvailableIPRange) + framework.ExpectNotEqual(oldUnderlayExternalSubnet.Status.V6UsingIPRange, newUnerlayExternalSubnet.Status.V6UsingIPRange) + } + // delete counting eip + oldUnderlayExternalSubnet = newUnerlayExternalSubnet + ovnEipClient.DeleteSync(countingEipName) + time.Sleep(3 * time.Second) + newUnerlayExternalSubnet = subnetClient.Get(underlaySubnetName) + if newUnerlayExternalSubnet.Spec.Protocol == kubeovnv1.ProtocolIPv4 { + framework.ExpectEqual(oldUnderlayExternalSubnet.Status.V4AvailableIPs+1, newUnerlayExternalSubnet.Status.V4AvailableIPs) + framework.ExpectEqual(oldUnderlayExternalSubnet.Status.V4UsingIPs-1, newUnerlayExternalSubnet.Status.V4UsingIPs) + framework.ExpectNotEqual(oldUnderlayExternalSubnet.Status.V4AvailableIPRange, newUnerlayExternalSubnet.Status.V4AvailableIPRange) + framework.ExpectNotEqual(oldUnderlayExternalSubnet.Status.V4UsingIPRange, newUnerlayExternalSubnet.Status.V4UsingIPRange) + } else { + framework.ExpectEqual(oldUnderlayExternalSubnet.Status.V6AvailableIPs+1, newUnerlayExternalSubnet.Status.V6AvailableIPs) + framework.ExpectEqual(oldUnderlayExternalSubnet.Status.V6UsingIPs-1, newUnerlayExternalSubnet.Status.V6UsingIPs) + framework.ExpectNotEqual(oldUnderlayExternalSubnet.Status.V6AvailableIPRange, newUnerlayExternalSubnet.Status.V6AvailableIPRange) + framework.ExpectNotEqual(oldUnderlayExternalSubnet.Status.V6UsingIPRange, newUnerlayExternalSubnet.Status.V6UsingIPRange) + } externalGwNodes := strings.Join(gwNodeNames, ",") ginkgo.By("Creating config map ovn-external-gw-config for centralized case") diff --git a/test/e2e/vip/e2e_test.go b/test/e2e/vip/e2e_test.go index 717e70dd28b..c22f1668bfd 100644 --- a/test/e2e/vip/e2e_test.go +++ b/test/e2e/vip/e2e_test.go @@ -8,6 +8,7 @@ import ( "sort" "strings" "testing" + "time" clientset "k8s.io/client-go/kubernetes" "k8s.io/klog/v2" @@ -121,7 +122,7 @@ var _ = framework.Describe("[group:vip]", func() { var switchLbVip1Name, switchLbVip2Name string // test allowed address pair vip - var vip1Name, vip2Name, aapPodName1, aapPodName2, aapPodName3 string + var countingVipName, vip1Name, vip2Name, aapPodName1, aapPodName2, aapPodName3 string // test allowed address pair connectivity in the security group scenario var securityGroupName string @@ -141,6 +142,9 @@ var _ = framework.Describe("[group:vip]", func() { switchLbVip1Name = "switch-lb-vip1-" + randomSuffix switchLbVip2Name = "switch-lb-vip2-" + randomSuffix + // subnet status counting vip + countingVipName = "counting-vip-" + randomSuffix + // should have different mac vip1Name = "vip1-" + randomSuffix vip2Name = "vip2-" + randomSuffix @@ -193,13 +197,46 @@ var _ = framework.Describe("[group:vip]", func() { framework.ConformanceIt("Test vip", func() { f.SkipVersionPriorTo(1, 13, "This feature was introduced in v1.13") - + ginkgo.By("0. Test subnet status counting vip") + oldSubnet := subnetClient.Get(subnetName) + countingVip := makeOvnVip(namespaceName, countingVipName, subnetName, "", "", "") + _ = vipClient.CreateSync(countingVip) + time.Sleep(3 * time.Second) + newSubnet := subnetClient.Get(subnetName) + if newSubnet.Spec.Protocol == apiv1.ProtocolIPv4 { + framework.ExpectEqual(oldSubnet.Status.V4AvailableIPs-1, newSubnet.Status.V4AvailableIPs) + framework.ExpectEqual(oldSubnet.Status.V4UsingIPs+1, newSubnet.Status.V4UsingIPs) + framework.ExpectNotEqual(oldSubnet.Status.V4AvailableIPRange, newSubnet.Status.V4AvailableIPRange) + framework.ExpectNotEqual(oldSubnet.Status.V4UsingIPRange, newSubnet.Status.V4UsingIPRange) + } else { + framework.ExpectEqual(oldSubnet.Status.V6AvailableIPs-1, newSubnet.Status.V6AvailableIPs) + framework.ExpectEqual(oldSubnet.Status.V6UsingIPs+1, newSubnet.Status.V6UsingIPs) + framework.ExpectNotEqual(oldSubnet.Status.V6AvailableIPRange, newSubnet.Status.V6AvailableIPRange) + framework.ExpectNotEqual(oldSubnet.Status.V6UsingIPRange, newSubnet.Status.V6UsingIPRange) + } + oldSubnet = newSubnet + // delete counting vip + vipClient.DeleteSync(countingVipName) + time.Sleep(3 * time.Second) + newSubnet = subnetClient.Get(subnetName) + if newSubnet.Spec.Protocol == apiv1.ProtocolIPv4 { + framework.ExpectEqual(oldSubnet.Status.V4AvailableIPs+1, newSubnet.Status.V4AvailableIPs) + framework.ExpectEqual(oldSubnet.Status.V4UsingIPs-1, newSubnet.Status.V4UsingIPs) + framework.ExpectNotEqual(oldSubnet.Status.V4AvailableIPRange, newSubnet.Status.V4AvailableIPRange) + framework.ExpectNotEqual(oldSubnet.Status.V4UsingIPRange, newSubnet.Status.V4UsingIPRange) + } else { + framework.ExpectEqual(oldSubnet.Status.V6AvailableIPs+1, newSubnet.Status.V6AvailableIPs) + framework.ExpectEqual(oldSubnet.Status.V6UsingIPs-1, newSubnet.Status.V6UsingIPs) + framework.ExpectNotEqual(oldSubnet.Status.V6AvailableIPRange, newSubnet.Status.V6AvailableIPRange) + framework.ExpectNotEqual(oldSubnet.Status.V6UsingIPRange, newSubnet.Status.V6UsingIPRange) + } ginkgo.By("1. Test allowed address pair vip") // create vip1 and vip2, should have different ip and mac ginkgo.By("Creating allowed address pair vip, should have different ip and mac") ginkgo.By("Creating allowed address pair vip " + vip1Name) vip1 := makeOvnVip(namespaceName, vip1Name, subnetName, "", "", "") vip1 = vipClient.CreateSync(vip1) + ginkgo.By("Creating allowed address pair vip " + vip2Name) vip2 := makeOvnVip(namespaceName, vip2Name, subnetName, "", "", "") vip2 = vipClient.CreateSync(vip2)