From 3e933249eb4a79ae46296c1fe838b01a5ede6c9c Mon Sep 17 00:00:00 2001 From: bobz965 Date: Fri, 14 Jun 2024 22:13:05 +0800 Subject: [PATCH] fix change vm subnet e2e (#4114) * fix change vm subnet e2e --------- Signed-off-by: bobz965 --- pkg/controller/pod.go | 51 ++++++++++----- test/e2e/kubevirt/e2e_test.go | 116 +++++++++++++++++++++++++++++++++- 2 files changed, 151 insertions(+), 16 deletions(-) diff --git a/pkg/controller/pod.go b/pkg/controller/pod.go index f3238486a3c..d5e57399715 100644 --- a/pkg/controller/pod.go +++ b/pkg/controller/pod.go @@ -1010,15 +1010,25 @@ func (c *Controller) handleDeletePod(key string) error { var keepIPCR bool if ok, sts := isStatefulSetPod(pod); ok { - toDel := isStatefulSetPodToDel(c.config.KubeClient, pod, sts) - isDelete, err := appendCheckPodToDel(c, pod, sts, util.StatefulSet) - if !pod.DeletionTimestamp.IsZero() { - // triggered by delete event - if !(toDel || (isDelete && err == nil)) { - return nil + if pod.DeletionTimestamp != nil { + klog.Infof("handle deletion of sts pod %s", podName) + toDel := isStatefulSetPodToDel(c.config.KubeClient, pod, sts) + if !toDel { + klog.Infof("try keep ip for sts pod %s", podKey) + keepIPCR = true + } + } + if keepIPCR { + isDelete, err := appendCheckPodToDel(c, pod, sts, util.StatefulSet) + if err != nil { + klog.Error(err) + return err + } + if isDelete { + klog.Infof("not keep ip for sts pod %s", podKey) + keepIPCR = false } } - keepIPCR = !toDel && !isDelete && err == nil } isVMPod, vmName := isVMPod(pod) if isVMPod && c.config.EnableKeepVMIP { @@ -1028,21 +1038,30 @@ func (c *Controller) handleDeletePod(key string) error { return err } for _, port := range ports { - klog.Infof("clean migrate options for vm lsp %s", port.Name) if err := c.OVNNbClient.CleanLogicalSwitchPortMigrateOptions(port.Name); err != nil { err = fmt.Errorf("failed to clean migrate options for vm lsp %s, %v", port.Name, err) klog.Error(err) return err } } - vmToBeDel := c.isVMToDel(pod, vmName) - isDelete, err := appendCheckPodToDel(c, pod, vmName, util.VMInstance) - if !pod.DeletionTimestamp.IsZero() { - // triggered by delete event - if !(vmToBeDel || (isDelete && err == nil)) { - return nil + if pod.DeletionTimestamp != nil { + klog.Infof("handle deletion of vm pod %s", podName) + vmToBeDel := c.isVMToDel(pod, vmName) + if !vmToBeDel { + klog.Infof("try keep ip for vm pod %s", podKey) + keepIPCR = true + } + } + if keepIPCR { + isDelete, err := appendCheckPodToDel(c, pod, vmName, util.VMInstance) + if err != nil { + klog.Error(err) + return err + } + if isDelete { + klog.Infof("not keep ip for vm pod %s", podKey) + keepIPCR = false } - klog.Infof("delete vm pod %s", podName) } } @@ -1876,6 +1895,7 @@ func appendCheckPodToDel(c *Controller, pod *v1.Pod, ownerRefName, ownerRefKind ss, err := c.config.KubeClient.AppsV1().StatefulSets(pod.Namespace).Get(context.Background(), ownerRefName, metav1.GetOptions{}) if err != nil { if k8serrors.IsNotFound(err) { + klog.Infof("Statefulset %s is not found", ownerRefName) return true, nil } klog.Errorf("failed to get StatefulSet %s, %v", ownerRefName, err) @@ -1889,6 +1909,7 @@ func appendCheckPodToDel(c *Controller, pod *v1.Pod, ownerRefName, ownerRefKind vm, err := c.config.KubevirtClient.VirtualMachine(pod.Namespace).Get(context.Background(), ownerRefName, &metav1.GetOptions{}) if err != nil { if k8serrors.IsNotFound(err) { + klog.Infof("VirtualMachine %s is not found", ownerRefName) return true, nil } klog.Errorf("failed to get VirtualMachine %s, %v", ownerRefName, err) diff --git a/test/e2e/kubevirt/e2e_test.go b/test/e2e/kubevirt/e2e_test.go index abefab2e3ec..f685ce6013e 100644 --- a/test/e2e/kubevirt/e2e_test.go +++ b/test/e2e/kubevirt/e2e_test.go @@ -4,6 +4,7 @@ import ( "context" "flag" "fmt" + "os/exec" "testing" "time" @@ -16,6 +17,7 @@ import ( "github.com/onsi/ginkgo/v2" + "github.com/kubeovn/kube-ovn/pkg/ovs" "github.com/kubeovn/kube-ovn/pkg/util" "github.com/kubeovn/kube-ovn/test/e2e/framework" ) @@ -43,6 +45,7 @@ var _ = framework.Describe("[group:kubevirt]", func() { var subnetClient *framework.SubnetClient var podClient *framework.PodClient var vmClient *framework.VMClient + var ipClient *framework.IPClient ginkgo.BeforeEach(func() { f.SkipVersionPriorTo(1, 12, "This feature was introduced in v1.12.") @@ -52,6 +55,7 @@ var _ = framework.Describe("[group:kubevirt]", func() { subnetClient = f.SubnetClient() podClient = f.PodClientNS(namespaceName) vmClient = f.VMClientNS(namespaceName) + ipClient = f.IPClient() ginkgo.By("Creating vm " + vmName) vm := framework.MakeVM(vmName, image, "small", true) @@ -124,9 +128,19 @@ var _ = framework.Describe("[group:kubevirt]", func() { ginkgo.By("Stopping vm " + vmName) vmClient.StopSync(vmName) + portName := ovs.PodNameToPortName(vmName, namespaceName, util.OvnProvider) + ginkgo.By("Check ip resource " + portName) + // the ip should exist after vm is stopped + oldVMIP := ipClient.Get(portName) + framework.ExpectNil(oldVMIP.DeletionTimestamp) ginkgo.By("Starting vm " + vmName) vmClient.StartSync(vmName) + // new ip name is the same as the old one + ginkgo.By("Check ip resource " + portName) + newVMIP := ipClient.Get(portName) + framework.ExpectEqual(oldVMIP.Spec, newVMIP.Spec) + ginkgo.By("Getting pod of vm " + vmName) podList, err = podClient.List(context.TODO(), metav1.ListOptions{ LabelSelector: labelSelector, @@ -144,7 +158,11 @@ var _ = framework.Describe("[group:kubevirt]", func() { framework.ExpectEqual(ips, pod.Status.PodIPs) }) - framework.ConformanceIt("should be able to handle subnet change", func() { + framework.ConformanceIt("restart vm should be able to handle subnet change after the namespace has its first subnet", func() { + // create a vm within a namespace, the namespace has no subnet, so the vm use ovn-default subnet + // create a subnet in the namespace later, the vm should use its own subnet + // stop the vm, the vm should delete the vm ip, because of the namespace only has one subnet but not ovn-default + // start the vm, the vm should use the namespace owned subnet ginkgo.By("Creating subnet " + subnetName) cidr := framework.RandomCIDR(f.ClusterIPFamily) subnet := framework.MakeSubnet(subnetName, "", cidr, "", "", "", nil, nil, []string{namespaceName}) @@ -165,9 +183,22 @@ var _ = framework.Describe("[group:kubevirt]", func() { framework.ExpectHaveKeyWithValue(pod.Annotations, util.VMAnnotation, vmName) ips := pod.Status.PodIPs + ginkgo.By("Checking old lsp ExternalIDs contains should contains default subnet " + util.DefaultSubnet) + portName := ovs.PodNameToPortName(vmName, namespaceName, util.OvnProvider) + conditions := fmt.Sprintf("name=%s", portName) + execCmd := "kubectl ko nbctl --format=list --data=bare --no-heading --columns=external_ids find logical-switch-port " + conditions + framework.Logf("exec cmd %s", execCmd) + output, err := exec.Command("bash", "-c", execCmd).CombinedOutput() + framework.ExpectNoError(err) + framework.ExpectContainSubstring(string(output), util.DefaultSubnet) + ginkgo.By("Stopping vm " + vmName) vmClient.StopSync(vmName) + // the ip is deleted + err = ipClient.WaitToDisappear(portName, 2*time.Second, 2*time.Minute) + framework.ExpectNoError(err) + ginkgo.By("Starting vm " + vmName) vmClient.StartSync(vmName) @@ -186,5 +217,88 @@ var _ = framework.Describe("[group:kubevirt]", func() { ginkgo.By("Checking whether pod ips are changed") framework.ExpectNotEqual(ips, pod.Status.PodIPs) + + ginkgo.By("Checking new lsp ExternalIDs should contains new subnet " + subnetName) + conditions = fmt.Sprintf("name=%s", portName) + execCmd = "kubectl ko nbctl --format=list --data=bare --no-heading --columns=external_ids find logical-switch-port " + conditions + framework.Logf("exec cmd %s", execCmd) + output, err = exec.Command("bash", "-c", execCmd).CombinedOutput() + framework.ExpectNoError(err) + framework.ExpectContainSubstring(string(output), subnetName) + }) + + framework.ConformanceIt("restart vm should be able to change vm subnet after deleting the old ip", func() { + // case: test change vm subnet after stop vm and delete old ip + // stop vm, delete the ip. + // create new subnet in the namespace. + // make sure ip changed after vm started + ginkgo.By("Getting pod of vm " + vmName) + labelSelector := fmt.Sprintf("%s=%s", v1.VirtualMachineNameLabel, vmName) + podList, err := podClient.List(context.TODO(), metav1.ListOptions{ + LabelSelector: labelSelector, + }) + framework.ExpectNoError(err) + framework.ExpectHaveLen(podList.Items, 1) + + ginkgo.By("Validating pod annotations") + pod := &podList.Items[0] + framework.ExpectHaveKeyWithValue(pod.Annotations, util.AllocatedAnnotation, "true") + framework.ExpectHaveKeyWithValue(pod.Annotations, util.RoutedAnnotation, "true") + framework.ExpectHaveKeyWithValue(pod.Annotations, util.VMAnnotation, vmName) + ginkgo.By("Stopping vm " + vmName) + vmClient.StopSync(vmName) + portName := ovs.PodNameToPortName(vmName, namespaceName, util.OvnProvider) + + ginkgo.By("Checking old lsp ExternalIDs contains should contains default subnet " + util.DefaultSubnet) + conditions := fmt.Sprintf("name=%s", portName) + execCmd := "kubectl ko nbctl --format=list --data=bare --no-heading --columns=external_ids find logical-switch-port " + conditions + framework.Logf("exec cmd %s", execCmd) + output, err := exec.Command("bash", "-c", execCmd).CombinedOutput() + framework.ExpectNoError(err) + framework.ExpectContainSubstring(string(output), util.DefaultSubnet) + + // make sure the vm ip is still exist + oldVMIP := ipClient.Get(portName) + framework.ExpectNotEmpty(oldVMIP.Spec.IPAddress) + ipClient.DeleteSync(portName) + // delete old ip to create the same name ip in other subnet + + ginkgo.By("Creating subnet " + subnetName) + cidr := framework.RandomCIDR(f.ClusterIPFamily) + subnet := framework.MakeSubnet(subnetName, "", cidr, "", "", "", nil, nil, []string{namespaceName}) + subnet = subnetClient.CreateSync(subnet) + ginkgo.By("Updating vm " + vmName + " to use new subnet " + subnet.Name) + + // the vm should use the new subnet in the namespace + ginkgo.By("Starting vm " + vmName) + vmClient.StartSync(vmName) + // new ip name is the same as the old one + newVMIP := ipClient.Get(portName) + framework.ExpectNotEmpty(newVMIP.Spec.IPAddress) + + ginkgo.By("Getting pod of vm " + vmName) + podList, err = podClient.List(context.TODO(), metav1.ListOptions{ + LabelSelector: labelSelector, + }) + framework.ExpectNoError(err) + framework.ExpectHaveLen(podList.Items, 1) + + ginkgo.By("Validating new pod annotations") + pod = &podList.Items[0] + framework.ExpectHaveKeyWithValue(pod.Annotations, util.AllocatedAnnotation, "true") + framework.ExpectHaveKeyWithValue(pod.Annotations, util.RoutedAnnotation, "true") + framework.ExpectHaveKeyWithValue(pod.Annotations, util.VMAnnotation, vmName) + framework.ExpectHaveKeyWithValue(pod.Annotations, util.LogicalSwitchAnnotation, subnetName) + + ginkgo.By("Checking whether pod ips are changed") + framework.ExpectNotEqual(newVMIP.Spec.IPAddress, oldVMIP.Spec.IPAddress) + + ginkgo.By("Checking new lsp ExternalIDs should contains new subnet " + subnetName) + conditions = fmt.Sprintf("name=%s", portName) + execCmd = "kubectl ko nbctl --format=list --data=bare --no-heading --columns=external_ids find logical-switch-port " + conditions + framework.Logf("exec cmd %s", execCmd) + output, err = exec.Command("bash", "-c", execCmd).CombinedOutput() + framework.ExpectNoError(err) + framework.ExpectContainSubstring(string(output), subnetName) }) })