Skip to content

Commit

Permalink
fix change vm subnet e2e (#4114)
Browse files Browse the repository at this point in the history 
* fix change vm subnet e2e

---------

Signed-off-by: bobz965 <[email protected]>
  • Loading branch information
@bobz965
bobz965 committed Jun 14, 2024
1 parent f4efa2c commit 3e93324
Show file tree
Hide file tree
Showing 2 changed files with 151 additions and 16 deletions.
51 changes: 36 additions & 15 deletions pkg/controller/pod.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1010,15 +1010,25 @@ func (c *Controller) handleDeletePod(key string) error {

var keepIPCR bool
if ok, sts := isStatefulSetPod(pod); ok {
toDel := isStatefulSetPodToDel(c.config.KubeClient, pod, sts)
isDelete, err := appendCheckPodToDel(c, pod, sts, util.StatefulSet)
if !pod.DeletionTimestamp.IsZero() {
// triggered by delete event
if !(toDel || (isDelete && err == nil)) {
return nil
if pod.DeletionTimestamp != nil {
klog.Infof("handle deletion of sts pod %s", podName)
toDel := isStatefulSetPodToDel(c.config.KubeClient, pod, sts)
if !toDel {
klog.Infof("try keep ip for sts pod %s", podKey)
keepIPCR = true
}
}
if keepIPCR {
isDelete, err := appendCheckPodToDel(c, pod, sts, util.StatefulSet)
if err != nil {
klog.Error(err)
return err
}
if isDelete {
klog.Infof("not keep ip for sts pod %s", podKey)
keepIPCR = false
}
}
keepIPCR = !toDel && !isDelete && err == nil
}
isVMPod, vmName := isVMPod(pod)
if isVMPod && c.config.EnableKeepVMIP {
Expand All @@ -1028,21 +1038,30 @@ func (c *Controller) handleDeletePod(key string) error {
return err
}
for _, port := range ports {
klog.Infof("clean migrate options for vm lsp %s", port.Name)
if err := c.OVNNbClient.CleanLogicalSwitchPortMigrateOptions(port.Name); err != nil {
err = fmt.Errorf("failed to clean migrate options for vm lsp %s, %v", port.Name, err)
klog.Error(err)
return err
}
}
vmToBeDel := c.isVMToDel(pod, vmName)
isDelete, err := appendCheckPodToDel(c, pod, vmName, util.VMInstance)
if !pod.DeletionTimestamp.IsZero() {
// triggered by delete event
if !(vmToBeDel || (isDelete && err == nil)) {
return nil
if pod.DeletionTimestamp != nil {
klog.Infof("handle deletion of vm pod %s", podName)
vmToBeDel := c.isVMToDel(pod, vmName)
if !vmToBeDel {
klog.Infof("try keep ip for vm pod %s", podKey)
keepIPCR = true
}
}
if keepIPCR {
isDelete, err := appendCheckPodToDel(c, pod, vmName, util.VMInstance)
if err != nil {
klog.Error(err)
return err
}
if isDelete {
klog.Infof("not keep ip for vm pod %s", podKey)
keepIPCR = false
}
klog.Infof("delete vm pod %s", podName)
}
}

Expand Down Expand Up @@ -1876,6 +1895,7 @@ func appendCheckPodToDel(c *Controller, pod *v1.Pod, ownerRefName, ownerRefKind
ss, err := c.config.KubeClient.AppsV1().StatefulSets(pod.Namespace).Get(context.Background(), ownerRefName, metav1.GetOptions{})
if err != nil {
if k8serrors.IsNotFound(err) {
klog.Infof("Statefulset %s is not found", ownerRefName)
return true, nil
}
klog.Errorf("failed to get StatefulSet %s, %v", ownerRefName, err)
Expand All @@ -1889,6 +1909,7 @@ func appendCheckPodToDel(c *Controller, pod *v1.Pod, ownerRefName, ownerRefKind
vm, err := c.config.KubevirtClient.VirtualMachine(pod.Namespace).Get(context.Background(), ownerRefName, &metav1.GetOptions{})
if err != nil {
if k8serrors.IsNotFound(err) {
klog.Infof("VirtualMachine %s is not found", ownerRefName)
return true, nil
}
klog.Errorf("failed to get VirtualMachine %s, %v", ownerRefName, err)
Expand Down
116 changes: 115 additions & 1 deletion test/e2e/kubevirt/e2e_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"context"
"flag"
"fmt"
"os/exec"
"testing"
"time"

Expand All @@ -16,6 +17,7 @@ import (

"github.com/onsi/ginkgo/v2"

"github.com/kubeovn/kube-ovn/pkg/ovs"
"github.com/kubeovn/kube-ovn/pkg/util"
"github.com/kubeovn/kube-ovn/test/e2e/framework"
)
Expand Down Expand Up @@ -43,6 +45,7 @@ var _ = framework.Describe("[group:kubevirt]", func() {
var subnetClient *framework.SubnetClient
var podClient *framework.PodClient
var vmClient *framework.VMClient
var ipClient *framework.IPClient
ginkgo.BeforeEach(func() {
f.SkipVersionPriorTo(1, 12, "This feature was introduced in v1.12.")

Expand All @@ -52,6 +55,7 @@ var _ = framework.Describe("[group:kubevirt]", func() {
subnetClient = f.SubnetClient()
podClient = f.PodClientNS(namespaceName)
vmClient = f.VMClientNS(namespaceName)
ipClient = f.IPClient()

ginkgo.By("Creating vm " + vmName)
vm := framework.MakeVM(vmName, image, "small", true)
Expand Down Expand Up @@ -124,9 +128,19 @@ var _ = framework.Describe("[group:kubevirt]", func() {
ginkgo.By("Stopping vm " + vmName)
vmClient.StopSync(vmName)

portName := ovs.PodNameToPortName(vmName, namespaceName, util.OvnProvider)
ginkgo.By("Check ip resource " + portName)
// the ip should exist after vm is stopped
oldVMIP := ipClient.Get(portName)
framework.ExpectNil(oldVMIP.DeletionTimestamp)
ginkgo.By("Starting vm " + vmName)
vmClient.StartSync(vmName)

// new ip name is the same as the old one
ginkgo.By("Check ip resource " + portName)
newVMIP := ipClient.Get(portName)
framework.ExpectEqual(oldVMIP.Spec, newVMIP.Spec)

ginkgo.By("Getting pod of vm " + vmName)
podList, err = podClient.List(context.TODO(), metav1.ListOptions{
LabelSelector: labelSelector,
Expand All @@ -144,7 +158,11 @@ var _ = framework.Describe("[group:kubevirt]", func() {
framework.ExpectEqual(ips, pod.Status.PodIPs)
})

framework.ConformanceIt("should be able to handle subnet change", func() {
framework.ConformanceIt("restart vm should be able to handle subnet change after the namespace has its first subnet", func() {
// create a vm within a namespace, the namespace has no subnet, so the vm use ovn-default subnet
// create a subnet in the namespace later, the vm should use its own subnet
// stop the vm, the vm should delete the vm ip, because of the namespace only has one subnet but not ovn-default
// start the vm, the vm should use the namespace owned subnet
ginkgo.By("Creating subnet " + subnetName)
cidr := framework.RandomCIDR(f.ClusterIPFamily)
subnet := framework.MakeSubnet(subnetName, "", cidr, "", "", "", nil, nil, []string{namespaceName})
Expand All @@ -165,9 +183,22 @@ var _ = framework.Describe("[group:kubevirt]", func() {
framework.ExpectHaveKeyWithValue(pod.Annotations, util.VMAnnotation, vmName)
ips := pod.Status.PodIPs

ginkgo.By("Checking old lsp ExternalIDs contains should contains default subnet " + util.DefaultSubnet)
portName := ovs.PodNameToPortName(vmName, namespaceName, util.OvnProvider)
conditions := fmt.Sprintf("name=%s", portName)
execCmd := "kubectl ko nbctl --format=list --data=bare --no-heading --columns=external_ids find logical-switch-port " + conditions
framework.Logf("exec cmd %s", execCmd)
output, err := exec.Command("bash", "-c", execCmd).CombinedOutput()
framework.ExpectNoError(err)
framework.ExpectContainSubstring(string(output), util.DefaultSubnet)

ginkgo.By("Stopping vm " + vmName)
vmClient.StopSync(vmName)

// the ip is deleted
err = ipClient.WaitToDisappear(portName, 2*time.Second, 2*time.Minute)
framework.ExpectNoError(err)

ginkgo.By("Starting vm " + vmName)
vmClient.StartSync(vmName)

Expand All @@ -186,5 +217,88 @@ var _ = framework.Describe("[group:kubevirt]", func() {

ginkgo.By("Checking whether pod ips are changed")
framework.ExpectNotEqual(ips, pod.Status.PodIPs)

ginkgo.By("Checking new lsp ExternalIDs should contains new subnet " + subnetName)
conditions = fmt.Sprintf("name=%s", portName)
execCmd = "kubectl ko nbctl --format=list --data=bare --no-heading --columns=external_ids find logical-switch-port " + conditions
framework.Logf("exec cmd %s", execCmd)
output, err = exec.Command("bash", "-c", execCmd).CombinedOutput()
framework.ExpectNoError(err)
framework.ExpectContainSubstring(string(output), subnetName)
})

framework.ConformanceIt("restart vm should be able to change vm subnet after deleting the old ip", func() {
// case: test change vm subnet after stop vm and delete old ip
// stop vm, delete the ip.
// create new subnet in the namespace.
// make sure ip changed after vm started
ginkgo.By("Getting pod of vm " + vmName)
labelSelector := fmt.Sprintf("%s=%s", v1.VirtualMachineNameLabel, vmName)
podList, err := podClient.List(context.TODO(), metav1.ListOptions{
LabelSelector: labelSelector,
})
framework.ExpectNoError(err)
framework.ExpectHaveLen(podList.Items, 1)

ginkgo.By("Validating pod annotations")
pod := &podList.Items[0]
framework.ExpectHaveKeyWithValue(pod.Annotations, util.AllocatedAnnotation, "true")
framework.ExpectHaveKeyWithValue(pod.Annotations, util.RoutedAnnotation, "true")
framework.ExpectHaveKeyWithValue(pod.Annotations, util.VMAnnotation, vmName)
ginkgo.By("Stopping vm " + vmName)
vmClient.StopSync(vmName)
portName := ovs.PodNameToPortName(vmName, namespaceName, util.OvnProvider)

ginkgo.By("Checking old lsp ExternalIDs contains should contains default subnet " + util.DefaultSubnet)
conditions := fmt.Sprintf("name=%s", portName)
execCmd := "kubectl ko nbctl --format=list --data=bare --no-heading --columns=external_ids find logical-switch-port " + conditions
framework.Logf("exec cmd %s", execCmd)
output, err := exec.Command("bash", "-c", execCmd).CombinedOutput()
framework.ExpectNoError(err)
framework.ExpectContainSubstring(string(output), util.DefaultSubnet)

// make sure the vm ip is still exist
oldVMIP := ipClient.Get(portName)
framework.ExpectNotEmpty(oldVMIP.Spec.IPAddress)
ipClient.DeleteSync(portName)
// delete old ip to create the same name ip in other subnet

ginkgo.By("Creating subnet " + subnetName)
cidr := framework.RandomCIDR(f.ClusterIPFamily)
subnet := framework.MakeSubnet(subnetName, "", cidr, "", "", "", nil, nil, []string{namespaceName})
subnet = subnetClient.CreateSync(subnet)
ginkgo.By("Updating vm " + vmName + " to use new subnet " + subnet.Name)

// the vm should use the new subnet in the namespace
ginkgo.By("Starting vm " + vmName)
vmClient.StartSync(vmName)
// new ip name is the same as the old one
newVMIP := ipClient.Get(portName)
framework.ExpectNotEmpty(newVMIP.Spec.IPAddress)

ginkgo.By("Getting pod of vm " + vmName)
podList, err = podClient.List(context.TODO(), metav1.ListOptions{
LabelSelector: labelSelector,
})
framework.ExpectNoError(err)
framework.ExpectHaveLen(podList.Items, 1)

ginkgo.By("Validating new pod annotations")
pod = &podList.Items[0]
framework.ExpectHaveKeyWithValue(pod.Annotations, util.AllocatedAnnotation, "true")
framework.ExpectHaveKeyWithValue(pod.Annotations, util.RoutedAnnotation, "true")
framework.ExpectHaveKeyWithValue(pod.Annotations, util.VMAnnotation, vmName)
framework.ExpectHaveKeyWithValue(pod.Annotations, util.LogicalSwitchAnnotation, subnetName)

ginkgo.By("Checking whether pod ips are changed")
framework.ExpectNotEqual(newVMIP.Spec.IPAddress, oldVMIP.Spec.IPAddress)

ginkgo.By("Checking new lsp ExternalIDs should contains new subnet " + subnetName)
conditions = fmt.Sprintf("name=%s", portName)
execCmd = "kubectl ko nbctl --format=list --data=bare --no-heading --columns=external_ids find logical-switch-port " + conditions
framework.Logf("exec cmd %s", execCmd)
output, err = exec.Command("bash", "-c", execCmd).CombinedOutput()
framework.ExpectNoError(err)
framework.ExpectContainSubstring(string(output), subnetName)
})
})

0 comments on commit 3e93324

Please sign in to comment.