Merge pull request spidernet-io#3086 from ty-dc/e2e/new-7

Fixed some incorrect labels and added some e2e.

test/doc/coordinator.md

@@ -14,8 +14,8 @@
 | C00010  | auto clean up the dirty rules(routing\neighborhood) while pod starting | p2 | | |
 | C00011  | In the default scenario (Do not specify the NIC where the default route is located in any way) , use 'ip r get 8.8.8.8' to see if default route NIC is `eth0` | p2 |  | done | |
 | C00012  | In multi-nic case , use 'ip r get <service_subnet> and <hostIP>' to see if src is from pod's eth0, note: only for ipv4. | p2 | | done | |
-| C00013 | Support `spec.externalTrafficPolicy` for service in Local mode, it works well | p2 | | | |
+| C00013  | Support `spec.externalTrafficPolicy` for service in Local mode, it works well | p2 | | | |
 | C00014  | Specify the NIC of the default route, but the NIC does not exist   | p3     |    |    |       |
-| C00015 | In multi-NIC mode, whether the NIC name is random and pods are created normally | p3 | | | |
+| C00015  | In multi-NIC mode, whether the NIC name is random and pods are created normally | p3 | | | |
 | C00016  | The table name can be customized by hostRuleTable               | p3     |    |    |       |
 | C00017  | TunePodRoutes If false, no routing will be coordinated          | p3     |    |    |       |

test/doc/spidermultus.md

@@ -7,18 +7,18 @@
 | M00003  | testing creating spiderMultusConfig with cniType: sriov and checking the net-attach-conf config if works                                                | p1       | smoke | done   |       |
 | M00004  | testing creating spiderMultusConfig with cniType: custom and checking the net-attach-conf config if works                                               | p1       | smoke | done   |       |
 | M00005  | testing creating spiderMultusConfig with cniType: custom and invalid json config, expect error happened                                                 | p2       |       | done   |       |
-| M00007  | testing creating spiderMultusConfig with cniType: macvlan with vlanId with two master with bond config and checking the net-attach-conf config if works | p1       | smoke | done   |       |
-| M00011  | After deleting spiderMultusConfig, the corresponding net-attach-conf will also be deleted                                                               | p2       |       | done   |       |
-| M00013  | Update spidermultusConfig: add new bond config                                                                                                          | p1       | smoke | done   |       |
-| M00014  | Manually delete the net-attach-conf of multus, it will be created automatically                                                                         | p1       |       | done   |       |
-| M00015  | Customize net-attach-conf name via annotation multus.spidernet.io/cr-name                                                                               | p2       |       | done   |       |
-| M00016  | webhook validation for multus.spidernet.io/cr-name                                                                                                      | p3       |       | done   |       |
-| M00017  | Change net-attach-conf version via annotation multus.spidernet.io/cni-version                                                                           | p2       |       | done   |       |
-| M00018  | webhook validation for multus.spidernet.io/cni-version                                                                                                  | p3       |       | done   |       |
-| M00020  | Already have multus cr, spidermultus should take care of it                                                                                             | p3       |       | done   |       |
-| M00022  | The value of webhook verification cniType is inconsistent with cniConf                                                                                  | p3       |       | done   |       |
-| M00023  | vlan is not in the range of 0-4094 and will not be created                                                                                              | p3       |       | done   |       |
-| M00024  | set disableIPAM to true and see if multus's nad has ipam config                                                                                         | p3       |       | done   |       |
-| M00025  | set sriov.enableRdma to true and see if multus's nad has rdma config                                                                                    | p3       |       | done   |       |
-| M00026  | set spidermultusconfig.spec to empty and see if works                                                                                                   | p3       |       | done   |       |
-| M00027  | annotating custom names that are too long or empty should fail                                                                                          | p2       |       | done   |       |                                                                                                | p3       |       |        |       |
+| M00006  | testing creating spiderMultusConfig with cniType: macvlan with vlanId with two master with bond config and checking the net-attach-conf config if works | p1       | smoke | done   |       |
+| M00007  | Manually delete the net-attach-conf of multus, it will be created automatically                                                                         | p1       | smoke | done   |       
+| M00008  | After deleting spiderMultusConfig, the corresponding net-attach-conf will also be deleted                                                               | p2       |       | done   |       |
+| M00009  | Update spidermultusConfig: add new bond config                                                                                                          | p1       | smoke | done   |       |
+| M00010  | Customize net-attach-conf name via annotation multus.spidernet.io/cr-name                                                                               | p2       |       | done   |       |
+| M00011  | webhook validation for multus.spidernet.io/cr-name                                                                                                      | p3       |       | done   |       |
+| M00012  | Change net-attach-conf version via annotation multus.spidernet.io/cni-version                                                                           | p2       |       | done   |       |
+| M00013  | webhook validation for multus.spidernet.io/cni-version                                                                                                  | p3       |       | done   |       |
+| M00014  | Already have multus cr, spidermultus should take care of it                                                                                             | p3       |       | done   |       |
+| M00015  | The value of webhook verification cniType is inconsistent with cniConf                                                                                  | p3       |       | done   |       |
+| M00016  | vlan is not in the range of 0-4094 and will not be created                                                                                              | p3       |       | done   |       |
+| M00017  | set disableIPAM to true and see if multus's nad has ipam config                                                                                         | p3       |       | done   |       |
+| M00018  | set sriov.enableRdma to true and see if multus's nad has rdma config                                                                                    | p3       |       | done   |       |
+| M00019  | set spidermultusconfig.spec to empty and see if works                                                                                                   | p3       |       | done   |       |
+| M00020  | annotating custom names that are too long or empty should fail                                                                                          | p3       |       | done   |       |

test/e2e/annotation/annotation_test.go

@@ -15,6 +15,7 @@ import (
 	"github.com/spidernet-io/e2eframework/tools"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/utils/pointer"
 
 	pkgconstant "github.com/spidernet-io/spiderpool/pkg/constant"
 	spiderpool "github.com/spidernet-io/spiderpool/pkg/k8s/apis/spiderpool.spidernet.io/v2beta1"
@@ -207,12 +208,14 @@ var _ = Describe("test annotation", Label("annotation"), func() {
 				}]`),
 	)
 
-	It("it fails to run a pod with different VLAN for ipv4 and ipv6 ippool", Pending, func() {
+	It("it fails to run a pod with different VLAN for ipv4 and ipv6 ippool", Pending, Label("A00001", "Deprecated"), func() {
 		var (
 			v4PoolName, v6PoolName   string
 			iPv4PoolObj, iPv6PoolObj *spiderpool.SpiderIPPool
 			err                      error
-			ipNum                    int = 2
+			ipNum                    int   = 2
+			ipv4Vlan                 int64 = 10
+			ipv6Vlan                 int64 = 20
 		)
 
 		// The case relies on a Dual-stack
@@ -223,6 +226,7 @@ var _ = Describe("test annotation", Label("annotation"), func() {
 		// Create IPv4Pool and IPv6Pool
 		Eventually(func() error {
 			v4PoolName, iPv4PoolObj = common.GenerateExampleIpv4poolObject(ipNum)
+			iPv4PoolObj.Spec.Vlan = pointer.Int64(ipv4Vlan)
 			GinkgoWriter.Printf("try to create ipv4pool: %v \n", v4PoolName)
 			if frame.Info.SpiderSubnetEnabled {
 				ctx, cancel := context.WithTimeout(context.Background(), common.PodStartTimeout)
@@ -237,6 +241,7 @@ var _ = Describe("test annotation", Label("annotation"), func() {
 			}
 
 			v6PoolName, iPv6PoolObj = common.GenerateExampleIpv6poolObject(ipNum)
+			iPv6PoolObj.Spec.Vlan = pointer.Int64(ipv6Vlan)
 			GinkgoWriter.Printf("try to create ipv6pool: %v \n", v6PoolName)
 			if frame.Info.SpiderSubnetEnabled {
 				ctx, cancel := context.WithTimeout(context.Background(), common.PodStartTimeout)

test/e2e/common/constant.go

@@ -90,6 +90,7 @@ var (
 	NIC2 string = "net1"
 	NIC3 string = "eth0.100"
 	NIC4 string = "eth0.200"
+	NIC5 string = "eth1"
 
 	// Spidercoodinator podCIDRType
 	PodCIDRTypeAuto    = "auto"

test/e2e/coordinator/macvlan-overlay-one/macvlan_overlay_one_test.go

@@ -247,7 +247,7 @@ var _ = Describe("MacvlanOverlayOne", Label("overlay", "one-nic", "coordinator")
 			})
 		})
 
-		It("the prefix of the pod mac address should be overridden and the default route should be on the specified NIC", Label("C00007", "C00005"), func() {
+		It("the prefix of the pod mac address should be overridden and the default route should be on the specified NIC", Label("C00007", "C00005", "C00008"), func() {
 			podIppoolsAnno := types.AnnoPodIPPoolsValue{
 				types.AnnoIPPoolItem{
 					NIC: common.NIC2,
@@ -283,7 +283,7 @@ var _ = Describe("MacvlanOverlayOne", Label("overlay", "one-nic", "coordinator")
 			data, err := frame.ExecCommandInPod(podList.Items[0].Name, podList.Items[0].Namespace, commandString, ctx)
 			Expect(err).NotTo(HaveOccurred(), "failed to execute command, error is: %v ", err)
 
-			// the prefix of the pod mac address should be overridden.
+			// C00008: the prefix of the pod mac address should be overridden.
 			Expect(strings.TrimRight(string(data), "\n")).To(Equal(macPrefix), "macperfix is not covered, %s != %s", string(data), macPrefix)
 
 			// Check the network card where the default route of the pod is located
@@ -384,7 +384,7 @@ var _ = Describe("MacvlanOverlayOne", Label("overlay", "one-nic", "coordinator")
 		// Gateway detection is turned off in the default spidercoodinator:default,
 		// turned on in the new multus configuration and takes effect.
 		// Therefore, verifying spidercoodinator has the lowest priority.
-		It("gateway connection detection", Label("C00008", "V00007"), func() {
+		It("gateway connection detection", Label("V00007", "C00009"), func() {
 			detectGatewayMultusName := "test-gateway-multus-" + common.GenerateString(10, true)
 			detectGateway := true
 

test/e2e/ifacer/ifacer_test.go

@@ -9,6 +9,7 @@ import (
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
+	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/utils/pointer"
 
@@ -19,14 +20,15 @@ import (
 )
 
 var _ = Describe("test ifacer", Label("ifacer"), func() {
-	var namespace, dsName, spiderMultusNadName string
+	var namespace, dsName, spiderMultusNadName, mainInterface string
 	var vlanInterface int
 	var spiderMultusConfig *spiderpoolv2beta1.SpiderMultusConfig
 
 	BeforeEach(func() {
 		dsName = "ds-" + common.GenerateString(10, true)
 		namespace = "ns" + tools.RandomName()
 		spiderMultusNadName = "test-multus-" + common.GenerateString(10, true)
+		mainInterface = common.NIC1
 
 		vlanInterface = 50
 		GinkgoWriter.Println("Generate vlan ID of sub-interface:", vlanInterface)
@@ -62,6 +64,18 @@ var _ = Describe("test ifacer", Label("ifacer"), func() {
 			}
 			GinkgoWriter.Printf("delete namespace %v. \n", namespace)
 			Expect(frame.DeleteNamespace(namespace)).NotTo(HaveOccurred())
+
+			// Delete the subinterface used by the test.
+			ctx, cancel := context.WithTimeout(context.Background(), common.ExecCommandTimeout)
+			defer cancel()
+			delVlanInterfaceString := fmt.Sprintf("ip link del %s.%v ", mainInterface, vlanInterface)
+			Eventually(func() bool {
+				for _, node := range frame.Info.KindNodeList {
+					_, err := frame.DockerExecCommand(ctx, node, delVlanInterfaceString)
+					Expect(err).NotTo(HaveOccurred(), "Failed to execute the delete sub-interface command on the node %s %v", node, err)
+				}
+				return true
+			}, common.ResourceDeleteTimeout, common.ForcedWaitingTime).Should(BeTrue())
 		})
 	})
 
@@ -197,4 +211,128 @@ var _ = Describe("test ifacer", Label("ifacer"), func() {
 		//		return true
 		//	}, common.ResourceDeleteTimeout, common.ForcedWaitingTime).Should(BeTrue())
 	})
+
+	// N00004: Different VLAN interfaces have the same VLAN id, an error is returned
+	// N00005: The master interface is down, setting it up and creating VLAN interface
+	It("Creating a VLAN interface sets the primary interface from down to up while disallowing subinterfaces with the same vlan ID.", Serial, Label("N00004", "N00005"), func() {
+
+		mainInterface = common.NIC5
+		ctx, cancel := context.WithTimeout(context.Background(), common.ExecCommandTimeout)
+		defer cancel()
+		showString := fmt.Sprintf("ip link show %s", mainInterface)
+		for _, node := range frame.Info.KindNodeList {
+			out, err := frame.DockerExecCommand(ctx, node, showString)
+			if err != nil {
+				Skip(fmt.Sprintf("Node does not have additional NIC '%s', result %v, ignore this It", mainInterface, string(out)))
+			}
+		}
+
+		Expect(frame.CreateSpiderMultusInstance(spiderMultusConfig)).NotTo(HaveOccurred())
+		GinkgoWriter.Printf("Create spidermultus config %v/%v \n", namespace, spiderMultusNadName)
+
+		GinkgoWriter.Println("The master interface is down, setting it up and creating VLAN interface")
+		ctx, cancel = context.WithTimeout(context.Background(), common.ExecCommandTimeout)
+		defer cancel()
+		setDownString := fmt.Sprintf("ip link set %s down", mainInterface)
+		Eventually(func() bool {
+			for _, node := range frame.Info.KindNodeList {
+				out, err := frame.DockerExecCommand(ctx, node, setDownString)
+				Expect(err).NotTo(HaveOccurred(), "Executing the set sub-interface to down command on the node %s fails, error: %v, log: %v", node, err, string(out))
+			}
+			return true
+		}, common.ResourceDeleteTimeout, common.ForcedWaitingTime).Should(BeTrue())
+
+		sameVlanInterface := 50
+		GinkgoWriter.Println("Generate vlan ID of sub-interface:", sameVlanInterface)
+		newSpiderMultusNadName := "new-test-multus-" + common.GenerateString(10, true)
+		spiderMultusConfig = &spiderpoolv2beta1.SpiderMultusConfig{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      newSpiderMultusNadName,
+				Namespace: namespace,
+			},
+			Spec: spiderpoolv2beta1.MultusCNIConfigSpec{
+				CniType: pointer.String(constant.MacvlanCNI),
+				MacvlanConfig: &spiderpoolv2beta1.SpiderMacvlanCniConfig{
+					Master: []string{mainInterface},
+					VlanID: pointer.Int32(int32(sameVlanInterface)),
+					SpiderpoolConfigPools: &spiderpoolv2beta1.SpiderpoolPools{
+						IPv4IPPool: []string{common.SpiderPoolIPv4PoolDefault},
+						IPv6IPPool: []string{common.SpiderPoolIPv6PoolDefault},
+					},
+				},
+			},
+		}
+		GinkgoWriter.Printf("Generate spiderMultusConfig %v \n", spiderMultusConfig)
+		Expect(frame.CreateSpiderMultusInstance(spiderMultusConfig)).NotTo(HaveOccurred())
+		GinkgoWriter.Printf("Create spidermultus config %v/%v \n", namespace, spiderMultusNadName)
+
+		dsName = "ds-1-" + common.GenerateString(10, true)
+		// Generate Deployment yaml and annotation
+		dsObject := common.GenerateExampleDaemonSetYaml(dsName, namespace)
+		dsObject.Spec.Template.Annotations = map[string]string{common.MultusNetworks: fmt.Sprintf("%s/%s", namespace, newSpiderMultusNadName)}
+		GinkgoWriter.Printf("Try to create Daemonset: %v/%v \n", namespace, dsName)
+		Expect(frame.CreateDaemonSet(dsObject)).NotTo(HaveOccurred())
+
+		ctx, cancel = context.WithTimeout(context.Background(), common.ExecCommandTimeout)
+		defer cancel()
+		err := frame.WaitPodListRunning(dsObject.Spec.Template.Labels, 2, ctx)
+		Expect(err).NotTo(HaveOccurred())
+
+		GinkgoWriter.Println("Check that each node where the Pod is located should have a vlan sub-interface.")
+		checkMasterUPString := fmt.Sprintf("ip link show up %s ", mainInterface)
+		checkIPLinkString := fmt.Sprintf("ip link show up %s.%d ", mainInterface, vlanInterface)
+		Eventually(func() bool {
+			for _, node := range frame.Info.KindNodeList {
+				showMasterResult, err := frame.DockerExecCommand(ctx, node, checkMasterUPString)
+				if err != nil {
+					GinkgoWriter.Printf("Failed to execute command %s on the node %s : %v \n", checkMasterUPString, node, showMasterResult)
+					return false
+				}
+
+				if string(showMasterResult) == "" {
+					GinkgoWriter.Printf("master interface %s is down, waiting \n", mainInterface)
+					return false
+				}
+
+				showResult, err := frame.DockerExecCommand(ctx, node, checkIPLinkString)
+				if err != nil {
+					GinkgoWriter.Printf("Failed to execute %s on the node %s: %v \n", checkIPLinkString, node, showResult)
+					return false
+				}
+
+				if string(showResult) == "" {
+					GinkgoWriter.Printf("vlan interface %s is down, waiting... \n", vlanInterface)
+					return false
+				}
+
+			}
+			return true
+		}, common.ResourceDeleteTimeout, common.ForcedWaitingTime).Should(BeTrue())
+
+		// Generate Deployment yaml and annotation
+		dsName = "ds-2-" + common.GenerateString(10, true)
+		dsObject = common.GenerateExampleDaemonSetYaml(dsName, namespace)
+		dsObject.Spec.Template.Annotations = map[string]string{common.MultusNetworks: fmt.Sprintf("%s/%s", namespace, spiderMultusNadName)}
+		GinkgoWriter.Printf("Try to create Daemonset: %v/%v \n", namespace, dsName)
+		Expect(frame.CreateDaemonSet(dsObject)).NotTo(HaveOccurred())
+
+		var podList *corev1.PodList
+		// Wait for Pod replicas on all nodes to be pulled up.
+		Eventually(func() bool {
+			podList, err = frame.GetPodListByLabel(dsObject.Spec.Template.Labels)
+			if err != nil {
+				GinkgoWriter.Printf("failed to get pod list by label, error is %v", err)
+				return false
+			}
+			return len(podList.Items) == len(frame.Info.KindNodeList)
+		}, common.PodStartTimeout, common.ForcedWaitingTime).Should(BeTrue())
+
+		sameVlanIdErrorString := fmt.Sprintf("cannot have multiple different vlan interfaces with the same vlanId %v on node at the same time", vlanInterface)
+		for _, pod := range podList.Items {
+			ctx, cancel = context.WithTimeout(context.Background(), common.EventOccurTimeout)
+			defer cancel()
+			err = frame.WaitExceptEventOccurred(ctx, common.OwnerPod, pod.Name, namespace, sameVlanIdErrorString)
+			Expect(err).NotTo(HaveOccurred())
+		}
+	})
 })