add e2e test cases for nat outgoing routes (#4187)

Signed-off-by: zhangzujian <[email protected]>
kubeovn · Jun 21, 2024 · f63172d · f63172d
1 parent 2942337
commit f63172d
Show file tree

Hide file tree

Showing 16 changed files with 147 additions and 123 deletions.
diff --git a/Makefile.e2e b/Makefile.e2e
@@ -51,7 +51,7 @@ K8S_CONFORMANCE_E2E_SKIP += "sig-network.*Services.*should serve endpoints on sa
 endif
 
 GINKGO_OUTPUT_OPT =
-GINKGO_PARALLEL_OPT = -p
+GINKGO_PARALLEL_OPT =
 ifeq ($(shell echo $${CI:-false}),true)
 GINKGO_OUTPUT_OPT = --github-output --silence-skips
 GINKGO_PARALLEL_OPT = --procs $$(($$(nproc) * $(GINKGO_PARALLEL_MULTIPLIER)))

diff --git a/dist/images/Dockerfile.base b/dist/images/Dockerfile.base
@@ -87,7 +87,7 @@ FROM ubuntu:22.04
 ARG DEBIAN_FRONTEND=noninteractive
 RUN apt update && apt upgrade -y && apt install ca-certificates python3 hostname libunwind8 netbase \
         ethtool iproute2 ncat libunbound8 procps libatomic1 kmod iptables python3-netifaces python3-sortedcontainers \
-        tcpdump ipvsadm ipset curl uuid-runtime openssl inetutils-ping arping ndisc6 conntrack iputils-tracepath \
+        tcpdump ipvsadm ipset curl uuid-runtime openssl inetutils-ping arping ndisc6 conntrack traceroute iputils-tracepath \
         logrotate dnsutils net-tools strongswan strongswan-pki libcharon-extra-plugins libmnl0 \
         libcharon-extauth-plugins libstrongswan-extra-plugins libstrongswan-standard-plugins -y --no-install-recommends && \
         rm -rf /var/lib/apt/lists/* && \

diff --git a/test/e2e/framework/framework.go b/test/e2e/framework/framework.go
@@ -47,6 +47,7 @@ type Framework struct {
 	ClusterIPFamily string
 	// overlay/underlay/underlay-hairpin
 	ClusterNetworkMode string
+	KubeOVNImage       string
 }
 
 func NewDefaultFramework(baseName string) *Framework {
@@ -102,8 +103,9 @@ func (f *Framework) useContext() error {
 func NewFrameworkWithContext(baseName, kubeContext string) *Framework {
 	f := &Framework{KubeContext: kubeContext}
 	ginkgo.BeforeEach(f.BeforeEach)
-
 	f.Framework = framework.NewDefaultFramework(baseName)
+	ginkgo.BeforeEach(f.BeforeEach)
+
 	f.NamespacePodSecurityEnforceLevel = admissionapi.LevelPrivileged
 	f.NamespacePodSecurityWarnLevel = admissionapi.LevelPrivileged
 	f.ClusterIPFamily = os.Getenv("E2E_IP_FAMILY")
@@ -120,10 +122,6 @@ func NewFrameworkWithContext(baseName, kubeContext string) *Framework {
 		f.ClusterVersionMajor, f.ClusterVersionMinor = 999, 999
 	}
 
-	ginkgo.BeforeEach(func() {
-		framework.TestContext.Host = ""
-	})
-
 	return f
 }
 
@@ -185,6 +183,12 @@ func (f *Framework) BeforeEach() {
 		ExpectNoError(err)
 	}
 
+	if f.KubeOVNImage == "" && f.ClientSet != nil {
+		framework.Logf("Getting Kube-OVN image")
+		f.KubeOVNImage = GetKubeOvnImage(f.ClientSet)
+		framework.Logf("Got Kube-OVN image: %s", f.KubeOVNImage)
+	}
+
 	framework.TestContext.Host = ""
 }
 

diff --git a/test/e2e/framework/kube-ovn.go b/test/e2e/framework/kube-ovn.go
@@ -12,5 +12,6 @@ func GetKubeOvnImage(cs clientset.Interface) string {
 	ginkgo.GinkgoHelper()
 	ds, err := cs.AppsV1().DaemonSets(KubeOvnNamespace).Get(context.TODO(), DaemonSetOvsOvn, metav1.GetOptions{})
 	ExpectNoError(err, "getting daemonset %s/%s", KubeOvnNamespace, DaemonSetOvsOvn)
+	ExpectNotNil(ds, "daemonset %s/%s not found", KubeOvnNamespace, DaemonSetOvsOvn)
 	return ds.Spec.Template.Spec.Containers[0].Image
 }
diff --git a/test/e2e/kube-ovn/kubectl-ko/kubectl-ko.go b/test/e2e/kube-ovn/kubectl-ko/kubectl-ko.go
@@ -116,9 +116,8 @@ var _ = framework.Describe("[group:kubectl-ko]", func() {
 		}
 
 		ginkgo.By("Creating pod " + podName)
-		image := framework.GetKubeOvnImage(cs)
 		cmd := []string{"sh", "-c", fmt.Sprintf(`while true; do %s -c1 -w1 %s; sleep 1; done`, ping, target)}
-		pod := framework.MakePod(namespaceName, podName, nil, nil, image, cmd, nil)
+		pod := framework.MakePod(namespaceName, podName, nil, nil, f.KubeOVNImage, cmd, nil)
 		pod = podClient.CreateSync(pod)
 
 		execOrDie(fmt.Sprintf("ko tcpdump %s/%s -c1", pod.Namespace, pod.Name))

diff --git a/test/e2e/kube-ovn/network-policy/network-policy.go b/test/e2e/kube-ovn/network-policy/network-policy.go
@@ -34,7 +34,7 @@ var _ = framework.SerialDescribe("[group:network-policy]", func() {
 	var netpolClient *framework.NetworkPolicyClient
 	var daemonSetClient *framework.DaemonSetClient
 	var namespaceName, netpolName, subnetName, podName string
-	var cidr, image string
+	var cidr string
 
 	ginkgo.BeforeEach(func() {
 		cs = f.ClientSet
@@ -48,10 +48,6 @@ var _ = framework.SerialDescribe("[group:network-policy]", func() {
 		podName = "pod-" + framework.RandomSuffix()
 		subnetName = "subnet-" + framework.RandomSuffix()
 		cidr = framework.RandomCIDR(f.ClusterIPFamily)
-
-		if image == "" {
-			image = framework.GetKubeOvnImage(cs)
-		}
 	})
 	ginkgo.AfterEach(func() {
 		ginkgo.By("Deleting pod " + podName)

diff --git a/test/e2e/kube-ovn/node/node.go b/test/e2e/kube-ovn/node/node.go
@@ -31,7 +31,7 @@ var _ = framework.OrderedDescribe("[group:node]", func() {
 	var podClient *framework.PodClient
 	var serviceClient *framework.ServiceClient
 	var subnetClient *framework.SubnetClient
-	var podName, hostPodName, serviceName, namespaceName, subnetName, image string
+	var podName, hostPodName, serviceName, namespaceName, subnetName string
 	var cidr string
 	ginkgo.BeforeEach(func() {
 		cs = f.ClientSet
@@ -44,10 +44,6 @@ var _ = framework.OrderedDescribe("[group:node]", func() {
 		serviceName = "service-" + framework.RandomSuffix()
 		subnetName = "subnet-" + framework.RandomSuffix()
 		cidr = framework.RandomCIDR(f.ClusterIPFamily)
-
-		if image == "" {
-			image = framework.GetKubeOvnImage(cs)
-		}
 	})
 	ginkgo.AfterEach(func() {
 		ginkgo.By("Deleting service " + serviceName)
@@ -85,7 +81,7 @@ var _ = framework.OrderedDescribe("[group:node]", func() {
 			podName = "pod-" + framework.RandomSuffix()
 			ginkgo.By("Creating pod " + podName + " with host network")
 			cmd := []string{"sh", "-c", "sleep infinity"}
-			pod := framework.MakePod(namespaceName, podName, nil, nil, image, cmd, nil)
+			pod := framework.MakePod(namespaceName, podName, nil, nil, f.KubeOVNImage, cmd, nil)
 			pod.Spec.NodeName = node.Name
 			pod.Spec.HostNetwork = true
 			pod = podClient.CreateSync(pod)
@@ -125,7 +121,7 @@ var _ = framework.OrderedDescribe("[group:node]", func() {
 
 		ginkgo.By("Creating pod " + hostPodName + " with host network")
 		cmd := []string{"sh", "-c", "sleep infinity"}
-		hostPod := framework.MakePod(namespaceName, hostPodName, nil, nil, image, cmd, nil)
+		hostPod := framework.MakePod(namespaceName, hostPodName, nil, nil, f.KubeOVNImage, cmd, nil)
 		hostPod.Spec.HostNetwork = true
 		hostPod = podClient.CreateSync(hostPod)
 
@@ -176,7 +172,7 @@ var _ = framework.OrderedDescribe("[group:node]", func() {
 
 		ginkgo.By("Creating pod " + hostPodName + " with host network")
 		cmd := []string{"sh", "-c", "sleep infinity"}
-		hostPod := framework.MakePod(namespaceName, hostPodName, nil, nil, image, cmd, nil)
+		hostPod := framework.MakePod(namespaceName, hostPodName, nil, nil, f.KubeOVNImage, cmd, nil)
 		hostPod.Spec.HostNetwork = true
 		hostPod = podClient.CreateSync(hostPod)
 
@@ -202,16 +198,13 @@ var _ = framework.SerialDescribe("[group:node]", func() {
 	var cs clientset.Interface
 	var podClient *framework.PodClient
 	var subnetClient *framework.SubnetClient
-	var podName, namespaceName, image string
+	var podName, namespaceName string
 	ginkgo.BeforeEach(func() {
 		cs = f.ClientSet
 		podClient = f.PodClient()
 		subnetClient = f.SubnetClient()
 		namespaceName = f.Namespace.Name
 		podName = "pod-" + framework.RandomSuffix()
-		if image == "" {
-			image = framework.GetKubeOvnImage(cs)
-		}
 	})
 	ginkgo.AfterEach(func() {
 		ginkgo.By("Deleting pod " + podName)
@@ -238,7 +231,7 @@ var _ = framework.SerialDescribe("[group:node]", func() {
 		podName = "pod-" + framework.RandomSuffix()
 		ginkgo.By("Creating pod " + podName + " with host network")
 		cmd := []string{"sh", "-c", "sleep infinity"}
-		pod := framework.MakePrivilegedPod(namespaceName, podName, nil, nil, image, cmd, nil)
+		pod := framework.MakePrivilegedPod(namespaceName, podName, nil, nil, f.KubeOVNImage, cmd, nil)
 		pod.Spec.NodeName = node.Name
 		pod.Spec.HostNetwork = true
 		pod = podClient.CreateSync(pod)

diff --git a/test/e2e/kube-ovn/pod/pod _routes.go b/test/e2e/kube-ovn/pod/pod _routes.go
@@ -4,8 +4,6 @@ import (
 	"encoding/json"
 	"strings"
 
-	clientset "k8s.io/client-go/kubernetes"
-
 	"github.com/onsi/ginkgo/v2"
 
 	apiv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1"
@@ -18,23 +16,18 @@ import (
 var _ = framework.Describe("[group:pod]", func() {
 	f := framework.NewDefaultFramework("pod")
 
-	var cs clientset.Interface
 	var podClient *framework.PodClient
 	var subnetClient *framework.SubnetClient
 	var namespaceName, subnetName, podName string
-	var cidr, image string
+	var cidr string
 
 	ginkgo.BeforeEach(func() {
-		cs = f.ClientSet
 		podClient = f.PodClient()
 		subnetClient = f.SubnetClient()
 		namespaceName = f.Namespace.Name
 		subnetName = "subnet-" + framework.RandomSuffix()
 		podName = "pod-" + framework.RandomSuffix()
 		cidr = framework.RandomCIDR(f.ClusterIPFamily)
-		if image == "" {
-			image = framework.GetKubeOvnImage(cs)
-		}
 	})
 	ginkgo.AfterEach(func() {
 		ginkgo.By("Deleting pod " + podName)
@@ -74,7 +67,7 @@ var _ = framework.Describe("[group:pod]", func() {
 			util.RoutesAnnotation: string(buff),
 		}
 		cmd := []string{"sh", "-c", "sleep infinity"}
-		pod := framework.MakePod(namespaceName, podName, nil, annotations, image, cmd, nil)
+		pod := framework.MakePod(namespaceName, podName, nil, annotations, f.KubeOVNImage, cmd, nil)
 		pod = podClient.CreateSync(pod)
 
 		ginkgo.By("Validating pod annoations")

diff --git a/test/e2e/kube-ovn/pod/vpc_pod_probe.go b/test/e2e/kube-ovn/pod/vpc_pod_probe.go
@@ -10,7 +10,6 @@ import (
 	"github.com/onsi/ginkgo/v2"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/util/intstr"
-	clientset "k8s.io/client-go/kubernetes"
 
 	apiv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1"
 	"github.com/kubeovn/kube-ovn/pkg/util"
@@ -21,18 +20,16 @@ import (
 var _ = framework.SerialDescribe("[group:pod]", func() {
 	f := framework.NewDefaultFramework("vpc-pod-probe")
 
-	var cs clientset.Interface
 	var podClient *framework.PodClient
 	var eventClient *framework.EventClient
 	var subnetClient *framework.SubnetClient
 	var vpcClient *framework.VpcClient
 	var namespaceName, subnetName, podName, vpcName string
 	var subnet *apiv1.Subnet
-	var cidr, image string
+	var cidr string
 	var extraSubnetNames []string
 
 	ginkgo.BeforeEach(func() {
-		cs = f.ClientSet
 		podClient = f.PodClient()
 		eventClient = f.EventClient()
 		subnetClient = f.SubnetClient()
@@ -41,9 +38,6 @@ var _ = framework.SerialDescribe("[group:pod]", func() {
 		podName = "pod-" + framework.RandomSuffix()
 		cidr = framework.RandomCIDR(f.ClusterIPFamily)
 		vpcClient = f.VpcClient()
-		if image == "" {
-			image = framework.GetKubeOvnImage(cs)
-		}
 
 		ginkgo.By("Creating subnet " + subnetName)
 		subnet = framework.MakeSubnet(subnetName, "", cidr, "", "", "", nil, nil, []string{namespaceName})

diff --git a/test/e2e/kube-ovn/service/service.go b/test/e2e/kube-ovn/service/service.go
@@ -29,7 +29,7 @@ var _ = framework.Describe("[group:service]", func() {
 	var serviceClient *framework.ServiceClient
 	var podClient *framework.PodClient
 	var subnetClient *framework.SubnetClient
-	var namespaceName, serviceName, podName, hostPodName, subnetName, cidr, image string
+	var namespaceName, serviceName, podName, hostPodName, subnetName, cidr string
 
 	ginkgo.BeforeEach(func() {
 		cs = f.ClientSet
@@ -42,9 +42,6 @@ var _ = framework.Describe("[group:service]", func() {
 		hostPodName = "pod-" + framework.RandomSuffix()
 		subnetName = "subnet-" + framework.RandomSuffix()
 		cidr = framework.RandomCIDR(f.ClusterIPFamily)
-		if image == "" {
-			image = framework.GetKubeOvnImage(cs)
-		}
 	})
 	ginkgo.AfterEach(func() {
 		ginkgo.By("Deleting service " + serviceName)
@@ -92,7 +89,7 @@ var _ = framework.Describe("[group:service]", func() {
 
 		ginkgo.By("Creating pod " + hostPodName + " with host network")
 		cmd := []string{"sh", "-c", "sleep infinity"}
-		hostPod := framework.MakePod(namespaceName, hostPodName, nil, nil, image, cmd, nil)
+		hostPod := framework.MakePod(namespaceName, hostPodName, nil, nil, f.KubeOVNImage, cmd, nil)
 		hostPod.Spec.HostNetwork = true
 		_ = podClient.CreateSync(hostPod)