diff --git a/test/e2e/cluster.go b/test/e2e/cluster.go index 4962af1..ec5b015 100644 --- a/test/e2e/cluster.go +++ b/test/e2e/cluster.go @@ -10,6 +10,7 @@ import ( "sync" "time" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -40,6 +41,8 @@ type Cluster struct { serviceEdgeNginx6 string serviceHostCloudNginx string serviceHostEdgeNginx string + serviceEdgeMySQL string + serviceEdgeMySQL6 string ready bool } @@ -154,6 +157,8 @@ func (c *Cluster) makeupServiceNames() { c.serviceCloudNginx6 = serviceCloudNginx6 c.serviceEdgeNginx = serviceEdgeNginx c.serviceEdgeNginx6 = serviceEdgeNginx6 + c.serviceEdgeMySQL = serviceEdgeMySQL + c.serviceEdgeMySQL6 = serviceEdgeMySQL6 c.serviceHostCloudNginx = serviceHostCloudNginx c.serviceHostEdgeNginx = serviceHostEdgeNginx } @@ -176,6 +181,15 @@ func (c Cluster) edgeNginxServiceNames() []string { return serviceNames } +func (c Cluster) edgeEdgeMySQLServiceNames() []string { + serviceNames := []string{c.serviceEdgeMySQL} + if framework.TestContext.IPv6Enabled { + serviceNames = append(serviceNames, c.serviceEdgeMySQL6) + } + + return serviceNames +} + func (c Cluster) prepareNamespace(namespace string) { if framework.TestContext.ReuseResource { var ns corev1.Namespace @@ -230,6 +244,85 @@ func (c Cluster) preparePodsOnEachNode(namespace string) { } } +func (c Cluster) prepareEdgeStatefulSet(serviceName, namespace string) { + var replicas int32 = 0 + var nodes corev1.NodeList + framework.ExpectNoError(c.client.List(context.TODO(), &nodes)) + for _, node := range nodes.Items { + if nodeutil.IsEdgeNode(node) { + replicas += 1 + } + } + + mysql := appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: serviceName, + Namespace: namespace, + Labels: map[string]string{ + labelKeyApp: appNetTool, + labelKeyInstance: instanceMySQL, + }, + }, + Spec: appsv1.StatefulSetSpec{ + Replicas: &replicas, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + labelKeyApp: appNetTool, + labelKeyInstance: instanceMySQL, + labelKeyService: serviceName, + }, + }, + ServiceName: serviceName, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + labelKeyApp: appNetTool, + labelKeyInstance: instanceMySQL, + labelKeyService: serviceName, + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "net-tool", + Image: framework.TestContext.NetToolImage, + ImagePullPolicy: corev1.PullIfNotPresent, + Ports: []corev1.ContainerPort{ + { + Name: "http", + ContainerPort: defaultHttpPort, + }, + { + Name: "https", + ContainerPort: defaultHttpsPort, + }, + }, + Env: []corev1.EnvVar{ + { + Name: "HTTP_PORT", + Value: fmt.Sprint(defaultHttpPort), + }, + { + Name: "HTTPS_PORT", + Value: fmt.Sprint(defaultHttpsPort), + }, + }, + }, + }, + Tolerations: []corev1.Toleration{ + { + Key: "", + Operator: corev1.TolerationOpExists, + }, + }, + }, + }, + }, + } + + createObject(c.client, &mysql) +} + func (c Cluster) prepareHostNetworkPodsOnEachNode(namespace string) { var nodes corev1.NodeList framework.ExpectNoError(c.client.List(context.TODO(), &nodes)) @@ -253,12 +346,46 @@ func (c Cluster) prepareService(name, namespace string, ipFamily corev1.IPFamily Namespace: namespace, }, Spec: corev1.ServiceSpec{ - Type: corev1.ServiceTypeClusterIP, + Type: corev1.ServiceTypeNodePort, IPFamilies: []corev1.IPFamily{ipFamily}, Selector: map[string]string{ labelKeyLocation: string(location), labelKeyUseHostNetwork: fmt.Sprint(useHostNetwork), }, + Ports: []corev1.ServicePort{ + { + Name: "http", + Port: 80, + TargetPort: intstr.FromInt(defaultHttpPort), + Protocol: corev1.ProtocolTCP, + }, + { + Name: "https", + Port: 443, + TargetPort: intstr.FromInt(defaultHttpPort), + Protocol: corev1.ProtocolTCP, + }, + }, + }, + } + + createObject(c.client, &svc) +} + +func (c Cluster) prepareHeadLessService(name, namespace string, ipFamily corev1.IPFamily) { + framework.Logf("create service %s/%s on %s", namespace, name, c.name) + svc := corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeClusterIP, + ClusterIP: "None", + IPFamilies: []corev1.IPFamily{ipFamily}, + Selector: map[string]string{ + labelKeyInstance: instanceMySQL, + }, Ports: []corev1.ServicePort{ { Name: "http", diff --git a/test/e2e/e2e.go b/test/e2e/e2e.go index 5550a53..4403551 100644 --- a/test/e2e/e2e.go +++ b/test/e2e/e2e.go @@ -54,9 +54,11 @@ const ( appNetTool = "fabedge-net-tool" instanceNetTool = "net-tool" instanceHostNetTool = "host-net-tool" + instanceMySQL = "mysql" labelKeyApp = "app" labelKeyInstance = "instance" + labelKeyService = "service" labelKeyLocation = "location" labelKeyUseHostNetwork = "use-host-network" // add a random label, prevent kubeedge to cache it @@ -66,6 +68,8 @@ const ( serviceCloudNginx6 = "cloud-nginx6" serviceEdgeNginx = "edge-nginx" serviceEdgeNginx6 = "edge-nginx6" + serviceEdgeMySQL = "edge-mysql" + serviceEdgeMySQL6 = "edge-mysql6" serviceHostCloudNginx = "host-cloud-nginx" serviceHostEdgeNginx = "host-edge-nginx" @@ -168,6 +172,7 @@ func singleClusterE2eTestPrepare() { } prepareClustersNamespace(namespaceSingle) preparePodsOnEachClusterNode(namespaceSingle) + prepareStatefulSets(namespaceSingle) cluster.prepareHostNetworkPodsOnEachNode(namespaceSingle) prepareServicesOnEachCluster(namespaceSingle) @@ -261,6 +266,15 @@ func preparePodsOnEachClusterNode(namespace string) { } } +func prepareStatefulSets(namespace string) { + for _, cluster := range clusters { + cluster.prepareEdgeStatefulSet(serviceEdgeMySQL, namespace) + if framework.TestContext.IPv6Enabled { + cluster.prepareEdgeStatefulSet(serviceEdgeMySQL6, namespace) + } + } +} + func prepareServicesOnEachCluster(namespace string) { for _, cluster := range clusters { cluster.prepareService(cluster.serviceCloudNginx, namespace, corev1.IPv4Protocol, LocationCloud, false) @@ -268,6 +282,11 @@ func prepareServicesOnEachCluster(namespace string) { cluster.prepareService(cluster.serviceCloudNginx6, namespace, corev1.IPv6Protocol, LocationCloud, false) } + cluster.prepareHeadLessService(cluster.serviceEdgeMySQL, namespace, corev1.IPv4Protocol) + if framework.TestContext.IPv6Enabled { + cluster.prepareHeadLessService(cluster.serviceEdgeMySQL6, namespace, corev1.IPv6Protocol) + } + if !framework.TestContext.IsMultiClusterTest() { cluster.prepareService(cluster.serviceEdgeNginx, namespace, corev1.IPv4Protocol, LocationEdge, false) if framework.TestContext.IPv6Enabled { diff --git a/test/e2e/service.go b/test/e2e/service.go index 9dadda3..ba1dab0 100644 --- a/test/e2e/service.go +++ b/test/e2e/service.go @@ -95,6 +95,35 @@ var _ = Describe("FabEdge", func() { } }) + It("let edge pods can access headless edge services [p2p][e2e]", func() { + _, edgePods, err := framework.ListCloudAndEdgePods(cluster.client, + client.InNamespace(namespaceSingle), + client.MatchingLabels{labelKeyInstance: instanceNetTool}, + ) + framework.ExpectNoError(err) + + for _, serviceName := range cluster.edgeEdgeMySQLServiceNames() { + servicePods, _, err := framework.ListCloudAndEdgePods(cluster.client, + client.InNamespace(namespaceSingle), + client.MatchingLabels{labelKeyService: serviceName}, + ) + framework.ExpectNoError(err) + + replicas := len(edgePods) + for _, pod := range edgePods { + framework.Logf("pod %s visit service %s", pod.Name, serviceName) + cluster.checkServiceAvailability(pod, serviceName, servicePods) + + for i := 0; i < replicas; i++ { + endpointName := fmt.Sprintf("%s-%d.%s:%d", serviceName, i, serviceName, defaultHttpPort) + framework.Logf("pod %s visit endpoint %s", pod.Name, endpointName) + _, _, e := cluster.execCurl(pod, endpointName) + framework.ExpectNoError(e) + } + } + } + }) + It("let edge pods can access edge host service [p2n][e2e]", func() { _, edgePods, err := framework.ListCloudAndEdgePods(cluster.client, client.InNamespace(namespaceSingle),