diff --git a/test/e2e/basic_scheduling/basic_scheduling_suite_test.go b/test/e2e/basic_scheduling/basic_scheduling_suite_test.go index ad560005f..7ba3c7113 100644 --- a/test/e2e/basic_scheduling/basic_scheduling_suite_test.go +++ b/test/e2e/basic_scheduling/basic_scheduling_suite_test.go @@ -54,5 +54,6 @@ var By = ginkgo.By var Ω = gomega.Ω var BeNil = gomega.BeNil +var dev string var HaveOccurred = gomega.HaveOccurred var BeEquivalentTo = gomega.BeEquivalentTo diff --git a/test/e2e/basic_scheduling/basic_scheduling_test.go b/test/e2e/basic_scheduling/basic_scheduling_test.go index 9f3dca890..e1f7211e8 100644 --- a/test/e2e/basic_scheduling/basic_scheduling_test.go +++ b/test/e2e/basic_scheduling/basic_scheduling_test.go @@ -37,7 +37,6 @@ var suiteName string var kClient k8s.KubeCtl var restClient yunikorn.RClient var sleepRespPod *v1.Pod -var dev = "dev" + common.RandSeq(5) var appsInfo *dao.ApplicationDAOInfo var oldConfigMap = new(v1.ConfigMap) @@ -52,20 +51,18 @@ var _ = ginkgo.BeforeSuite(func() { gomega.Ω(kClient.SetClient()).To(gomega.BeNil()) // Initializing rest client restClient = yunikorn.RClient{} - yunikorn.EnsureYuniKornConfigsPresent() - By("Port-forward the scheduler pod") err := kClient.PortForwardYkSchedulerPod() Ω(err).NotTo(HaveOccurred()) - yunikorn.UpdateConfigMapWrapper(oldConfigMap, "fifo") - +}) +var _ = ginkgo.BeforeEach(func() { + dev = "dev" + common.RandSeq(5) ginkgo.By("create development namespace") ns1, err := kClient.CreateNamespace(dev, nil) gomega.Ω(err).NotTo(gomega.HaveOccurred()) gomega.Ω(ns1.Status.Phase).To(gomega.Equal(v1.NamespaceActive)) - ginkgo.By("Deploy the sleep pod to the development namespace") initPod, podErr := k8s.InitSleepPod(sleepPodConfigs) gomega.Ω(podErr).NotTo(gomega.HaveOccurred()) @@ -74,17 +71,17 @@ var _ = ginkgo.BeforeSuite(func() { // Wait for pod to move to running state err = kClient.WaitForPodRunning(dev, sleepPodConfigs.Name, 30*time.Second) gomega.Ω(err).NotTo(gomega.HaveOccurred()) - appsInfo, err = restClient.GetAppInfo("default", "root."+dev, sleepRespPod.ObjectMeta.Labels["applicationId"]) gomega.Ω(err).NotTo(gomega.HaveOccurred()) gomega.Ω(appsInfo).NotTo(gomega.BeNil()) }) - -var _ = ginkgo.AfterSuite(func() { +var _ = ginkgo.AfterEach(func() { ginkgo.By("Tear down namespace: " + dev) err := kClient.TearDownNamespace(dev) Ω(err).NotTo(HaveOccurred()) +}) +var _ = ginkgo.AfterSuite(func() { yunikorn.RestoreConfigMapWrapper(oldConfigMap) }) diff --git a/test/e2e/persistent_volume/persistent_volume_suite_test.go b/test/e2e/persistent_volume/persistent_volume_suite_test.go index e20f973c7..d82a4c619 100644 --- a/test/e2e/persistent_volume/persistent_volume_suite_test.go +++ b/test/e2e/persistent_volume/persistent_volume_suite_test.go @@ -48,3 +48,4 @@ func TestPersistentVolume(t *testing.T) { var Ω = gomega.Ω var HaveOccurred = gomega.HaveOccurred +var dev string diff --git a/test/e2e/persistent_volume/persistent_volume_test.go b/test/e2e/persistent_volume/persistent_volume_test.go index 8957d29d9..c82196f78 100644 --- a/test/e2e/persistent_volume/persistent_volume_test.go +++ b/test/e2e/persistent_volume/persistent_volume_test.go @@ -40,7 +40,6 @@ import ( var suiteName string var kClient k8s.KubeCtl var restClient yunikorn.RClient -var dev = "dev-" + common.RandSeq(5) const ( LocalTypePv = "Local" @@ -57,27 +56,30 @@ var _ = ginkgo.BeforeSuite(func() { // Initializing rest client restClient = yunikorn.RClient{} Ω(restClient).NotTo(gomega.BeNil()) - yunikorn.EnsureYuniKornConfigsPresent() - +}) +var _ = ginkgo.BeforeEach(func() { // Create namespace + dev = "dev-" + common.RandSeq(5) ginkgo.By("Create namespace " + dev) ns, err := kClient.CreateNamespace(dev, nil) Ω(err).NotTo(HaveOccurred()) Ω(ns.Status.Phase).To(gomega.Equal(v1.NamespaceActive)) }) +var _ = ginkgo.AfterEach(func() { + ginkgo.By("Tearing down namespace: " + dev) + err := kClient.TearDownNamespace(dev) + Ω(err).NotTo(HaveOccurred()) +}) + var _ = ginkgo.AfterSuite(func() { // Clean up ginkgo.By("Deleting PVCs and PVs") err := kClient.DeletePVCs(dev) err2 := kClient.DeletePVs(dev) - ginkgo.By("Tearing down namespace: " + dev) - err3 := kClient.TearDownNamespace(dev) - Ω(err).NotTo(HaveOccurred()) Ω(err2).NotTo(HaveOccurred()) - Ω(err3).NotTo(HaveOccurred()) }) var _ = ginkgo.Describe("PersistentVolume", func() { diff --git a/test/e2e/recovery_and_restart/recovery_and_restart_suite_test.go b/test/e2e/recovery_and_restart/recovery_and_restart_suite_test.go index 8a8203636..b79653f4a 100644 --- a/test/e2e/recovery_and_restart/recovery_and_restart_suite_test.go +++ b/test/e2e/recovery_and_restart/recovery_and_restart_suite_test.go @@ -50,3 +50,4 @@ func TestRecoveryAndRestart(t *testing.T) { } var Ω = gomega.Ω +var dev string diff --git a/test/e2e/recovery_and_restart/recovery_and_restart_test.go b/test/e2e/recovery_and_restart/recovery_and_restart_test.go index d78326f99..a94df26c4 100644 --- a/test/e2e/recovery_and_restart/recovery_and_restart_test.go +++ b/test/e2e/recovery_and_restart/recovery_and_restart_test.go @@ -52,7 +52,6 @@ var kClient k8s.KubeCtl var restClient yunikorn.RClient var oldConfigMap = new(v1.ConfigMap) var sleepRespPod *v1.Pod -var dev = "dev" + common.RandSeq(5) // Define sleepPod var sleepPodConfigs = k8s.SleepPodConfig{Name: "sleepjob", NS: dev} @@ -70,6 +69,15 @@ var _ = ginkgo.BeforeSuite(func() { yunikorn.EnsureYuniKornConfigsPresent() yunikorn.UpdateConfigMapWrapper(oldConfigMap, "") + ginkgo.By("Restart the scheduler pod") + yunikorn.RestartYunikorn(&kClient) + + ginkgo.By("Port-forward scheduler pod after restart") + yunikorn.RestorePortForwarding(&kClient) +}) + +var _ = ginkgo.BeforeEach(func() { + dev = "dev" + common.RandSeq(5) ginkgo.By("create development namespace") ns1, err := kClient.CreateNamespace(dev, nil) gomega.Ω(err).NotTo(gomega.HaveOccurred()) @@ -85,13 +93,6 @@ var _ = ginkgo.BeforeSuite(func() { fmt.Sprintf("applicationId=%s", sleepRespPod.ObjectMeta.Labels["applicationId"]), 60) gomega.Ω(err).NotTo(gomega.HaveOccurred()) - - ginkgo.By("Restart the scheduler pod") - yunikorn.RestartYunikorn(&kClient) - - ginkgo.By("Port-forward scheduler pod after restart") - yunikorn.RestorePortForwarding(&kClient) - ginkgo.By("Deploy 2nd sleep pod to the development namespace") sleepObj2, podErr := k8s.InitSleepPod(sleepPod2Configs) Ω(podErr).NotTo(gomega.HaveOccurred()) @@ -104,10 +105,13 @@ var _ = ginkgo.BeforeSuite(func() { gomega.Ω(err).NotTo(gomega.HaveOccurred()) }) -var _ = ginkgo.AfterSuite(func() { +var _ = ginkgo.AfterEach(func() { ginkgo.By("Tear down namespace: " + dev) err := kClient.TearDownNamespace(dev) Ω(err).NotTo(gomega.HaveOccurred()) +}) + +var _ = ginkgo.AfterSuite(func() { // call the healthCheck api to check scheduler health ginkgo.By("Check Yunikorn's health") diff --git a/test/e2e/restart_changed_config/restart_changed_config_test.go b/test/e2e/restart_changed_config/restart_changed_config_test.go index a77a929d1..792ad4268 100644 --- a/test/e2e/restart_changed_config/restart_changed_config_test.go +++ b/test/e2e/restart_changed_config/restart_changed_config_test.go @@ -62,10 +62,11 @@ var _ = ginkgo.BeforeSuite(func() { ginkgo.By("Port-forward the scheduler pod") var err = kClient.PortForwardYkSchedulerPod() Ω(err).NotTo(gomega.HaveOccurred()) +}) +var _ = ginkgo.BeforeEach(func() { ginkgo.By("create development namespace") - var ns *v1.Namespace - ns, err = kClient.CreateNamespace(dev, nil) + ns, err := kClient.CreateNamespace(dev, nil) gomega.Ω(err).NotTo(gomega.HaveOccurred()) gomega.Ω(ns.Status.Phase).To(gomega.Equal(v1.NamespaceActive)) ns, err = kClient.CreateNamespace(test, nil) @@ -74,14 +75,6 @@ var _ = ginkgo.BeforeSuite(func() { }) var _ = ginkgo.AfterSuite(func() { - ginkgo.By("Tear down namespace: " + dev) - err := kClient.TearDownNamespace(dev) - Ω(err).NotTo(gomega.HaveOccurred()) - - ginkgo.By("Tear down namespace: " + test) - err = kClient.TearDownNamespace(test) - Ω(err).NotTo(gomega.HaveOccurred()) - // call the healthCheck api to check scheduler health ginkgo.By("Check YuniKorn's health") checks, err2 := yunikorn.GetFailedHealthChecks() @@ -172,5 +165,12 @@ var _ = ginkgo.Describe("PodInRecoveryQueue", func() { ginkgo.AfterEach(func() { tests.DumpClusterInfoIfSpecFailed(suiteName, []string{dev, test}) + ginkgo.By("Tear down namespace: " + dev) + err := kClient.TearDownNamespace(dev) + Ω(err).NotTo(gomega.HaveOccurred()) + + ginkgo.By("Tear down namespace: " + test) + err = kClient.TearDownNamespace(test) + Ω(err).NotTo(gomega.HaveOccurred()) }) }) diff --git a/test/e2e/simple_preemptor/simple_preemptor_suite_test.go b/test/e2e/simple_preemptor/simple_preemptor_suite_test.go index 20dd27eba..d0372bf68 100644 --- a/test/e2e/simple_preemptor/simple_preemptor_suite_test.go +++ b/test/e2e/simple_preemptor/simple_preemptor_suite_test.go @@ -48,3 +48,4 @@ func TestSimplePreemptor(t *testing.T) { var Ω = gomega.Ω var HaveOccurred = gomega.HaveOccurred +var dev string diff --git a/test/e2e/simple_preemptor/simple_preemptor_test.go b/test/e2e/simple_preemptor/simple_preemptor_test.go index 6af05a0c1..8f902a23e 100644 --- a/test/e2e/simple_preemptor/simple_preemptor_test.go +++ b/test/e2e/simple_preemptor/simple_preemptor_test.go @@ -24,24 +24,22 @@ import ( "strings" "time" - "github.com/onsi/ginkgo/v2" - "github.com/onsi/gomega" - v1 "k8s.io/api/core/v1" - k8serrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/api/resource" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - tests "github.com/apache/yunikorn-k8shim/test/e2e" + "github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/common" "github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/k8s" "github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/yunikorn" + + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" + + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" ) var suiteName string var kClient k8s.KubeCtl var restClient yunikorn.RClient -var ns *v1.Namespace -var dev = "dev" + common.RandSeq(5) var oldConfigMap = new(v1.ConfigMap) // Nodes @@ -71,11 +69,6 @@ var _ = ginkgo.BeforeSuite(func() { var err = kClient.PortForwardYkSchedulerPod() Ω(err).NotTo(gomega.HaveOccurred()) - ginkgo.By("create development namespace") - ns, err = kClient.CreateNamespace(dev, nil) - gomega.Ω(err).NotTo(gomega.HaveOccurred()) - gomega.Ω(ns.Status.Phase).To(gomega.Equal(v1.NamespaceActive)) - var nodes *v1.NodeList nodes, err = kClient.GetNodes() Ω(err).NotTo(gomega.HaveOccurred()) @@ -126,6 +119,14 @@ var _ = ginkgo.BeforeSuite(func() { sleepPodMemLimit2 = int64(float64(Worker2Res.Value())/3.5) / (1000 * 1000) }) +var _ = ginkgo.BeforeEach(func() { + dev = "dev" + common.RandSeq(5) + ginkgo.By("create development namespace") + ns, err := kClient.CreateNamespace(dev, nil) + gomega.Ω(err).NotTo(gomega.HaveOccurred()) + gomega.Ω(ns.Status.Phase).To(gomega.Equal(v1.NamespaceActive)) +}) + var _ = ginkgo.AfterSuite(func() { ginkgo.By("Untainting some nodes") @@ -136,10 +137,6 @@ var _ = ginkgo.AfterSuite(func() { checks, err := yunikorn.GetFailedHealthChecks() Ω(err).NotTo(gomega.HaveOccurred()) Ω(checks).To(gomega.Equal(""), checks) - ginkgo.By("Tearing down namespace: " + ns.Name) - err = kClient.TearDownNamespace(ns.Name) - Ω(err).NotTo(gomega.HaveOccurred()) - yunikorn.RestoreConfigMapWrapper(oldConfigMap) }) @@ -205,9 +202,7 @@ var _ = ginkgo.Describe("SimplePreemptor", func() { gomega.Ω(err).NotTo(gomega.HaveOccurred()) // Wait for pod to move to running state - err = kClient.WaitForPodBySelectorRunning(dev, - fmt.Sprintf("app=%s", sleepRespPod.ObjectMeta.Labels["app"]), - 600) + err = kClient.WaitForPodRunning(dev, sleepRespPod.Name, 1*time.Minute) gomega.Ω(err).NotTo(gomega.HaveOccurred()) // assert sleeppod4 is killed @@ -217,29 +212,9 @@ var _ = ginkgo.Describe("SimplePreemptor", func() { }) ginkgo.AfterEach(func() { - tests.DumpClusterInfoIfSpecFailed(suiteName, []string{ns.Name}) - - // Delete all sleep pods - ginkgo.By("Delete all sleep pods") - pods, err := kClient.GetPodNamesFromNS(ns.Name) - if err == nil { - for _, each := range pods { - if strings.Contains(each, "sleep") { - ginkgo.By("Deleting sleep pod: " + each) - err = kClient.DeletePod(each, ns.Name) - if err != nil { - if statusErr, ok := err.(*k8serrors.StatusError); ok { - if statusErr.ErrStatus.Reason == metav1.StatusReasonNotFound { - fmt.Fprintf(ginkgo.GinkgoWriter, "Failed to delete pod %s - reason is %s, it "+ - "has been deleted in the meantime\n", each, statusErr.ErrStatus.Reason) - continue - } - } - } - } - } - } else { - fmt.Fprintf(ginkgo.GinkgoWriter, "Failed to get pods from namespace %s - reason is %s\n", ns.Name, err.Error()) - } + tests.DumpClusterInfoIfSpecFailed(suiteName, []string{dev}) + ginkgo.By("Tearing down namespace: " + dev) + err := kClient.TearDownNamespace(dev) + Ω(err).NotTo(gomega.HaveOccurred()) }) }) diff --git a/test/e2e/user_group_limit/user_group_limit_suite_test.go b/test/e2e/user_group_limit/user_group_limit_suite_test.go index bf06fa0ec..5bdfd3bc8 100644 --- a/test/e2e/user_group_limit/user_group_limit_suite_test.go +++ b/test/e2e/user_group_limit/user_group_limit_suite_test.go @@ -48,3 +48,4 @@ func TestUserGroupLimit(t *testing.T) { var Ω = gomega.Ω var HaveOccurred = gomega.HaveOccurred +var dev string diff --git a/test/e2e/user_group_limit/user_group_limit_test.go b/test/e2e/user_group_limit/user_group_limit_test.go index b92a341bd..7eb3d016b 100644 --- a/test/e2e/user_group_limit/user_group_limit_test.go +++ b/test/e2e/user_group_limit/user_group_limit_test.go @@ -68,8 +68,6 @@ var ( suiteName string kClient k8s.KubeCtl restClient yunikorn.RClient - ns *v1.Namespace - dev = "dev" + common.RandSeq(5) oldConfigMap = new(v1.ConfigMap) admissionCustomConfig = map[string]string{ "log.core.scheduler.ugm.level": "debug", @@ -92,9 +90,12 @@ var _ = ginkgo.BeforeSuite(func() { ginkgo.By("Port-forward the scheduler pod") var err = kClient.PortForwardYkSchedulerPod() Ω(err).NotTo(gomega.HaveOccurred()) +}) +var _ = ginkgo.BeforeEach(func() { + dev = "dev" + common.RandSeq(5) ginkgo.By("create development namespace") - ns, err = kClient.CreateNamespace(dev, nil) + ns, err := kClient.CreateNamespace(dev, nil) gomega.Ω(err).NotTo(gomega.HaveOccurred()) gomega.Ω(ns.Status.Phase).To(gomega.Equal(v1.NamespaceActive)) }) @@ -104,9 +105,6 @@ var _ = ginkgo.AfterSuite(func() { checks, err := yunikorn.GetFailedHealthChecks() Ω(err).NotTo(gomega.HaveOccurred()) Ω(checks).To(gomega.Equal(""), checks) - ginkgo.By("Tearing down namespace: " + ns.Name) - err = kClient.TearDownNamespace(ns.Name) - Ω(err).NotTo(gomega.HaveOccurred()) }) var _ = ginkgo.Describe("UserGroupLimit", func() { @@ -912,15 +910,10 @@ var _ = ginkgo.Describe("UserGroupLimit", func() { }) ginkgo.AfterEach(func() { - tests.DumpClusterInfoIfSpecFailed(suiteName, []string{ns.Name}) - - // Delete all sleep pods - ginkgo.By("Delete all sleep pods") - err := kClient.DeletePods(ns.Name) - if err != nil { - fmt.Fprintf(ginkgo.GinkgoWriter, "Failed to delete pods in namespace %s - reason is %s\n", ns.Name, err.Error()) - } - + tests.DumpClusterInfoIfSpecFailed(suiteName, []string{dev}) + ginkgo.By("Tearing down namespace: " + dev) + err := kClient.TearDownNamespace(dev) + Ω(err).NotTo(gomega.HaveOccurred()) // reset config ginkgo.By("Restoring YuniKorn configuration") yunikorn.RestoreConfigMapWrapper(oldConfigMap)