-
Notifications
You must be signed in to change notification settings - Fork 136
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
[YUNIKORN-2068] E2E Test for Preemption #705
Changes from 1 commit
b83bdd5
3830426
72ef0d8
f3e8b57
46eb327
856b629
7e22832
353bf51
2206722
c972734
daac46d
93ab76e
d916cd9
bca7152
b30f5ec
726c154
ad8e7eb
c4f17b0
ce0351b
fae4a0f
d2d1130
5fbc97f
c86acb1
d85fa9a
87494ce
7a30f8e
461cff1
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
- Loading branch information
There are no files selected for viewing
Original file line number | Diff line number | Diff line change | ||||
---|---|---|---|---|---|---|
|
@@ -23,24 +23,24 @@ import ( | |||||
"strings" | ||||||
"time" | ||||||
|
||||||
"github.com/onsi/ginkgo/v2" | ||||||
"github.com/onsi/gomega" | ||||||
v1 "k8s.io/api/core/v1" | ||||||
schedulingv1 "k8s.io/api/scheduling/v1" | ||||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" | ||||||
|
||||||
"github.com/apache/yunikorn-core/pkg/common/configs" | ||||||
"github.com/apache/yunikorn-k8shim/pkg/common/constants" | ||||||
tests "github.com/apache/yunikorn-k8shim/test/e2e" | ||||||
"github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/common" | ||||||
"github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/k8s" | ||||||
"github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/yunikorn" | ||||||
"github.com/onsi/ginkgo/v2" | ||||||
"github.com/onsi/gomega" | ||||||
v1 "k8s.io/api/core/v1" | ||||||
schedulingv1 "k8s.io/api/scheduling/v1" | ||||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" | ||||||
) | ||||||
|
||||||
var kClient k8s.KubeCtl | ||||||
var restClient yunikorn.RClient | ||||||
var ns *v1.Namespace | ||||||
var dev = "dev" + common.RandSeq(5) | ||||||
var devNew = "dev" + common.RandSeq(5) | ||||||
var oldConfigMap = new(v1.ConfigMap) | ||||||
var annotation = "ann-" + common.RandSeq(10) | ||||||
|
||||||
|
@@ -587,20 +587,23 @@ var _ = ginkgo.Describe("Preemption", func() { | |||||
return nil | ||||||
}) | ||||||
|
||||||
newNamespace, err := kClient.CreateNamespace(devNew, nil) | ||||||
gomega.Ω(err).NotTo(gomega.HaveOccurred()) | ||||||
gomega.Ω(newNamespace.Status.Phase).To(gomega.Equal(v1.NamespaceActive)) | ||||||
// Define sleepPod | ||||||
sleepPodConfigs := createSandbox1SleepPodCofigsWithStaticNode(3, 600) | ||||||
sleepPod4Config := k8s.SleepPodConfig{Name: "sleepjob4", NS: dev, Mem: sleepPodMemLimit, Time: 600, Optedout: k8s.Allow, Labels: map[string]string{"queue": "root.sandbox2"}, RequiredNode: nodeName} | ||||||
sleepPod4Config := k8s.SleepPodConfig{Name: "sleepjob4", NS: devNew, Mem: sleepPodMemLimit, Time: 600, Optedout: k8s.Allow, Labels: map[string]string{"queue": "root.sandbox2"}, RequiredNode: nodeName} | ||||||
sleepPodConfigs = append(sleepPodConfigs, sleepPod4Config) | ||||||
|
||||||
for _, config := range sleepPodConfigs { | ||||||
ginkgo.By("Deploy the sleep pod " + config.Name + " to the development namespace") | ||||||
sleepObj, podErr := k8s.InitSleepPod(config) | ||||||
Ω(podErr).NotTo(gomega.HaveOccurred()) | ||||||
sleepRespPod, podErr := kClient.CreatePod(sleepObj, dev) | ||||||
sleepRespPod, podErr := kClient.CreatePod(sleepObj, devNew) | ||||||
gomega.Ω(podErr).NotTo(gomega.HaveOccurred()) | ||||||
|
||||||
// Wait for pod to move to running state | ||||||
podErr = kClient.WaitForPodBySelectorRunning(dev, | ||||||
podErr = kClient.WaitForPodBySelectorRunning(devNew, | ||||||
fmt.Sprintf("app=%s", sleepRespPod.ObjectMeta.Labels["app"]), | ||||||
120) | ||||||
gomega.Ω(podErr).NotTo(gomega.HaveOccurred()) | ||||||
|
@@ -609,7 +612,7 @@ var _ = ginkgo.Describe("Preemption", func() { | |||||
// assert one of the pods in root.sandbox1 is preempted | ||||||
ginkgo.By("One of the pods in root.sanbox1 is preempted") | ||||||
sandbox1RunningPodsCnt := 0 | ||||||
pods, err := kClient.ListPodsByLabelSelector(dev, "queue=root.sandbox1") | ||||||
pods, err := kClient.ListPodsByLabelSelector(devNew, "queue=root.sandbox1") | ||||||
gomega.Ω(err).NotTo(gomega.HaveOccurred()) | ||||||
for _, pod := range pods.Items { | ||||||
if pod.DeletionTimestamp != nil { | ||||||
|
@@ -620,6 +623,10 @@ var _ = ginkgo.Describe("Preemption", func() { | |||||
} | ||||||
} | ||||||
Ω(sandbox1RunningPodsCnt).To(gomega.Equal(2), "One of the pods in root.sandbox1 should be preempted") | ||||||
errNew := kClient.DeletePods(newNamespace.Name) | ||||||
if errNew != nil { | ||||||
fmt.Fprintf(ginkgo.GinkgoWriter, "Failed to delete pods in namespace %s - reason is %s\n", newNamespace.Name, err.Error()) | ||||||
} | ||||||
}) | ||||||
|
||||||
ginkgo.AfterEach(func() { | ||||||
|
@@ -652,7 +659,7 @@ func createSandbox1SleepPodCofigs(cnt, time int) []k8s.SleepPodConfig { | |||||
func createSandbox1SleepPodCofigsWithStaticNode(cnt, time int) []k8s.SleepPodConfig { | ||||||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. "WithRequiredNode" or "WithNodeSelector" sounds better There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Acknowledged. |
||||||
sandbox1Configs := make([]k8s.SleepPodConfig, 0, cnt) | ||||||
for i := 0; i < cnt; i++ { | ||||||
sandbox1Configs = append(sandbox1Configs, k8s.SleepPodConfig{Name: fmt.Sprintf("sleepjob%d", i+1), NS: dev, Mem: sleepPodMemLimit2, Time: time, Optedout: k8s.Allow, Labels: map[string]string{"queue": "root.sandbox1"}, RequiredNode: nodeName}) | ||||||
sandbox1Configs = append(sandbox1Configs, k8s.SleepPodConfig{Name: fmt.Sprintf("sleepjob%d", i+1), NS: devNew, Mem: sleepPodMemLimit2, Time: time, Optedout: k8s.Allow, Labels: map[string]string{"queue": "root.sandbox1"}, RequiredNode: nodeName}) | ||||||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This part is very suspicious. You're using the "RequiredNode" setting here, which will trigger the "simple" (aka. RequiredNode) preemption logic inside Yunikorn. That is not the generic preemption logic that Craig worked on. Please check if the console log of Yunikorn contains this:
If this is the case (which is VERY likely), changes are necessary. A trivial solution is to enhance the
If DaemonSet == false, then this code doesn't run: yunikorn-k8shim/test/e2e/framework/helpers/k8s/pod_conf.go Lines 76 to 77 in 28995cd
So this line effectively becomes:
The existing tests inside There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Have you looked at this? Again, the tests should address YUNIKORN-2068. RequiredNodePreemptor does not use the predicates. |
||||||
} | ||||||
return sandbox1Configs | ||||||
} |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Already performed inside AfterEach()
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
After each function is doing cleanup for a global namespace which is created for all the tests, In this test I am creating a new namespace and doing cleanup at the end of the test function to ensure there is no resources are available. I tried with the Global namespace and was facing some issue so trying to create inside the test script only and clean once execution is done.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
If you need a separate namespace for each test case, that should be done in BeforeEach/AfterEach. If something fails before this code, it will never be cleaned up.
Example: