From ec1040ac153eac9c8efc40af5a077408ccabfce8 Mon Sep 17 00:00:00 2001 From: 0yukali0 Date: Tue, 15 Aug 2023 15:42:16 +0800 Subject: [PATCH 01/16] function, rest and basic test are ready --- test/e2e/framework/configmanager/constants.go | 2 + .../framework/helpers/k8s/pod_annotation.go | 6 + test/e2e/framework/helpers/k8s/pod_conf.go | 16 +++ .../helpers/yunikorn/rest_api_utils.go | 23 ++++ .../user_qauta_tracing_suite_test.go | 29 +++++ .../user_qauta_tracing_test.go | 105 ++++++++++++++++++ 6 files changed, 181 insertions(+) create mode 100644 test/e2e/user_qauta_tracing/user_qauta_tracing_suite_test.go create mode 100644 test/e2e/user_qauta_tracing/user_qauta_tracing_test.go diff --git a/test/e2e/framework/configmanager/constants.go b/test/e2e/framework/configmanager/constants.go index e00ae7764..2c86e3bea 100644 --- a/test/e2e/framework/configmanager/constants.go +++ b/test/e2e/framework/configmanager/constants.go @@ -46,7 +46,9 @@ const ( ClustersPath = "ws/v1/clusters" NodesPath = "ws/v1/partition/%s/nodes" UserUsagePath = "ws/v1/partition/%s/usage/user/%s" + UsersUsagePath = "ws/v1/partition/%s/usage/users" GroupUsagePath = "ws/v1/partition/%s/usage/group/%s" + GroupsUsagePath = "ws/v1/partition/%s/usage/groups" HealthCheckPath = "ws/v1/scheduler/healthcheck" ValidateConfPath = "ws/v1/validate-conf" diff --git a/test/e2e/framework/helpers/k8s/pod_annotation.go b/test/e2e/framework/helpers/k8s/pod_annotation.go index a014697d1..a27f86c2f 100644 --- a/test/e2e/framework/helpers/k8s/pod_annotation.go +++ b/test/e2e/framework/helpers/k8s/pod_annotation.go @@ -24,9 +24,15 @@ type PodAnnotation struct { TaskGroupName string `json:"yunikorn.apache.org/task-group-name,omitempty"` TaskGroups []interfaces.TaskGroup `json:"-"` SchedulingPolicyParams string `json:"yunikorn.apache.org/schedulingPolicyParameters,omitempty"` + Info UserInfo `json:"yunikorn.apache.org/user.info,omitempty"` Other map[string]string `json:"-"` } +type UserInfo struct { + User string `"json:user,omitempty"` + Groups []string `"json:groups,omitempty"` +} + const ( TaskGroupName = "yunikorn.apache.org/task-group-name" TaskGroups = "yunikorn.apache.org/task-groups" diff --git a/test/e2e/framework/helpers/k8s/pod_conf.go b/test/e2e/framework/helpers/k8s/pod_conf.go index 5da113eea..69ac96cf3 100644 --- a/test/e2e/framework/helpers/k8s/pod_conf.go +++ b/test/e2e/framework/helpers/k8s/pod_conf.go @@ -41,6 +41,8 @@ type SleepPodConfig struct { RequiredNode string Optedout bool Labels map[string]string + UserName string + GroupNames []string } // TestPodConfig template for sleepPods @@ -62,6 +64,17 @@ func InitSleepPod(conf SleepPodConfig) (*v1.Pod, error) { conf.Mem = 50 } + var podAnnotations *PodAnnotation = nil + if len(conf.UserName) != 0 || len(conf.GroupNames) != 0 { + podAnnotations = &PodAnnotation{} + if len(conf.UserName) != 0 { + podAnnotations.Info.User = conf.UserName + } + if len(conf.GroupNames) != 0 { + podAnnotations.Info.Groups = conf.GroupNames + } + } + var owners []metav1.OwnerReference affinity := &v1.Affinity{} if conf.RequiredNode != "" { @@ -125,6 +138,9 @@ func InitSleepPod(conf SleepPodConfig) (*v1.Pod, error) { OwnerReferences: owners, } + if podAnnotations != nil { + testPodConfig.Annotations = podAnnotations + } return InitTestPod(testPodConfig) } diff --git a/test/e2e/framework/helpers/yunikorn/rest_api_utils.go b/test/e2e/framework/helpers/yunikorn/rest_api_utils.go index 26289dff1..4961e0262 100644 --- a/test/e2e/framework/helpers/yunikorn/rest_api_utils.go +++ b/test/e2e/framework/helpers/yunikorn/rest_api_utils.go @@ -398,6 +398,7 @@ func (c *RClient) GetPartitions(partition string) (*dao.PartitionQueueDAOInfo, e _, err = c.do(req, &partitions) return partitions, err } +<<<<<<< HEAD func (c *RClient) GetUserUsage(partition string, userName string) (*dao.UserResourceUsageDAOInfo, error) { req, err := c.newRequest("GET", fmt.Sprintf(configmanager.UserUsagePath, partition, userName), nil) @@ -409,6 +410,8 @@ func (c *RClient) GetUserUsage(partition string, userName string) (*dao.UserReso return userUsage, err } + + func (c *RClient) GetGroupUsage(partition string, groupName string) (*dao.GroupResourceUsageDAOInfo, error) { req, err := c.newRequest("GET", fmt.Sprintf(configmanager.GroupUsagePath, partition, groupName), nil) if err != nil { @@ -418,3 +421,23 @@ func (c *RClient) GetGroupUsage(partition string, groupName string) (*dao.GroupR _, err = c.do(req, &groupUsage) return groupUsage, err } + +func (c *RClient) GetUserssage(partition string) ([]*dao.UserResourceUsageDAOInfo, error) { + req, err := c.newRequest("GET", fmt.Sprintf(configmanager.UsersTrackerPath, partition), nil) + if err != nil { + return nil, err + } + var usersUsage []*dao.UserResourceUsageDAOInfo + _, err = c.do(req, usersUsage) + return usersUsage, err +} + +func (c *RClient) GetGroupsUsage(partition string) ([]*dao.GroupResourceUsageDAOInfo, error) { + req, err := c.newRequest("GET", fmt.Sprintf(configmanager.GroupsTrackerPath, partition), nil) + if err != nil { + return nil, err + } + var groupsUsage []*dao.GroupResourceUsageDAOInfo + _, err = c.do(req, groupsUsage) + return groupsUsage, err +} diff --git a/test/e2e/user_qauta_tracing/user_qauta_tracing_suite_test.go b/test/e2e/user_qauta_tracing/user_qauta_tracing_suite_test.go new file mode 100644 index 000000000..a52b45213 --- /dev/null +++ b/test/e2e/user_qauta_tracing/user_qauta_tracing_suite_test.go @@ -0,0 +1,29 @@ +package user_qauta_tracing_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + v1 "k8s.io/api/core/v1" + + "github.com/apache/yunikorn-k8shim/test/e2e/framework/configmanager" + "github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/common" + "github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/k8s" + "github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/yunikorn" +) + +func init() { + configmanager.YuniKornTestConfig.ParseFlags() +} + +var oldConfigMap = new(v1.ConfigMap) +var annotation = "ann-" + common.RandSeq(10) +var kClient = k8s.KubeCtl{} //nolint + +var _ = BeforeSuite(func() { + Ω(kClient.SetClient()).To(BeNil()) + yunikorn.EnsureYuniKornConfigsPresent() +}) + +var _ = AfterSuite(func() { + yunikorn.RestoreConfigMapWrapper(oldConfigMap, annotation) +}) diff --git a/test/e2e/user_qauta_tracing/user_qauta_tracing_test.go b/test/e2e/user_qauta_tracing/user_qauta_tracing_test.go new file mode 100644 index 000000000..e4ff905ba --- /dev/null +++ b/test/e2e/user_qauta_tracing/user_qauta_tracing_test.go @@ -0,0 +1,105 @@ +package user_qauta_tracing_test + +import ( + "fmt" + "strings" + + v1 "k8s.io/api/core/v1" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/apache/yunikorn-core/pkg/common/configs" + tests "github.com/apache/yunikorn-k8shim/test/e2e" + "github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/common" + "github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/k8s" + "github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/yunikorn" +) + +const ( + NANESPACE_LENGTH = 10 + WAIT_INTERVAL = 60 + DEFAULT_PARTITION = "default" + ROOT_QUEUE = "root" + DOT = "." +) + +var _ = Describe("UserQuataTracing: Two leaf queus for two groups", func() { + var ns string + BeforeEach(func() { + ns = "ns-" + common.RandSeq(NANESPACE_LENGTH) + By(fmt.Sprintf("Create namespace: %s", ns)) + var ns1, err1 = kClient.CreateNamespace(ns, nil) + Ω(err1).NotTo(HaveOccurred()) + Ω(ns1.Status.Phase).To(Equal(v1.NamespaceActive)) + }) + + It("User qauta trace with 3 users and 2 groups", func() { + yunikorn.UpdateCustomConfigMapWrapper(oldConfigMap, "", annotation, func(sc *configs.SchedulerConfig) error { + // remove placement rules so we can control queue + sc.Partitions[0].PlacementRules = nil + queuesConfigs := []struct { + partition, parentQueue, QueueName string + }{ + {DEFAULT_PARTITION, ROOT_QUEUE, "students_resources"}, + {DEFAULT_PARTITION, ROOT_QUEUE, "staff_resources"}, + } + for _, queueConfig := range queuesConfigs { + By(fmt.Sprintf("Add child queue %s to the parent queue %s", queueConfig.QueueName, queueConfig.parentQueue)) + config := configs.QueueConfig{ + Name: queueConfig.QueueName, + } + if err := common.AddQueue(sc, queueConfig.partition, queueConfig.parentQueue, config); err != nil { + return err + } + } + return nil + }) + + configs := []struct { + AppID, Queue, UserName string + GroupsNames []string + }{ + {"teacher-app-01", strings.Join([]string{ROOT_QUEUE, "staff_resources"}, DOT), "teacher", []string{"staff"}}, + {"students-app-01", strings.Join([]string{ROOT_QUEUE, "students_resources"}, DOT), "student", []string{"students"}}, + {"assistant-app-01", strings.Join([]string{ROOT_QUEUE, "students_resources"}, DOT), "assistant", []string{"students"}}, + {"assistant-app-02", strings.Join([]string{ROOT_QUEUE, "staff_resources"}, DOT), "assistant", []string{"staff"}}, + } + for _, config := range configs { + By(fmt.Sprintf("Deploy the sleep app %s to the %s namespace", config.AppID, ns)) + sleepObj, podErr := k8s.InitSleepPod(k8s.SleepPodConfig{ + AppID: config.AppID, + NS: ns, + UserName: config.UserName, + GroupNames: config.GroupsNames, + Labels: map[string]string{"queue": config.Queue}, + }) + Ω(podErr).NotTo(HaveOccurred()) + sleepRespPod, podErr := kClient.CreatePod(sleepObj, ns) + Ω(podErr).NotTo(HaveOccurred()) + + // Wait for pod to move to running state + podErr = kClient.WaitForPodBySelectorRunning(ns, + fmt.Sprintf("app=%s", sleepRespPod.ObjectMeta.Labels["app"]), WAIT_INTERVAL) + Ω(podErr).NotTo(HaveOccurred()) + } + restClient := yunikorn.RClient{} + users, err := restClient.GetUsersResourceUsage(DEFAULT_PARTITION) + Ω(err).NotTo(HaveOccurred()) + Ω(len(users)).To(Equal(3), "Total number of users is not correct") + groups, err := restClient.GetGroupsResourceUsage(DEFAULT_PARTITION) + Ω(err).NotTo(HaveOccurred()) + Ω(len(groups)).To(Equal(2), "Total number of groups is not correct") + }) + + AfterEach(func() { + testDescription := CurrentSpecReport() + if testDescription.Failed() { + tests.LogTestClusterInfoWrapper(testDescription.FailureMessage(), []string{ns}) + tests.LogYunikornContainer(testDescription.FailureMessage()) + } + By("Tear down namespace: " + ns) + err := kClient.DeleteNamespace(ns) + Ω(err).NotTo(HaveOccurred()) + }) +}) From 8e165362c0f31643706c478221b19e9911120684 Mon Sep 17 00:00:00 2001 From: 0yukali0 Date: Tue, 15 Aug 2023 20:01:11 +0800 Subject: [PATCH 02/16] [YUNIKORN-1901] A basic example for the user tracing and the group tracing --- .../framework/helpers/k8s/pod_annotation.go | 4 +- .../helpers/yunikorn/rest_api_utils.go | 73 ++++++++++++++++- .../user_qauta_tracing_test.go | 78 +++++++++++++++---- 3 files changed, 137 insertions(+), 18 deletions(-) diff --git a/test/e2e/framework/helpers/k8s/pod_annotation.go b/test/e2e/framework/helpers/k8s/pod_annotation.go index a27f86c2f..d2a3967af 100644 --- a/test/e2e/framework/helpers/k8s/pod_annotation.go +++ b/test/e2e/framework/helpers/k8s/pod_annotation.go @@ -29,8 +29,8 @@ type PodAnnotation struct { } type UserInfo struct { - User string `"json:user,omitempty"` - Groups []string `"json:groups,omitempty"` + User string `json:"username,omitempty"` + Groups []string `json:"groups,omitempty"` } const ( diff --git a/test/e2e/framework/helpers/yunikorn/rest_api_utils.go b/test/e2e/framework/helpers/yunikorn/rest_api_utils.go index 4961e0262..284fe50dc 100644 --- a/test/e2e/framework/helpers/yunikorn/rest_api_utils.go +++ b/test/e2e/framework/helpers/yunikorn/rest_api_utils.go @@ -398,7 +398,6 @@ func (c *RClient) GetPartitions(partition string) (*dao.PartitionQueueDAOInfo, e _, err = c.do(req, &partitions) return partitions, err } -<<<<<<< HEAD func (c *RClient) GetUserUsage(partition string, userName string) (*dao.UserResourceUsageDAOInfo, error) { req, err := c.newRequest("GET", fmt.Sprintf(configmanager.UserUsagePath, partition, userName), nil) @@ -425,19 +424,87 @@ func (c *RClient) GetGroupUsage(partition string, groupName string) (*dao.GroupR func (c *RClient) GetUserssage(partition string) ([]*dao.UserResourceUsageDAOInfo, error) { req, err := c.newRequest("GET", fmt.Sprintf(configmanager.UsersTrackerPath, partition), nil) if err != nil { - return nil, err + return nil, err } var usersUsage []*dao.UserResourceUsageDAOInfo _, err = c.do(req, usersUsage) return usersUsage, err } +<<<<<<< HEAD func (c *RClient) GetGroupsUsage(partition string) ([]*dao.GroupResourceUsageDAOInfo, error) { +======= +func (c *RClient) GetUserResourceUsage(partition string, user string) (*dao.UserResourceUsageDAOInfo, error) { + req, err := c.newRequest("GET", fmt.Sprintf(configmanager.UserTrackerPath, partition, user), nil) + if err != nil { + return nil, err + } + var userUsage *dao.UserResourceUsageDAOInfo + _, err = c.do(req, userUsage) + return userUsage, err +} + +func (c *RClient) GetGroupsResourceUsage(partition string) ([]*dao.GroupResourceUsageDAOInfo, error) { +>>>>>>> 97e86d38 ([YUNIKORN-1901] A basic example for the user tracing and the group tracing) req, err := c.newRequest("GET", fmt.Sprintf(configmanager.GroupsTrackerPath, partition), nil) if err != nil { - return nil, err + return nil, err } +<<<<<<< HEAD var groupsUsage []*dao.GroupResourceUsageDAOInfo _, err = c.do(req, groupsUsage) return groupsUsage, err +======= + var gourpsUsage []*dao.GroupResourceUsageDAOInfo + _, err = c.do(req, gourpsUsage) + return gourpsUsage, err +} + +func (c *RClient) GetGroupResourceUsage(partition string, user string) (*dao.GroupResourceUsageDAOInfo, error) { + req, err := c.newRequest("GET", fmt.Sprintf(configmanager.GroupTrackerPath, partition, user), nil) + if err != nil { + return nil, err + } + var groupUsage *dao.GroupResourceUsageDAOInfo + _, err = c.do(req, groupUsage) + return groupUsage, err +>>>>>>> 97e86d38 ([YUNIKORN-1901] A basic example for the user tracing and the group tracing) +} + +func (c *RClient) GetQueueFromUserResourceUsage(usages []*dao.UserResourceUsageDAOInfo, queueName string, user string) (*dao.ResourceUsageDAOInfo, error) { + var result *dao.ResourceUsageDAOInfo + for _, usage := range usages { + if usage.UserName == user { + result = usage.Queues + } + } + return QueueFromResourceUsage(result, queueName) +} + +func (c *RClient) GetQueueFromGroupResourceUsage(usages []*dao.GroupResourceUsageDAOInfo, queueName string, group string) (*dao.ResourceUsageDAOInfo, error) { + var result *dao.ResourceUsageDAOInfo + for _, usage := range usages { + if usage.GroupName == group { + result = usage.Queues + } + } + return QueueFromResourceUsage(result, queueName) +} + +func QueueFromResourceUsage(root *dao.ResourceUsageDAOInfo, queueName string) (*dao.ResourceUsageDAOInfo, error) { + if root == nil { + return nil, fmt.Errorf("ResourceUsage not found: %s", queueName) + } + + if queueName == "root" { + return root, nil + } + + var allSubQueues = root.Children + for _, subQ := range allSubQueues { + if subQ.QueuePath == queueName { + return subQ, nil + } + } + return nil, fmt.Errorf("ResourceUsage not found: %s", queueName) } diff --git a/test/e2e/user_qauta_tracing/user_qauta_tracing_test.go b/test/e2e/user_qauta_tracing/user_qauta_tracing_test.go index e4ff905ba..11f43df5d 100644 --- a/test/e2e/user_qauta_tracing/user_qauta_tracing_test.go +++ b/test/e2e/user_qauta_tracing/user_qauta_tracing_test.go @@ -2,7 +2,6 @@ package user_qauta_tracing_test import ( "fmt" - "strings" v1 "k8s.io/api/core/v1" @@ -10,18 +9,18 @@ import ( . "github.com/onsi/gomega" "github.com/apache/yunikorn-core/pkg/common/configs" + "github.com/apache/yunikorn-core/pkg/common/resources" tests "github.com/apache/yunikorn-k8shim/test/e2e" "github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/common" "github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/k8s" "github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/yunikorn" + siCommon "github.com/apache/yunikorn-scheduler-interface/lib/go/common" ) const ( NANESPACE_LENGTH = 10 WAIT_INTERVAL = 60 DEFAULT_PARTITION = "default" - ROOT_QUEUE = "root" - DOT = "." ) var _ = Describe("UserQuataTracing: Two leaf queus for two groups", func() { @@ -35,14 +34,17 @@ var _ = Describe("UserQuataTracing: Two leaf queus for two groups", func() { }) It("User qauta trace with 3 users and 2 groups", func() { + groups := []string{"staff", "stduent"} + users := []string{"teacher", "student", "assistant"} + queuePath := []string{"root", "root.staff_resources", "root.students_resources"} yunikorn.UpdateCustomConfigMapWrapper(oldConfigMap, "", annotation, func(sc *configs.SchedulerConfig) error { // remove placement rules so we can control queue sc.Partitions[0].PlacementRules = nil queuesConfigs := []struct { partition, parentQueue, QueueName string }{ - {DEFAULT_PARTITION, ROOT_QUEUE, "students_resources"}, - {DEFAULT_PARTITION, ROOT_QUEUE, "staff_resources"}, + {DEFAULT_PARTITION, queuePath[0], "students_resources"}, + {DEFAULT_PARTITION, queuePath[0], "staff_resources"}, } for _, queueConfig := range queuesConfigs { By(fmt.Sprintf("Add child queue %s to the parent queue %s", queueConfig.QueueName, queueConfig.parentQueue)) @@ -60,10 +62,10 @@ var _ = Describe("UserQuataTracing: Two leaf queus for two groups", func() { AppID, Queue, UserName string GroupsNames []string }{ - {"teacher-app-01", strings.Join([]string{ROOT_QUEUE, "staff_resources"}, DOT), "teacher", []string{"staff"}}, - {"students-app-01", strings.Join([]string{ROOT_QUEUE, "students_resources"}, DOT), "student", []string{"students"}}, - {"assistant-app-01", strings.Join([]string{ROOT_QUEUE, "students_resources"}, DOT), "assistant", []string{"students"}}, - {"assistant-app-02", strings.Join([]string{ROOT_QUEUE, "staff_resources"}, DOT), "assistant", []string{"staff"}}, + {"teacher-app-01", queuePath[1], users[0], []string{groups[0]}}, + {"students-app-01", queuePath[2], users[1], []string{groups[1]}}, + {"assistant-app-01", queuePath[2], users[2], []string{groups[1]}}, + {"assistant-app-02", queuePath[1], users[2], []string{groups[0]}}, } for _, config := range configs { By(fmt.Sprintf("Deploy the sleep app %s to the %s namespace", config.AppID, ns)) @@ -84,12 +86,54 @@ var _ = Describe("UserQuataTracing: Two leaf queus for two groups", func() { Ω(podErr).NotTo(HaveOccurred()) } restClient := yunikorn.RClient{} - users, err := restClient.GetUsersResourceUsage(DEFAULT_PARTITION) + var usedResource yunikorn.ResourceUsage + By("Check total number of users and groups") + usersUsage, err := restClient.GetUsersResourceUsage(DEFAULT_PARTITION) Ω(err).NotTo(HaveOccurred()) - Ω(len(users)).To(Equal(3), "Total number of users is not correct") - groups, err := restClient.GetGroupsResourceUsage(DEFAULT_PARTITION) + Ω(len(usersUsage)).To(Equal(3), "Total number of users is not correct") + groupsUsage, err := restClient.GetGroupsResourceUsage(DEFAULT_PARTITION) Ω(err).NotTo(HaveOccurred()) - Ω(len(groups)).To(Equal(2), "Total number of groups is not correct") + Ω(len(groupsUsage)).To(Equal(2), "Total number of groups is not correct") + + By("Check user resource usage of assistant in each queue") + userUsage, err := restClient.GetUserResourceUsage(DEFAULT_PARTITION, users[2]) + Ω(err).NotTo(HaveOccurred()) + Ω(userUsage).To(Equal(map[string]string{configs[2].AppID: groups[1], configs[3].AppID: groups[0]})) + queue, err := restClient.GetQueueFromUserResourceUsage(usersUsage, queuePath[0], users[2]) + Ω(err).NotTo(HaveOccurred()) + Ω(queue.RunningApplications).To(Equal([]string{configs[2].AppID, configs[3].AppID})) + usedResource.ParseResourceUsage(parseResource(queue.ResourceUsage)) + Ω(usedResource).To(Equal(map[string]int64{siCommon.CPU: 200, siCommon.Memory: 100})) + queue, err = restClient.GetQueueFromUserResourceUsage(usersUsage, queuePath[1], users[2]) + Ω(err).NotTo(HaveOccurred()) + Ω(queue.RunningApplications).To(Equal([]string{configs[3].AppID})) + usedResource.ParseResourceUsage(parseResource(queue.ResourceUsage)) + Ω(usedResource).To(Equal(map[string]int64{siCommon.CPU: 100, siCommon.Memory: 50})) + queue, err = restClient.GetQueueFromUserResourceUsage(usersUsage, queuePath[2], users[2]) + Ω(err).NotTo(HaveOccurred()) + Ω(queue.RunningApplications).To(Equal([]string{configs[2].AppID})) + usedResource.ParseResourceUsage(parseResource(queue.ResourceUsage)) + Ω(usedResource).To(Equal(map[string]int64{siCommon.CPU: 100, siCommon.Memory: 50})) + + By("Check group resource usage of staff in each queue") + groupUsage, err := restClient.GetGroupResourceUsage(DEFAULT_PARTITION, groups[0]) + Ω(err).NotTo(HaveOccurred()) + Ω(groupUsage.Applications).To(Equal([]string{configs[0].AppID, configs[3].AppID}), "running application IDs are not expected") + queue, err = restClient.GetQueueFromGroupResourceUsage(groupsUsage, queuePath[0], groups[0]) + Ω(err).NotTo(HaveOccurred()) + Ω(queue.RunningApplications).To(Equal([]string{configs[0].AppID, configs[3].AppID})) + usedResource.ParseResourceUsage(parseResource(queue.ResourceUsage)) + Ω(usedResource).To(Equal(map[string]int64{siCommon.CPU: 200, siCommon.Memory: 100})) + queue, err = restClient.GetQueueFromGroupResourceUsage(groupsUsage, queuePath[1], groups[0]) + Ω(err).NotTo(HaveOccurred()) + Ω(queue.RunningApplications).To(Equal([]string{configs[0].AppID, configs[3].AppID})) + usedResource.ParseResourceUsage(parseResource(queue.ResourceUsage)) + Ω(usedResource).To(Equal(map[string]int64{siCommon.CPU: 200, siCommon.Memory: 100})) + queue, err = restClient.GetQueueFromGroupResourceUsage(groupsUsage, queuePath[2], groups[0]) + Ω(err).NotTo(HaveOccurred()) + Ω(queue.RunningApplications).To(Equal([]string{})) + usedResource.ParseResourceUsage(parseResource(queue.ResourceUsage)) + Ω(usedResource).To(Equal(map[string]int64{siCommon.CPU: 0, siCommon.Memory: 0})) }) AfterEach(func() { @@ -103,3 +147,11 @@ var _ = Describe("UserQuataTracing: Two leaf queus for two groups", func() { Ω(err).NotTo(HaveOccurred()) }) }) + +func parseResource(res *resources.Resource) map[string]int64 { + result := make(map[string]int64) + for key, value := range res.Resources { + result[key] = int64(value) + } + return result +} From 62d9af3c74768e0b5562043e80062ed10878da38 Mon Sep 17 00:00:00 2001 From: 0yukali0 Date: Tue, 15 Aug 2023 20:06:09 +0800 Subject: [PATCH 03/16] [YUNIKORN-1901] A basic example for the user tracing and the group tracing --- .../user_qauta_tracing_suite_test.go | 18 ++++++++++++++++++ .../user_qauta_tracing_test.go | 18 ++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/test/e2e/user_qauta_tracing/user_qauta_tracing_suite_test.go b/test/e2e/user_qauta_tracing/user_qauta_tracing_suite_test.go index a52b45213..dc9cf69e1 100644 --- a/test/e2e/user_qauta_tracing/user_qauta_tracing_suite_test.go +++ b/test/e2e/user_qauta_tracing/user_qauta_tracing_suite_test.go @@ -1,3 +1,21 @@ +/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + package user_qauta_tracing_test import ( diff --git a/test/e2e/user_qauta_tracing/user_qauta_tracing_test.go b/test/e2e/user_qauta_tracing/user_qauta_tracing_test.go index 11f43df5d..14e55fa27 100644 --- a/test/e2e/user_qauta_tracing/user_qauta_tracing_test.go +++ b/test/e2e/user_qauta_tracing/user_qauta_tracing_test.go @@ -1,3 +1,21 @@ +/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + package user_qauta_tracing_test import ( From 3de70cc97eaf01124ba7e48a3fe74e1da33c3b72 Mon Sep 17 00:00:00 2001 From: 0yukali0 Date: Wed, 16 Aug 2023 07:16:02 +0800 Subject: [PATCH 04/16] [YUNIKORN-1901] A basic example for the user tracing and the group tracing --- test/e2e/framework/helpers/k8s/pod_annotation.go | 6 ------ test/e2e/framework/helpers/k8s/pod_conf.go | 16 ---------------- .../user_qauta_tracing_test.go | 3 +-- 3 files changed, 1 insertion(+), 24 deletions(-) diff --git a/test/e2e/framework/helpers/k8s/pod_annotation.go b/test/e2e/framework/helpers/k8s/pod_annotation.go index d2a3967af..a014697d1 100644 --- a/test/e2e/framework/helpers/k8s/pod_annotation.go +++ b/test/e2e/framework/helpers/k8s/pod_annotation.go @@ -24,15 +24,9 @@ type PodAnnotation struct { TaskGroupName string `json:"yunikorn.apache.org/task-group-name,omitempty"` TaskGroups []interfaces.TaskGroup `json:"-"` SchedulingPolicyParams string `json:"yunikorn.apache.org/schedulingPolicyParameters,omitempty"` - Info UserInfo `json:"yunikorn.apache.org/user.info,omitempty"` Other map[string]string `json:"-"` } -type UserInfo struct { - User string `json:"username,omitempty"` - Groups []string `json:"groups,omitempty"` -} - const ( TaskGroupName = "yunikorn.apache.org/task-group-name" TaskGroups = "yunikorn.apache.org/task-groups" diff --git a/test/e2e/framework/helpers/k8s/pod_conf.go b/test/e2e/framework/helpers/k8s/pod_conf.go index 69ac96cf3..5da113eea 100644 --- a/test/e2e/framework/helpers/k8s/pod_conf.go +++ b/test/e2e/framework/helpers/k8s/pod_conf.go @@ -41,8 +41,6 @@ type SleepPodConfig struct { RequiredNode string Optedout bool Labels map[string]string - UserName string - GroupNames []string } // TestPodConfig template for sleepPods @@ -64,17 +62,6 @@ func InitSleepPod(conf SleepPodConfig) (*v1.Pod, error) { conf.Mem = 50 } - var podAnnotations *PodAnnotation = nil - if len(conf.UserName) != 0 || len(conf.GroupNames) != 0 { - podAnnotations = &PodAnnotation{} - if len(conf.UserName) != 0 { - podAnnotations.Info.User = conf.UserName - } - if len(conf.GroupNames) != 0 { - podAnnotations.Info.Groups = conf.GroupNames - } - } - var owners []metav1.OwnerReference affinity := &v1.Affinity{} if conf.RequiredNode != "" { @@ -138,9 +125,6 @@ func InitSleepPod(conf SleepPodConfig) (*v1.Pod, error) { OwnerReferences: owners, } - if podAnnotations != nil { - testPodConfig.Annotations = podAnnotations - } return InitTestPod(testPodConfig) } diff --git a/test/e2e/user_qauta_tracing/user_qauta_tracing_test.go b/test/e2e/user_qauta_tracing/user_qauta_tracing_test.go index 14e55fa27..0d53c676b 100644 --- a/test/e2e/user_qauta_tracing/user_qauta_tracing_test.go +++ b/test/e2e/user_qauta_tracing/user_qauta_tracing_test.go @@ -90,10 +90,9 @@ var _ = Describe("UserQuataTracing: Two leaf queus for two groups", func() { sleepObj, podErr := k8s.InitSleepPod(k8s.SleepPodConfig{ AppID: config.AppID, NS: ns, - UserName: config.UserName, - GroupNames: config.GroupsNames, Labels: map[string]string{"queue": config.Queue}, }) + sleepObj.ObjectMeta.Annotations["yunikorn.apache.org/user.info"] = fmt.Sprintf("{username:%s, groups:{%s}}", config.UserName, config.GroupsNames) Ω(podErr).NotTo(HaveOccurred()) sleepRespPod, podErr := kClient.CreatePod(sleepObj, ns) Ω(podErr).NotTo(HaveOccurred()) From 8c48a3150f525e17c14e5e425624ea8d510fc673 Mon Sep 17 00:00:00 2001 From: 0yukali0 Date: Wed, 16 Aug 2023 07:34:12 +0800 Subject: [PATCH 05/16] [YUNIKORN-1901] A basic example for the user tracing and the group tracing --- test/e2e/user_qauta_tracing/user_qauta_tracing_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/e2e/user_qauta_tracing/user_qauta_tracing_test.go b/test/e2e/user_qauta_tracing/user_qauta_tracing_test.go index 0d53c676b..db69cc3cd 100644 --- a/test/e2e/user_qauta_tracing/user_qauta_tracing_test.go +++ b/test/e2e/user_qauta_tracing/user_qauta_tracing_test.go @@ -88,9 +88,9 @@ var _ = Describe("UserQuataTracing: Two leaf queus for two groups", func() { for _, config := range configs { By(fmt.Sprintf("Deploy the sleep app %s to the %s namespace", config.AppID, ns)) sleepObj, podErr := k8s.InitSleepPod(k8s.SleepPodConfig{ - AppID: config.AppID, - NS: ns, - Labels: map[string]string{"queue": config.Queue}, + AppID: config.AppID, + NS: ns, + Labels: map[string]string{"queue": config.Queue}, }) sleepObj.ObjectMeta.Annotations["yunikorn.apache.org/user.info"] = fmt.Sprintf("{username:%s, groups:{%s}}", config.UserName, config.GroupsNames) Ω(podErr).NotTo(HaveOccurred()) From 46e694b72d73ef187ca2521c42903a5b04357296 Mon Sep 17 00:00:00 2001 From: 0yukali0 Date: Sat, 26 Aug 2023 18:54:25 +0800 Subject: [PATCH 06/16] fix the wrong words --- .../user_qauta_tracing_test.go | 2 +- .../user_quota_tracing_suite_test.go} | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename test/e2e/{user_qauta_tracing => user_quota_tracing}/user_qauta_tracing_test.go (99%) rename test/e2e/{user_qauta_tracing/user_qauta_tracing_suite_test.go => user_quota_tracing/user_quota_tracing_suite_test.go} (97%) diff --git a/test/e2e/user_qauta_tracing/user_qauta_tracing_test.go b/test/e2e/user_quota_tracing/user_qauta_tracing_test.go similarity index 99% rename from test/e2e/user_qauta_tracing/user_qauta_tracing_test.go rename to test/e2e/user_quota_tracing/user_qauta_tracing_test.go index db69cc3cd..ba618f00e 100644 --- a/test/e2e/user_qauta_tracing/user_qauta_tracing_test.go +++ b/test/e2e/user_quota_tracing/user_qauta_tracing_test.go @@ -16,7 +16,7 @@ limitations under the License. */ -package user_qauta_tracing_test +package user_quota_tracing_test import ( "fmt" diff --git a/test/e2e/user_qauta_tracing/user_qauta_tracing_suite_test.go b/test/e2e/user_quota_tracing/user_quota_tracing_suite_test.go similarity index 97% rename from test/e2e/user_qauta_tracing/user_qauta_tracing_suite_test.go rename to test/e2e/user_quota_tracing/user_quota_tracing_suite_test.go index dc9cf69e1..0b807e116 100644 --- a/test/e2e/user_qauta_tracing/user_qauta_tracing_suite_test.go +++ b/test/e2e/user_quota_tracing/user_quota_tracing_suite_test.go @@ -16,7 +16,7 @@ limitations under the License. */ -package user_qauta_tracing_test +package user_quota_tracing_test import ( . "github.com/onsi/ginkgo/v2" From b332b1e9eb3067cdf3f81966ffc5a6328410e585 Mon Sep 17 00:00:00 2001 From: 0yukali0 Date: Mon, 4 Sep 2023 00:05:22 +0800 Subject: [PATCH 07/16] [YUNIKORN-1901] A basic example for the user tracing and the group tracing --- .../helpers/yunikorn/rest_api_utils.go | 33 ++- .../user_qauta_tracing_test.go | 174 ------------ .../user_quota_tracing_test.go | 261 ++++++++++++++++++ 3 files changed, 281 insertions(+), 187 deletions(-) delete mode 100644 test/e2e/user_quota_tracing/user_qauta_tracing_test.go create mode 100644 test/e2e/user_quota_tracing/user_quota_tracing_test.go diff --git a/test/e2e/framework/helpers/yunikorn/rest_api_utils.go b/test/e2e/framework/helpers/yunikorn/rest_api_utils.go index 284fe50dc..6af1c1639 100644 --- a/test/e2e/framework/helpers/yunikorn/rest_api_utils.go +++ b/test/e2e/framework/helpers/yunikorn/rest_api_utils.go @@ -31,6 +31,7 @@ import ( v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/wait" + "github.com/apache/yunikorn-core/pkg/common/resources" "github.com/apache/yunikorn-core/pkg/webservice/dao" "github.com/apache/yunikorn-k8shim/test/e2e/framework/configmanager" ) @@ -471,27 +472,25 @@ func (c *RClient) GetGroupResourceUsage(partition string, user string) (*dao.Gro >>>>>>> 97e86d38 ([YUNIKORN-1901] A basic example for the user tracing and the group tracing) } -func (c *RClient) GetQueueFromUserResourceUsage(usages []*dao.UserResourceUsageDAOInfo, queueName string, user string) (*dao.ResourceUsageDAOInfo, error) { - var result *dao.ResourceUsageDAOInfo - for _, usage := range usages { - if usage.UserName == user { - result = usage.Queues +func GetUserUsageFromUsersUsage(users []*dao.UserResourceUsageDAOInfo, target string) (*dao.UserResourceUsageDAOInfo, error) { + for _, user := range users { + if user.UserName == target { + return user, nil } } - return QueueFromResourceUsage(result, queueName) + return nil, fmt.Errorf("UserUsage %s doesn't exist", target) } -func (c *RClient) GetQueueFromGroupResourceUsage(usages []*dao.GroupResourceUsageDAOInfo, queueName string, group string) (*dao.ResourceUsageDAOInfo, error) { - var result *dao.ResourceUsageDAOInfo - for _, usage := range usages { - if usage.GroupName == group { - result = usage.Queues +func GetGroupUsageFromGroupsUsage(groups []*dao.GroupResourceUsageDAOInfo, target string) (*dao.GroupResourceUsageDAOInfo, error) { + for _, group := range groups { + if group.GroupName == target { + return group, nil } } - return QueueFromResourceUsage(result, queueName) + return nil, fmt.Errorf("GroupUsage %s doesn't exist", target) } -func QueueFromResourceUsage(root *dao.ResourceUsageDAOInfo, queueName string) (*dao.ResourceUsageDAOInfo, error) { +func GetQueueResourceUsage(root *dao.ResourceUsageDAOInfo, queueName string) (*dao.ResourceUsageDAOInfo, error) { if root == nil { return nil, fmt.Errorf("ResourceUsage not found: %s", queueName) } @@ -508,3 +507,11 @@ func QueueFromResourceUsage(root *dao.ResourceUsageDAOInfo, queueName string) (* } return nil, fmt.Errorf("ResourceUsage not found: %s", queueName) } + +func ParseResource(res *resources.Resource) map[string]int64 { + result := make(map[string]int64) + for key, value := range res.Resources { + result[key] = int64(value) + } + return result +} diff --git a/test/e2e/user_quota_tracing/user_qauta_tracing_test.go b/test/e2e/user_quota_tracing/user_qauta_tracing_test.go deleted file mode 100644 index ba618f00e..000000000 --- a/test/e2e/user_quota_tracing/user_qauta_tracing_test.go +++ /dev/null @@ -1,174 +0,0 @@ -/* - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package user_quota_tracing_test - -import ( - "fmt" - - v1 "k8s.io/api/core/v1" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - "github.com/apache/yunikorn-core/pkg/common/configs" - "github.com/apache/yunikorn-core/pkg/common/resources" - tests "github.com/apache/yunikorn-k8shim/test/e2e" - "github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/common" - "github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/k8s" - "github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/yunikorn" - siCommon "github.com/apache/yunikorn-scheduler-interface/lib/go/common" -) - -const ( - NANESPACE_LENGTH = 10 - WAIT_INTERVAL = 60 - DEFAULT_PARTITION = "default" -) - -var _ = Describe("UserQuataTracing: Two leaf queus for two groups", func() { - var ns string - BeforeEach(func() { - ns = "ns-" + common.RandSeq(NANESPACE_LENGTH) - By(fmt.Sprintf("Create namespace: %s", ns)) - var ns1, err1 = kClient.CreateNamespace(ns, nil) - Ω(err1).NotTo(HaveOccurred()) - Ω(ns1.Status.Phase).To(Equal(v1.NamespaceActive)) - }) - - It("User qauta trace with 3 users and 2 groups", func() { - groups := []string{"staff", "stduent"} - users := []string{"teacher", "student", "assistant"} - queuePath := []string{"root", "root.staff_resources", "root.students_resources"} - yunikorn.UpdateCustomConfigMapWrapper(oldConfigMap, "", annotation, func(sc *configs.SchedulerConfig) error { - // remove placement rules so we can control queue - sc.Partitions[0].PlacementRules = nil - queuesConfigs := []struct { - partition, parentQueue, QueueName string - }{ - {DEFAULT_PARTITION, queuePath[0], "students_resources"}, - {DEFAULT_PARTITION, queuePath[0], "staff_resources"}, - } - for _, queueConfig := range queuesConfigs { - By(fmt.Sprintf("Add child queue %s to the parent queue %s", queueConfig.QueueName, queueConfig.parentQueue)) - config := configs.QueueConfig{ - Name: queueConfig.QueueName, - } - if err := common.AddQueue(sc, queueConfig.partition, queueConfig.parentQueue, config); err != nil { - return err - } - } - return nil - }) - - configs := []struct { - AppID, Queue, UserName string - GroupsNames []string - }{ - {"teacher-app-01", queuePath[1], users[0], []string{groups[0]}}, - {"students-app-01", queuePath[2], users[1], []string{groups[1]}}, - {"assistant-app-01", queuePath[2], users[2], []string{groups[1]}}, - {"assistant-app-02", queuePath[1], users[2], []string{groups[0]}}, - } - for _, config := range configs { - By(fmt.Sprintf("Deploy the sleep app %s to the %s namespace", config.AppID, ns)) - sleepObj, podErr := k8s.InitSleepPod(k8s.SleepPodConfig{ - AppID: config.AppID, - NS: ns, - Labels: map[string]string{"queue": config.Queue}, - }) - sleepObj.ObjectMeta.Annotations["yunikorn.apache.org/user.info"] = fmt.Sprintf("{username:%s, groups:{%s}}", config.UserName, config.GroupsNames) - Ω(podErr).NotTo(HaveOccurred()) - sleepRespPod, podErr := kClient.CreatePod(sleepObj, ns) - Ω(podErr).NotTo(HaveOccurred()) - - // Wait for pod to move to running state - podErr = kClient.WaitForPodBySelectorRunning(ns, - fmt.Sprintf("app=%s", sleepRespPod.ObjectMeta.Labels["app"]), WAIT_INTERVAL) - Ω(podErr).NotTo(HaveOccurred()) - } - restClient := yunikorn.RClient{} - var usedResource yunikorn.ResourceUsage - By("Check total number of users and groups") - usersUsage, err := restClient.GetUsersResourceUsage(DEFAULT_PARTITION) - Ω(err).NotTo(HaveOccurred()) - Ω(len(usersUsage)).To(Equal(3), "Total number of users is not correct") - groupsUsage, err := restClient.GetGroupsResourceUsage(DEFAULT_PARTITION) - Ω(err).NotTo(HaveOccurred()) - Ω(len(groupsUsage)).To(Equal(2), "Total number of groups is not correct") - - By("Check user resource usage of assistant in each queue") - userUsage, err := restClient.GetUserResourceUsage(DEFAULT_PARTITION, users[2]) - Ω(err).NotTo(HaveOccurred()) - Ω(userUsage).To(Equal(map[string]string{configs[2].AppID: groups[1], configs[3].AppID: groups[0]})) - queue, err := restClient.GetQueueFromUserResourceUsage(usersUsage, queuePath[0], users[2]) - Ω(err).NotTo(HaveOccurred()) - Ω(queue.RunningApplications).To(Equal([]string{configs[2].AppID, configs[3].AppID})) - usedResource.ParseResourceUsage(parseResource(queue.ResourceUsage)) - Ω(usedResource).To(Equal(map[string]int64{siCommon.CPU: 200, siCommon.Memory: 100})) - queue, err = restClient.GetQueueFromUserResourceUsage(usersUsage, queuePath[1], users[2]) - Ω(err).NotTo(HaveOccurred()) - Ω(queue.RunningApplications).To(Equal([]string{configs[3].AppID})) - usedResource.ParseResourceUsage(parseResource(queue.ResourceUsage)) - Ω(usedResource).To(Equal(map[string]int64{siCommon.CPU: 100, siCommon.Memory: 50})) - queue, err = restClient.GetQueueFromUserResourceUsage(usersUsage, queuePath[2], users[2]) - Ω(err).NotTo(HaveOccurred()) - Ω(queue.RunningApplications).To(Equal([]string{configs[2].AppID})) - usedResource.ParseResourceUsage(parseResource(queue.ResourceUsage)) - Ω(usedResource).To(Equal(map[string]int64{siCommon.CPU: 100, siCommon.Memory: 50})) - - By("Check group resource usage of staff in each queue") - groupUsage, err := restClient.GetGroupResourceUsage(DEFAULT_PARTITION, groups[0]) - Ω(err).NotTo(HaveOccurred()) - Ω(groupUsage.Applications).To(Equal([]string{configs[0].AppID, configs[3].AppID}), "running application IDs are not expected") - queue, err = restClient.GetQueueFromGroupResourceUsage(groupsUsage, queuePath[0], groups[0]) - Ω(err).NotTo(HaveOccurred()) - Ω(queue.RunningApplications).To(Equal([]string{configs[0].AppID, configs[3].AppID})) - usedResource.ParseResourceUsage(parseResource(queue.ResourceUsage)) - Ω(usedResource).To(Equal(map[string]int64{siCommon.CPU: 200, siCommon.Memory: 100})) - queue, err = restClient.GetQueueFromGroupResourceUsage(groupsUsage, queuePath[1], groups[0]) - Ω(err).NotTo(HaveOccurred()) - Ω(queue.RunningApplications).To(Equal([]string{configs[0].AppID, configs[3].AppID})) - usedResource.ParseResourceUsage(parseResource(queue.ResourceUsage)) - Ω(usedResource).To(Equal(map[string]int64{siCommon.CPU: 200, siCommon.Memory: 100})) - queue, err = restClient.GetQueueFromGroupResourceUsage(groupsUsage, queuePath[2], groups[0]) - Ω(err).NotTo(HaveOccurred()) - Ω(queue.RunningApplications).To(Equal([]string{})) - usedResource.ParseResourceUsage(parseResource(queue.ResourceUsage)) - Ω(usedResource).To(Equal(map[string]int64{siCommon.CPU: 0, siCommon.Memory: 0})) - }) - - AfterEach(func() { - testDescription := CurrentSpecReport() - if testDescription.Failed() { - tests.LogTestClusterInfoWrapper(testDescription.FailureMessage(), []string{ns}) - tests.LogYunikornContainer(testDescription.FailureMessage()) - } - By("Tear down namespace: " + ns) - err := kClient.DeleteNamespace(ns) - Ω(err).NotTo(HaveOccurred()) - }) -}) - -func parseResource(res *resources.Resource) map[string]int64 { - result := make(map[string]int64) - for key, value := range res.Resources { - result[key] = int64(value) - } - return result -} diff --git a/test/e2e/user_quota_tracing/user_quota_tracing_test.go b/test/e2e/user_quota_tracing/user_quota_tracing_test.go new file mode 100644 index 000000000..3af2836fd --- /dev/null +++ b/test/e2e/user_quota_tracing/user_quota_tracing_test.go @@ -0,0 +1,261 @@ +/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package user_quota_tracing_test + +import ( + "fmt" + + v1 "k8s.io/api/core/v1" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/apache/yunikorn-core/pkg/common/configs" + tests "github.com/apache/yunikorn-k8shim/test/e2e" + "github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/common" + "github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/k8s" + "github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/yunikorn" + siCommon "github.com/apache/yunikorn-scheduler-interface/lib/go/common" +) + +const ( + NANESPACE_LENGTH = 10 + WAIT_INTERVAL = 60 + DEFAULT_PARTITION = "default" +) + +var _ = Describe("QuotaTracking: Two leaf queus for two groups", func() { + var ns string + BeforeEach(func() { + ns = "ns-" + common.RandSeq(NANESPACE_LENGTH) + By(fmt.Sprintf("Create namespace: %s", ns)) + var ns1, err1 = kClient.CreateNamespace(ns, nil) + Ω(err1).NotTo(HaveOccurred()) + Ω(ns1.Status.Phase).To(Equal(v1.NamespaceActive)) + }) + + It("User qauta trace with 3 users and 2 groups", func() { + groups := []string{"group1", "group2"} + users := []string{"user1", "user2", "user3"} + queuePath := []string{"root", "root.group1_resources", "root.group2_resources"} + zeroPod := map[string]int64{siCommon.CPU: 0, siCommon.Memory: 0} + onePod := map[string]int64{siCommon.CPU: 100, siCommon.Memory: 50} + twoPods := map[string]int64{siCommon.CPU: 200, siCommon.Memory: 100} + yunikorn.UpdateCustomConfigMapWrapper(oldConfigMap, "", annotation, func(sc *configs.SchedulerConfig) error { + // remove placement rules so we can control queue + sc.Partitions[0].PlacementRules = nil + queuesConfigs := []struct { + partition, parentQueue, QueueName string + }{ + {DEFAULT_PARTITION, queuePath[0], "group1_resources"}, + {DEFAULT_PARTITION, queuePath[0], "group2_resources"}, + } + for _, queueConfig := range queuesConfigs { + By(fmt.Sprintf("Add child queue %s to the parent queue %s", queueConfig.QueueName, queueConfig.parentQueue)) + config := configs.QueueConfig{ + Name: queueConfig.QueueName, + } + if err := common.AddQueue(sc, queueConfig.partition, queueConfig.parentQueue, config); err != nil { + return err + } + } + return nil + }) + + /* + * groups: group1, group2 + * users: user1, user2, user3 + * user1 -> group1 + * user2 -> group2 + * user3 -> group1, group2 + */ + configs := []struct { + AppID, Queue, UserName string + GroupsNames []string + }{ + {"teacher-app-01", queuePath[1], users[0], []string{groups[0]}}, + {"students-app-01", queuePath[2], users[1], []string{groups[1]}}, + {"assistant-app-01", queuePath[2], users[2], []string{groups[1]}}, + {"assistant-app-02", queuePath[1], users[2], []string{groups[0]}}, + } + for _, config := range configs { + By(fmt.Sprintf("Deploy the sleep app %s to the %s namespace", config.AppID, ns)) + sleepObj, podErr := k8s.InitSleepPod(k8s.SleepPodConfig{ + AppID: config.AppID, + NS: ns, + Labels: map[string]string{"queue": config.Queue}, + }) + sleepObj.ObjectMeta.Annotations["yunikorn.apache.org/user.info"] = fmt.Sprintf("{username:%s, groups:{%s}}", config.UserName, config.GroupsNames) + Ω(podErr).NotTo(HaveOccurred()) + sleepRespPod, podErr := kClient.CreatePod(sleepObj, ns) + Ω(podErr).NotTo(HaveOccurred()) + + // Wait for pod to move to running state + podErr = kClient.WaitForPodBySelectorRunning(ns, + fmt.Sprintf("app=%s", sleepRespPod.ObjectMeta.Labels["app"]), WAIT_INTERVAL) + Ω(podErr).NotTo(HaveOccurred()) + } + + type Queue struct { + RunningApplications []string + Resources map[string]int64 + } + type GroupQuota struct { + RunningApplications []string + Queues map[string]Queue + } + type GroupsQuota struct { + NumberOfGroups int + Groups map[string]GroupQuota + } + expectedGroups := GroupsQuota{ + 2, + map[string]GroupQuota{ + groups[0]: { + []string{configs[0].AppID, configs[3].AppID}, + map[string]Queue{ + queuePath[0]: {[]string{configs[0].AppID, configs[3].AppID}, twoPods}, + queuePath[1]: {[]string{configs[0].AppID, configs[3].AppID}, twoPods}, + queuePath[2]: {[]string{}, zeroPod}, + }, + }, + groups[1]: { + []string{configs[1].AppID, configs[2].AppID}, + map[string]Queue{ + queuePath[0]: {[]string{configs[1].AppID, configs[2].AppID}, twoPods}, + queuePath[1]: {[]string{}, zeroPod}, + queuePath[2]: {[]string{configs[1].AppID, configs[2].AppID}, twoPods}, + }, + }, + }, + } + restClient := yunikorn.RClient{} + var usedResource yunikorn.ResourceUsage + By("GroupsTracker: Check total number of groups") + groupsUsage, err := restClient.GetGroupsResourceUsage(DEFAULT_PARTITION) + Ω(err).NotTo(HaveOccurred()) + Ω(len(groupsUsage)).To(Equal(expectedGroups.NumberOfGroups), "Total number of groups is not correct") + for group, quota := range expectedGroups.Groups { + By(fmt.Sprintf("GroupTracker: Check group resource usage of %s in each queue", group)) + groupUsage, getGroupErr := yunikorn.GetGroupUsageFromGroupsUsage(groupsUsage, group) + Ω(getGroupErr).NotTo(HaveOccurred()) + Ω(groupUsage.Applications).To(Equal(quota.RunningApplications), "running application IDs are not expected") + for queuePath, queueQuota := range quota.Queues { + queue, getQueueErr := yunikorn.GetQueueResourceUsage(groupUsage.Queues, queuePath) + Ω(getQueueErr).NotTo(HaveOccurred()) + Ω(queue.RunningApplications).To(Equal(queueQuota.RunningApplications)) + usedResource.ParseResourceUsage(yunikorn.ParseResource(queue.ResourceUsage)) + Ω(usedResource).To(Equal(queueQuota.Resources)) + } + } + + for group, quota := range expectedGroups.Groups { + By(fmt.Sprintf("GroupTracker: Check group resource usage of %s in each queue", group)) + groupUsage, getGroupErr := restClient.GetGroupResourceUsage(DEFAULT_PARTITION, group) + Ω(getGroupErr).NotTo(HaveOccurred()) + Ω(groupUsage.Applications).To(Equal(quota.RunningApplications), "running application IDs are not expected") + for queuePath, queueQuota := range quota.Queues { + queue, getQueueErr := yunikorn.GetQueueResourceUsage(groupUsage.Queues, queuePath) + Ω(getQueueErr).NotTo(HaveOccurred()) + Ω(queue.RunningApplications).To(Equal(queueQuota.RunningApplications)) + usedResource.ParseResourceUsage(yunikorn.ParseResource(queue.ResourceUsage)) + Ω(usedResource).To(Equal(queueQuota.Resources)) + } + } + + type UserQuota struct { + AppBelongingGroup map[string]string + Queues map[string]Queue + } + expectedUsers := struct { + NumberOfUsers int + Users map[string]UserQuota + }{ + 3, + map[string]UserQuota{ + users[0]: { + map[string]string{configs[0].AppID: groups[0]}, + map[string]Queue{ + queuePath[0]: {[]string{configs[0].AppID}, onePod}, + queuePath[1]: {[]string{configs[0].AppID}, onePod}, + queuePath[2]: {[]string{}, zeroPod}, + }, + }, + users[1]: { + map[string]string{configs[1].AppID: groups[1]}, + map[string]Queue{ + queuePath[0]: {[]string{configs[1].AppID}, onePod}, + queuePath[1]: {[]string{configs[1].AppID}, onePod}, + queuePath[2]: {[]string{}, zeroPod}, + }, + }, + users[2]: { + map[string]string{configs[2].AppID: groups[1], configs[3].AppID: groups[0]}, + map[string]Queue{ + queuePath[0]: {[]string{configs[2].AppID, configs[3].AppID}, twoPods}, + queuePath[1]: {[]string{configs[3].AppID}, onePod}, + queuePath[2]: {[]string{configs[2].AppID}, onePod}, + }, + }, + }, + } + By("UsersTrackers: Check total number of users") + usersUsage, err := restClient.GetUsersResourceUsage(DEFAULT_PARTITION) + Ω(err).NotTo(HaveOccurred()) + Ω(len(usersUsage)).To(Equal(expectedUsers.NumberOfUsers), "Total number of users is not correct") + for user, quota := range expectedUsers.Users { + By(fmt.Sprintf("UsersTrackers: Check user resource usage of %s in each queue", user)) + userUsage, getUserErr := yunikorn.GetUserUsageFromUsersUsage(usersUsage, user) + Ω(getUserErr).NotTo(HaveOccurred()) + Ω(userUsage).To(Equal(quota.AppBelongingGroup)) + for queuePath, queueQuota := range quota.Queues { + queue, getQueueerr := yunikorn.GetQueueResourceUsage(userUsage.Queues, queuePath) + Ω(getQueueerr).NotTo(HaveOccurred()) + Ω(queue.RunningApplications).To(Equal(queueQuota.RunningApplications)) + usedResource.ParseResourceUsage(yunikorn.ParseResource(queue.ResourceUsage)) + Ω(usedResource).To(Equal(queueQuota.Resources)) + } + } + + for user, quota := range expectedUsers.Users { + By(fmt.Sprintf("UserTracker: Check user resource usage of %s in each queue", user)) + userUsage, err := restClient.GetUserResourceUsage(DEFAULT_PARTITION, user) + Ω(err).NotTo(HaveOccurred()) + Ω(userUsage).To(Equal(quota.AppBelongingGroup)) + for queuePath, queueQuota := range quota.Queues { + queue, err := yunikorn.GetQueueResourceUsage(userUsage.Queues, queuePath) + Ω(err).NotTo(HaveOccurred()) + Ω(queue.RunningApplications).To(Equal(queueQuota.RunningApplications)) + usedResource.ParseResourceUsage(yunikorn.ParseResource(queue.ResourceUsage)) + Ω(usedResource).To(Equal(queueQuota.Resources)) + } + } + }) + + AfterEach(func() { + testDescription := CurrentSpecReport() + if testDescription.Failed() { + tests.LogTestClusterInfoWrapper(testDescription.FailureMessage(), []string{ns}) + tests.LogYunikornContainer(testDescription.FailureMessage()) + } + By("Tear down namespace: " + ns) + err := kClient.DeleteNamespace(ns) + Ω(err).NotTo(HaveOccurred()) + }) +}) From 496f0fa9b3d5884772bfadd7e2447391990464c1 Mon Sep 17 00:00:00 2001 From: 0yukali0 Date: Mon, 4 Sep 2023 00:08:08 +0800 Subject: [PATCH 08/16] [YUNIKORN-1901] A basic example for the user tracing and the group tracing --- .../user_quota_tracing_test.go | 40 +++++++++---------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/test/e2e/user_quota_tracing/user_quota_tracing_test.go b/test/e2e/user_quota_tracing/user_quota_tracing_test.go index 3af2836fd..b2ba63b70 100644 --- a/test/e2e/user_quota_tracing/user_quota_tracing_test.go +++ b/test/e2e/user_quota_tracing/user_quota_tracing_test.go @@ -147,13 +147,9 @@ var _ = Describe("QuotaTracking: Two leaf queus for two groups", func() { } restClient := yunikorn.RClient{} var usedResource yunikorn.ResourceUsage - By("GroupsTracker: Check total number of groups") - groupsUsage, err := restClient.GetGroupsResourceUsage(DEFAULT_PARTITION) - Ω(err).NotTo(HaveOccurred()) - Ω(len(groupsUsage)).To(Equal(expectedGroups.NumberOfGroups), "Total number of groups is not correct") for group, quota := range expectedGroups.Groups { By(fmt.Sprintf("GroupTracker: Check group resource usage of %s in each queue", group)) - groupUsage, getGroupErr := yunikorn.GetGroupUsageFromGroupsUsage(groupsUsage, group) + groupUsage, getGroupErr := restClient.GetGroupResourceUsage(DEFAULT_PARTITION, group) Ω(getGroupErr).NotTo(HaveOccurred()) Ω(groupUsage.Applications).To(Equal(quota.RunningApplications), "running application IDs are not expected") for queuePath, queueQuota := range quota.Queues { @@ -165,9 +161,13 @@ var _ = Describe("QuotaTracking: Two leaf queus for two groups", func() { } } + By("GroupsTracker: Check total number of groups") + groupsUsage, err := restClient.GetGroupsResourceUsage(DEFAULT_PARTITION) + Ω(err).NotTo(HaveOccurred()) + Ω(len(groupsUsage)).To(Equal(expectedGroups.NumberOfGroups), "Total number of groups is not correct") for group, quota := range expectedGroups.Groups { By(fmt.Sprintf("GroupTracker: Check group resource usage of %s in each queue", group)) - groupUsage, getGroupErr := restClient.GetGroupResourceUsage(DEFAULT_PARTITION, group) + groupUsage, getGroupErr := yunikorn.GetGroupUsageFromGroupsUsage(groupsUsage, group) Ω(getGroupErr).NotTo(HaveOccurred()) Ω(groupUsage.Applications).To(Equal(quota.RunningApplications), "running application IDs are not expected") for queuePath, queueQuota := range quota.Queues { @@ -215,32 +215,32 @@ var _ = Describe("QuotaTracking: Two leaf queus for two groups", func() { }, }, } - By("UsersTrackers: Check total number of users") - usersUsage, err := restClient.GetUsersResourceUsage(DEFAULT_PARTITION) - Ω(err).NotTo(HaveOccurred()) - Ω(len(usersUsage)).To(Equal(expectedUsers.NumberOfUsers), "Total number of users is not correct") for user, quota := range expectedUsers.Users { - By(fmt.Sprintf("UsersTrackers: Check user resource usage of %s in each queue", user)) - userUsage, getUserErr := yunikorn.GetUserUsageFromUsersUsage(usersUsage, user) - Ω(getUserErr).NotTo(HaveOccurred()) + By(fmt.Sprintf("UserTracker: Check user resource usage of %s in each queue", user)) + userUsage, err := restClient.GetUserResourceUsage(DEFAULT_PARTITION, user) + Ω(err).NotTo(HaveOccurred()) Ω(userUsage).To(Equal(quota.AppBelongingGroup)) for queuePath, queueQuota := range quota.Queues { - queue, getQueueerr := yunikorn.GetQueueResourceUsage(userUsage.Queues, queuePath) - Ω(getQueueerr).NotTo(HaveOccurred()) + queue, err := yunikorn.GetQueueResourceUsage(userUsage.Queues, queuePath) + Ω(err).NotTo(HaveOccurred()) Ω(queue.RunningApplications).To(Equal(queueQuota.RunningApplications)) usedResource.ParseResourceUsage(yunikorn.ParseResource(queue.ResourceUsage)) Ω(usedResource).To(Equal(queueQuota.Resources)) } } + By("UsersTrackers: Check total number of users") + usersUsage, err := restClient.GetUsersResourceUsage(DEFAULT_PARTITION) + Ω(err).NotTo(HaveOccurred()) + Ω(len(usersUsage)).To(Equal(expectedUsers.NumberOfUsers), "Total number of users is not correct") for user, quota := range expectedUsers.Users { - By(fmt.Sprintf("UserTracker: Check user resource usage of %s in each queue", user)) - userUsage, err := restClient.GetUserResourceUsage(DEFAULT_PARTITION, user) - Ω(err).NotTo(HaveOccurred()) + By(fmt.Sprintf("UsersTrackers: Check user resource usage of %s in each queue", user)) + userUsage, getUserErr := yunikorn.GetUserUsageFromUsersUsage(usersUsage, user) + Ω(getUserErr).NotTo(HaveOccurred()) Ω(userUsage).To(Equal(quota.AppBelongingGroup)) for queuePath, queueQuota := range quota.Queues { - queue, err := yunikorn.GetQueueResourceUsage(userUsage.Queues, queuePath) - Ω(err).NotTo(HaveOccurred()) + queue, getQueueerr := yunikorn.GetQueueResourceUsage(userUsage.Queues, queuePath) + Ω(getQueueerr).NotTo(HaveOccurred()) Ω(queue.RunningApplications).To(Equal(queueQuota.RunningApplications)) usedResource.ParseResourceUsage(yunikorn.ParseResource(queue.ResourceUsage)) Ω(usedResource).To(Equal(queueQuota.Resources)) From c50d30135e4952112b1873edb48c27e6480a71af Mon Sep 17 00:00:00 2001 From: 0yukali0 Date: Mon, 4 Sep 2023 00:52:45 +0800 Subject: [PATCH 09/16] [YUNIKORN-1901] A basic example for the user tracing and the group tracing --- .../user_quota_tracing/user_quota_tracing_test.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test/e2e/user_quota_tracing/user_quota_tracing_test.go b/test/e2e/user_quota_tracing/user_quota_tracing_test.go index b2ba63b70..a447e00e7 100644 --- a/test/e2e/user_quota_tracing/user_quota_tracing_test.go +++ b/test/e2e/user_quota_tracing/user_quota_tracing_test.go @@ -217,12 +217,12 @@ var _ = Describe("QuotaTracking: Two leaf queus for two groups", func() { } for user, quota := range expectedUsers.Users { By(fmt.Sprintf("UserTracker: Check user resource usage of %s in each queue", user)) - userUsage, err := restClient.GetUserResourceUsage(DEFAULT_PARTITION, user) - Ω(err).NotTo(HaveOccurred()) + userUsage, getUserErr := restClient.GetUserResourceUsage(DEFAULT_PARTITION, user) + Ω(getUserErr).NotTo(HaveOccurred()) Ω(userUsage).To(Equal(quota.AppBelongingGroup)) for queuePath, queueQuota := range quota.Queues { - queue, err := yunikorn.GetQueueResourceUsage(userUsage.Queues, queuePath) - Ω(err).NotTo(HaveOccurred()) + queue, getQueueErr := yunikorn.GetQueueResourceUsage(userUsage.Queues, queuePath) + Ω(getQueueErr).NotTo(HaveOccurred()) Ω(queue.RunningApplications).To(Equal(queueQuota.RunningApplications)) usedResource.ParseResourceUsage(yunikorn.ParseResource(queue.ResourceUsage)) Ω(usedResource).To(Equal(queueQuota.Resources)) @@ -239,8 +239,8 @@ var _ = Describe("QuotaTracking: Two leaf queus for two groups", func() { Ω(getUserErr).NotTo(HaveOccurred()) Ω(userUsage).To(Equal(quota.AppBelongingGroup)) for queuePath, queueQuota := range quota.Queues { - queue, getQueueerr := yunikorn.GetQueueResourceUsage(userUsage.Queues, queuePath) - Ω(getQueueerr).NotTo(HaveOccurred()) + queue, getQueueErr := yunikorn.GetQueueResourceUsage(userUsage.Queues, queuePath) + Ω(getQueueErr).NotTo(HaveOccurred()) Ω(queue.RunningApplications).To(Equal(queueQuota.RunningApplications)) usedResource.ParseResourceUsage(yunikorn.ParseResource(queue.ResourceUsage)) Ω(usedResource).To(Equal(queueQuota.Resources)) From 4e07a6cace6fb8fdfb03745fdeabc643edba2034 Mon Sep 17 00:00:00 2001 From: 0yukali0 Date: Sun, 10 Sep 2023 00:49:53 +0800 Subject: [PATCH 10/16] rebasing, adding functions and directly using literals --- .../helpers/yunikorn/rest_api_utils.go | 49 +-- .../user_quota_tracing_suite_test.go | 1 + .../user_quota_tracing_test.go | 302 +++++++++--------- 3 files changed, 162 insertions(+), 190 deletions(-) diff --git a/test/e2e/framework/helpers/yunikorn/rest_api_utils.go b/test/e2e/framework/helpers/yunikorn/rest_api_utils.go index 6af1c1639..8eb85f8f2 100644 --- a/test/e2e/framework/helpers/yunikorn/rest_api_utils.go +++ b/test/e2e/framework/helpers/yunikorn/rest_api_utils.go @@ -410,20 +410,8 @@ func (c *RClient) GetUserUsage(partition string, userName string) (*dao.UserReso return userUsage, err } - - -func (c *RClient) GetGroupUsage(partition string, groupName string) (*dao.GroupResourceUsageDAOInfo, error) { - req, err := c.newRequest("GET", fmt.Sprintf(configmanager.GroupUsagePath, partition, groupName), nil) - if err != nil { - return nil, err - } - var groupUsage *dao.GroupResourceUsageDAOInfo - _, err = c.do(req, &groupUsage) - return groupUsage, err -} - -func (c *RClient) GetUserssage(partition string) ([]*dao.UserResourceUsageDAOInfo, error) { - req, err := c.newRequest("GET", fmt.Sprintf(configmanager.UsersTrackerPath, partition), nil) +func (c *RClient) GetUsersUsage(partition string) ([]*dao.UserResourceUsageDAOInfo, error) { + req, err := c.newRequest("GET", fmt.Sprintf(configmanager.UsersUsagePath, partition), nil) if err != nil { return nil, err } @@ -432,45 +420,26 @@ func (c *RClient) GetUserssage(partition string) ([]*dao.UserResourceUsageDAOInf return usersUsage, err } -<<<<<<< HEAD -func (c *RClient) GetGroupsUsage(partition string) ([]*dao.GroupResourceUsageDAOInfo, error) { -======= -func (c *RClient) GetUserResourceUsage(partition string, user string) (*dao.UserResourceUsageDAOInfo, error) { - req, err := c.newRequest("GET", fmt.Sprintf(configmanager.UserTrackerPath, partition, user), nil) +func (c *RClient) GetGroupUsage(partition string, groupName string) (*dao.GroupResourceUsageDAOInfo, error) { + req, err := c.newRequest("GET", fmt.Sprintf(configmanager.GroupUsagePath, partition, groupName), nil) if err != nil { return nil, err } - var userUsage *dao.UserResourceUsageDAOInfo - _, err = c.do(req, userUsage) - return userUsage, err + var groupUsage *dao.GroupResourceUsageDAOInfo + _, err = c.do(req, &groupUsage) + return groupUsage, err } -func (c *RClient) GetGroupsResourceUsage(partition string) ([]*dao.GroupResourceUsageDAOInfo, error) { ->>>>>>> 97e86d38 ([YUNIKORN-1901] A basic example for the user tracing and the group tracing) - req, err := c.newRequest("GET", fmt.Sprintf(configmanager.GroupsTrackerPath, partition), nil) +func (c *RClient) GetGroupsUsage(partition string) ([]*dao.GroupResourceUsageDAOInfo, error) { + req, err := c.newRequest("GET", fmt.Sprintf(configmanager.GroupsUsagePath, partition), nil) if err != nil { return nil, err } -<<<<<<< HEAD var groupsUsage []*dao.GroupResourceUsageDAOInfo _, err = c.do(req, groupsUsage) return groupsUsage, err -======= - var gourpsUsage []*dao.GroupResourceUsageDAOInfo - _, err = c.do(req, gourpsUsage) - return gourpsUsage, err } -func (c *RClient) GetGroupResourceUsage(partition string, user string) (*dao.GroupResourceUsageDAOInfo, error) { - req, err := c.newRequest("GET", fmt.Sprintf(configmanager.GroupTrackerPath, partition, user), nil) - if err != nil { - return nil, err - } - var groupUsage *dao.GroupResourceUsageDAOInfo - _, err = c.do(req, groupUsage) - return groupUsage, err ->>>>>>> 97e86d38 ([YUNIKORN-1901] A basic example for the user tracing and the group tracing) -} func GetUserUsageFromUsersUsage(users []*dao.UserResourceUsageDAOInfo, target string) (*dao.UserResourceUsageDAOInfo, error) { for _, user := range users { diff --git a/test/e2e/user_quota_tracing/user_quota_tracing_suite_test.go b/test/e2e/user_quota_tracing/user_quota_tracing_suite_test.go index 0b807e116..9ba167d7f 100644 --- a/test/e2e/user_quota_tracing/user_quota_tracing_suite_test.go +++ b/test/e2e/user_quota_tracing/user_quota_tracing_suite_test.go @@ -36,6 +36,7 @@ func init() { var oldConfigMap = new(v1.ConfigMap) var annotation = "ann-" + common.RandSeq(10) var kClient = k8s.KubeCtl{} //nolint +var RestClient = yunikorn.RClient{} var _ = BeforeSuite(func() { Ω(kClient.SetClient()).To(BeNil()) diff --git a/test/e2e/user_quota_tracing/user_quota_tracing_test.go b/test/e2e/user_quota_tracing/user_quota_tracing_test.go index a447e00e7..82ea52d26 100644 --- a/test/e2e/user_quota_tracing/user_quota_tracing_test.go +++ b/test/e2e/user_quota_tracing/user_quota_tracing_test.go @@ -27,6 +27,7 @@ import ( . "github.com/onsi/gomega" "github.com/apache/yunikorn-core/pkg/common/configs" + "github.com/apache/yunikorn-core/pkg/webservice/dao" tests "github.com/apache/yunikorn-k8shim/test/e2e" "github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/common" "github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/k8s" @@ -40,7 +41,7 @@ const ( DEFAULT_PARTITION = "default" ) -var _ = Describe("QuotaTracking: Two leaf queus for two groups", func() { +var _ = Describe("QuotaTracking: Two leaf queues for two groups", func() { var ns string BeforeEach(func() { ns = "ns-" + common.RandSeq(NANESPACE_LENGTH) @@ -50,10 +51,7 @@ var _ = Describe("QuotaTracking: Two leaf queus for two groups", func() { Ω(ns1.Status.Phase).To(Equal(v1.NamespaceActive)) }) - It("User qauta trace with 3 users and 2 groups", func() { - groups := []string{"group1", "group2"} - users := []string{"user1", "user2", "user3"} - queuePath := []string{"root", "root.group1_resources", "root.group2_resources"} + It("User quota trace with 3 users and 2 groups", func() { zeroPod := map[string]int64{siCommon.CPU: 0, siCommon.Memory: 0} onePod := map[string]int64{siCommon.CPU: 100, siCommon.Memory: 50} twoPods := map[string]int64{siCommon.CPU: 200, siCommon.Memory: 100} @@ -63,8 +61,8 @@ var _ = Describe("QuotaTracking: Two leaf queus for two groups", func() { queuesConfigs := []struct { partition, parentQueue, QueueName string }{ - {DEFAULT_PARTITION, queuePath[0], "group1_resources"}, - {DEFAULT_PARTITION, queuePath[0], "group2_resources"}, + {DEFAULT_PARTITION, "root", "group1_resources"}, + {DEFAULT_PARTITION, "root", "group2_resources"}, } for _, queueConfig := range queuesConfigs { By(fmt.Sprintf("Add child queue %s to the parent queue %s", queueConfig.QueueName, queueConfig.parentQueue)) @@ -85,167 +83,66 @@ var _ = Describe("QuotaTracking: Two leaf queus for two groups", func() { * user2 -> group2 * user3 -> group1, group2 */ - configs := []struct { - AppID, Queue, UserName string - GroupsNames []string - }{ - {"teacher-app-01", queuePath[1], users[0], []string{groups[0]}}, - {"students-app-01", queuePath[2], users[1], []string{groups[1]}}, - {"assistant-app-01", queuePath[2], users[2], []string{groups[1]}}, - {"assistant-app-02", queuePath[1], users[2], []string{groups[0]}}, - } - for _, config := range configs { - By(fmt.Sprintf("Deploy the sleep app %s to the %s namespace", config.AppID, ns)) - sleepObj, podErr := k8s.InitSleepPod(k8s.SleepPodConfig{ - AppID: config.AppID, - NS: ns, - Labels: map[string]string{"queue": config.Queue}, - }) - sleepObj.ObjectMeta.Annotations["yunikorn.apache.org/user.info"] = fmt.Sprintf("{username:%s, groups:{%s}}", config.UserName, config.GroupsNames) - Ω(podErr).NotTo(HaveOccurred()) - sleepRespPod, podErr := kClient.CreatePod(sleepObj, ns) - Ω(podErr).NotTo(HaveOccurred()) + deploySleepPod("user1-app-01", "root.group1_resources", "user1", []string{"group1"}, ns) + deploySleepPod("user2-app-01", "root.group2_resources", "user2", []string{"group2"}, ns) + deploySleepPod("user3-app-01", "root.group2_resources", "user3", []string{"group2"}, ns) + deploySleepPod("user3-app-02", "root.group1_resources", "user3", []string{"group1"}, ns) - // Wait for pod to move to running state - podErr = kClient.WaitForPodBySelectorRunning(ns, - fmt.Sprintf("app=%s", sleepRespPod.ObjectMeta.Labels["app"]), WAIT_INTERVAL) - Ω(podErr).NotTo(HaveOccurred()) - } - - type Queue struct { - RunningApplications []string - Resources map[string]int64 - } - type GroupQuota struct { - RunningApplications []string - Queues map[string]Queue - } - type GroupsQuota struct { - NumberOfGroups int - Groups map[string]GroupQuota - } - expectedGroups := GroupsQuota{ + expectedGroups := ExpectedGroupsQuota{ 2, - map[string]GroupQuota{ - groups[0]: { - []string{configs[0].AppID, configs[3].AppID}, - map[string]Queue{ - queuePath[0]: {[]string{configs[0].AppID, configs[3].AppID}, twoPods}, - queuePath[1]: {[]string{configs[0].AppID, configs[3].AppID}, twoPods}, - queuePath[2]: {[]string{}, zeroPod}, + map[string]ExpectedGroupQuota{ + "group1": { + []string{"user1-app-01", "user3-app-02"}, + map[string]ExpectedQueue{ + "root": {[]string{"user1-app-01", "user3-app-02"}, twoPods}, + "root.group1_resources": {[]string{"user1-app-01", "user3-app-02"}, twoPods}, + "root.group2_resources": {[]string{}, zeroPod}, }, }, - groups[1]: { - []string{configs[1].AppID, configs[2].AppID}, - map[string]Queue{ - queuePath[0]: {[]string{configs[1].AppID, configs[2].AppID}, twoPods}, - queuePath[1]: {[]string{}, zeroPod}, - queuePath[2]: {[]string{configs[1].AppID, configs[2].AppID}, twoPods}, + "group2": { + []string{"user2-app-01", "user3-app-01"}, + map[string]ExpectedQueue{ + "root": {[]string{"user2-app-01", "user3-app-01"}, twoPods}, + "root.group1_resources": {[]string{}, zeroPod}, + "root.group2_resources": {[]string{"user2-app-01", "user3-app-01"}, twoPods}, }, }, }, } - restClient := yunikorn.RClient{} - var usedResource yunikorn.ResourceUsage - for group, quota := range expectedGroups.Groups { - By(fmt.Sprintf("GroupTracker: Check group resource usage of %s in each queue", group)) - groupUsage, getGroupErr := restClient.GetGroupResourceUsage(DEFAULT_PARTITION, group) - Ω(getGroupErr).NotTo(HaveOccurred()) - Ω(groupUsage.Applications).To(Equal(quota.RunningApplications), "running application IDs are not expected") - for queuePath, queueQuota := range quota.Queues { - queue, getQueueErr := yunikorn.GetQueueResourceUsage(groupUsage.Queues, queuePath) - Ω(getQueueErr).NotTo(HaveOccurred()) - Ω(queue.RunningApplications).To(Equal(queueQuota.RunningApplications)) - usedResource.ParseResourceUsage(yunikorn.ParseResource(queue.ResourceUsage)) - Ω(usedResource).To(Equal(queueQuota.Resources)) - } - } + checkGroupUsage(true, nil, expectedGroups.Groups) + checkGroupsUsage(expectedGroups) - By("GroupsTracker: Check total number of groups") - groupsUsage, err := restClient.GetGroupsResourceUsage(DEFAULT_PARTITION) - Ω(err).NotTo(HaveOccurred()) - Ω(len(groupsUsage)).To(Equal(expectedGroups.NumberOfGroups), "Total number of groups is not correct") - for group, quota := range expectedGroups.Groups { - By(fmt.Sprintf("GroupTracker: Check group resource usage of %s in each queue", group)) - groupUsage, getGroupErr := yunikorn.GetGroupUsageFromGroupsUsage(groupsUsage, group) - Ω(getGroupErr).NotTo(HaveOccurred()) - Ω(groupUsage.Applications).To(Equal(quota.RunningApplications), "running application IDs are not expected") - for queuePath, queueQuota := range quota.Queues { - queue, getQueueErr := yunikorn.GetQueueResourceUsage(groupUsage.Queues, queuePath) - Ω(getQueueErr).NotTo(HaveOccurred()) - Ω(queue.RunningApplications).To(Equal(queueQuota.RunningApplications)) - usedResource.ParseResourceUsage(yunikorn.ParseResource(queue.ResourceUsage)) - Ω(usedResource).To(Equal(queueQuota.Resources)) - } - } - - type UserQuota struct { - AppBelongingGroup map[string]string - Queues map[string]Queue - } - expectedUsers := struct { - NumberOfUsers int - Users map[string]UserQuota - }{ + expectedUsers := ExpectedUsersQuota{ 3, - map[string]UserQuota{ - users[0]: { - map[string]string{configs[0].AppID: groups[0]}, - map[string]Queue{ - queuePath[0]: {[]string{configs[0].AppID}, onePod}, - queuePath[1]: {[]string{configs[0].AppID}, onePod}, - queuePath[2]: {[]string{}, zeroPod}, + map[string]ExpectedUserQuota{ + "user1": { + map[string]string{"user1-app-01": "group1"}, + map[string]ExpectedQueue{ + "root": {[]string{"user1-app-01"}, onePod}, + "root.group1_resources": {[]string{"user1-app-01"}, onePod}, + "root.group2_resources": {[]string{}, zeroPod}, }, }, - users[1]: { - map[string]string{configs[1].AppID: groups[1]}, - map[string]Queue{ - queuePath[0]: {[]string{configs[1].AppID}, onePod}, - queuePath[1]: {[]string{configs[1].AppID}, onePod}, - queuePath[2]: {[]string{}, zeroPod}, + "user2": { + map[string]string{"user2-app-01": "group2"}, + map[string]ExpectedQueue{ + "root": {[]string{"user2-app-01"}, onePod}, + "root.group1_resources": {[]string{"user2-app-01"}, onePod}, + "root.group2_resources": {[]string{}, zeroPod}, }, }, - users[2]: { - map[string]string{configs[2].AppID: groups[1], configs[3].AppID: groups[0]}, - map[string]Queue{ - queuePath[0]: {[]string{configs[2].AppID, configs[3].AppID}, twoPods}, - queuePath[1]: {[]string{configs[3].AppID}, onePod}, - queuePath[2]: {[]string{configs[2].AppID}, onePod}, + "user3": { + map[string]string{"user3-app-01": "group2", "user3-app-02": "group1"}, + map[string]ExpectedQueue{ + "root": {[]string{"user3-app-01", "user3-app-02"}, twoPods}, + "root.group1_resources": {[]string{"user3-app-02"}, onePod}, + "root.group2_resources": {[]string{"user3-app-01"}, onePod}, }, }, }, } - for user, quota := range expectedUsers.Users { - By(fmt.Sprintf("UserTracker: Check user resource usage of %s in each queue", user)) - userUsage, getUserErr := restClient.GetUserResourceUsage(DEFAULT_PARTITION, user) - Ω(getUserErr).NotTo(HaveOccurred()) - Ω(userUsage).To(Equal(quota.AppBelongingGroup)) - for queuePath, queueQuota := range quota.Queues { - queue, getQueueErr := yunikorn.GetQueueResourceUsage(userUsage.Queues, queuePath) - Ω(getQueueErr).NotTo(HaveOccurred()) - Ω(queue.RunningApplications).To(Equal(queueQuota.RunningApplications)) - usedResource.ParseResourceUsage(yunikorn.ParseResource(queue.ResourceUsage)) - Ω(usedResource).To(Equal(queueQuota.Resources)) - } - } - - By("UsersTrackers: Check total number of users") - usersUsage, err := restClient.GetUsersResourceUsage(DEFAULT_PARTITION) - Ω(err).NotTo(HaveOccurred()) - Ω(len(usersUsage)).To(Equal(expectedUsers.NumberOfUsers), "Total number of users is not correct") - for user, quota := range expectedUsers.Users { - By(fmt.Sprintf("UsersTrackers: Check user resource usage of %s in each queue", user)) - userUsage, getUserErr := yunikorn.GetUserUsageFromUsersUsage(usersUsage, user) - Ω(getUserErr).NotTo(HaveOccurred()) - Ω(userUsage).To(Equal(quota.AppBelongingGroup)) - for queuePath, queueQuota := range quota.Queues { - queue, getQueueErr := yunikorn.GetQueueResourceUsage(userUsage.Queues, queuePath) - Ω(getQueueErr).NotTo(HaveOccurred()) - Ω(queue.RunningApplications).To(Equal(queueQuota.RunningApplications)) - usedResource.ParseResourceUsage(yunikorn.ParseResource(queue.ResourceUsage)) - Ω(usedResource).To(Equal(queueQuota.Resources)) - } - } + checkUserUsage(true, nil, expectedUsers.Users) + checkUsersUsage(expectedUsers) }) AfterEach(func() { @@ -259,3 +156,108 @@ var _ = Describe("QuotaTracking: Two leaf queus for two groups", func() { Ω(err).NotTo(HaveOccurred()) }) }) + +func deploySleepPod(appID, queue, userName string, groupsName []string, ns string) { + By(fmt.Sprintf("Deploy the sleep app %s to the %s namespace", appID, ns)) + sleepObj, podErr := k8s.InitSleepPod(k8s.SleepPodConfig{ + AppID: appID, + NS: ns, + Labels: map[string]string{"queue": queue}, + }) + sleepObj.ObjectMeta.Annotations["yunikorn.apache.org/user.info"] = fmt.Sprintf("{username:%s, groups:{%s}}", userName, groupsName) + Ω(podErr).NotTo(HaveOccurred()) + sleepRespPod, podErr := kClient.CreatePod(sleepObj, ns) + Ω(podErr).NotTo(HaveOccurred()) + + // Wait for pod to move to running state + podErr = kClient.WaitForPodBySelectorRunning(ns, + fmt.Sprintf("app=%s", sleepRespPod.ObjectMeta.Labels["app"]), WAIT_INTERVAL) + Ω(podErr).NotTo(HaveOccurred()) +} + +type ExpectedQueue struct { + RunningApplications []string + Resources map[string]int64 +} + +func checkQueue(queues *dao.ResourceUsageDAOInfo, expectedQueues map[string]ExpectedQueue) { + var usedResource yunikorn.ResourceUsage + for queuePath, expectedQueue := range expectedQueues { + queue, getQueueErr := yunikorn.GetQueueResourceUsage(queues, queuePath) + Ω(getQueueErr).NotTo(HaveOccurred()) + Ω(queue.RunningApplications).To(Equal(expectedQueue.RunningApplications)) + usedResource.ParseResourceUsage(yunikorn.ParseResource(queue.ResourceUsage)) + Ω(usedResource).To(Equal(expectedQueue.Resources)) + } +} + +type ExpectedGroupQuota struct { + RunningApplications []string + Queues map[string]ExpectedQueue +} + +func checkGroupUsage(FromREST bool, groupsUsage []*dao.GroupResourceUsageDAOInfo, expectedGroupUsage map[string]ExpectedGroupQuota) { + var groupUsage *dao.GroupResourceUsageDAOInfo + var err error + for groupName, expectedGroup := range expectedGroupUsage { + By(fmt.Sprintf("GroupUsage: Check group resource usage of %s in each queue", groupName)) + if FromREST { + groupUsage, err = RestClient.GetGroupUsage(DEFAULT_PARTITION, groupName) + Ω(err).NotTo(HaveOccurred()) + } else { + groupUsage, err = yunikorn.GetGroupUsageFromGroupsUsage(groupsUsage, groupName) + Ω(err).NotTo(HaveOccurred()) + } + Ω(groupUsage.Applications).To(Equal(expectedGroup.RunningApplications), "running application IDs are not expected") + checkQueue(groupUsage.Queues, expectedGroup.Queues) + } +} + +type ExpectedGroupsQuota struct { + NumberOfGroups int + Groups map[string]ExpectedGroupQuota +} + +func checkGroupsUsage(expectedGroups ExpectedGroupsQuota) { + By("GroupsUsage: Check total number of groups") + groupsUsage, err := RestClient.GetGroupsUsage(DEFAULT_PARTITION) + Ω(err).NotTo(HaveOccurred()) + Ω(len(groupsUsage)).To(Equal(expectedGroups.NumberOfGroups), "Total number of groups is not correct") + checkGroupUsage(false, groupsUsage, expectedGroups.Groups) +} + +type ExpectedUserQuota struct { + AppBelongingGroup map[string]string + Queues map[string]ExpectedQueue +} + +func checkUserUsage(FromREST bool, usersUsage []*dao.UserResourceUsageDAOInfo, expectedUserUsage map[string]ExpectedUserQuota) { + var userUsage *dao.UserResourceUsageDAOInfo + var err error + for userName, expectedUserUsage := range expectedUserUsage { + By(fmt.Sprintf("UserUsage: Check group resource usage of %s in each queue", userName)) + if FromREST { + userUsage, err = RestClient.GetUserUsage(DEFAULT_PARTITION, userName) + Ω(err).NotTo(HaveOccurred()) + } else { + userUsage, err = yunikorn.GetUserUsageFromUsersUsage(usersUsage, userName) + Ω(err).NotTo(HaveOccurred()) + } + Ω(err).NotTo(HaveOccurred()) + Ω(userUsage).To(Equal(expectedUserUsage.AppBelongingGroup)) + checkQueue(userUsage.Queues, expectedUserUsage.Queues) + } +} + +type ExpectedUsersQuota struct { + NumberOfUsers int + Users map[string]ExpectedUserQuota +} + +func checkUsersUsage(expectedUsersUsage ExpectedUsersQuota) { + By("UsersUsage: Check total number of users") + usersUsage, err := RestClient.GetUsersUsage(DEFAULT_PARTITION) + Ω(err).NotTo(HaveOccurred()) + Ω(len(usersUsage)).To(Equal(expectedUsersUsage.NumberOfUsers), "Total number of users is not correct") + checkUserUsage(false, usersUsage, expectedUsersUsage.Users) +} \ No newline at end of file From 04c70e54e53b9a3a4640f849a8ff795bae4cf9ea Mon Sep 17 00:00:00 2001 From: 0yukali0 Date: Sun, 10 Sep 2023 07:59:50 +0800 Subject: [PATCH 11/16] fix lint problem --- .../helpers/yunikorn/rest_api_utils.go | 1 - .../user_quota_tracing_test.go | 40 +++++++++---------- 2 files changed, 20 insertions(+), 21 deletions(-) diff --git a/test/e2e/framework/helpers/yunikorn/rest_api_utils.go b/test/e2e/framework/helpers/yunikorn/rest_api_utils.go index 8eb85f8f2..0c7fea0ea 100644 --- a/test/e2e/framework/helpers/yunikorn/rest_api_utils.go +++ b/test/e2e/framework/helpers/yunikorn/rest_api_utils.go @@ -440,7 +440,6 @@ func (c *RClient) GetGroupsUsage(partition string) ([]*dao.GroupResourceUsageDAO return groupsUsage, err } - func GetUserUsageFromUsersUsage(users []*dao.UserResourceUsageDAOInfo, target string) (*dao.UserResourceUsageDAOInfo, error) { for _, user := range users { if user.UserName == target { diff --git a/test/e2e/user_quota_tracing/user_quota_tracing_test.go b/test/e2e/user_quota_tracing/user_quota_tracing_test.go index 82ea52d26..b786b7bcc 100644 --- a/test/e2e/user_quota_tracing/user_quota_tracing_test.go +++ b/test/e2e/user_quota_tracing/user_quota_tracing_test.go @@ -94,7 +94,7 @@ var _ = Describe("QuotaTracking: Two leaf queues for two groups", func() { "group1": { []string{"user1-app-01", "user3-app-02"}, map[string]ExpectedQueue{ - "root": {[]string{"user1-app-01", "user3-app-02"}, twoPods}, + "root": {[]string{"user1-app-01", "user3-app-02"}, twoPods}, "root.group1_resources": {[]string{"user1-app-01", "user3-app-02"}, twoPods}, "root.group2_resources": {[]string{}, zeroPod}, }, @@ -102,7 +102,7 @@ var _ = Describe("QuotaTracking: Two leaf queues for two groups", func() { "group2": { []string{"user2-app-01", "user3-app-01"}, map[string]ExpectedQueue{ - "root": {[]string{"user2-app-01", "user3-app-01"}, twoPods}, + "root": {[]string{"user2-app-01", "user3-app-01"}, twoPods}, "root.group1_resources": {[]string{}, zeroPod}, "root.group2_resources": {[]string{"user2-app-01", "user3-app-01"}, twoPods}, }, @@ -118,7 +118,7 @@ var _ = Describe("QuotaTracking: Two leaf queues for two groups", func() { "user1": { map[string]string{"user1-app-01": "group1"}, map[string]ExpectedQueue{ - "root": {[]string{"user1-app-01"}, onePod}, + "root": {[]string{"user1-app-01"}, onePod}, "root.group1_resources": {[]string{"user1-app-01"}, onePod}, "root.group2_resources": {[]string{}, zeroPod}, }, @@ -126,7 +126,7 @@ var _ = Describe("QuotaTracking: Two leaf queues for two groups", func() { "user2": { map[string]string{"user2-app-01": "group2"}, map[string]ExpectedQueue{ - "root": {[]string{"user2-app-01"}, onePod}, + "root": {[]string{"user2-app-01"}, onePod}, "root.group1_resources": {[]string{"user2-app-01"}, onePod}, "root.group2_resources": {[]string{}, zeroPod}, }, @@ -134,7 +134,7 @@ var _ = Describe("QuotaTracking: Two leaf queues for two groups", func() { "user3": { map[string]string{"user3-app-01": "group2", "user3-app-02": "group1"}, map[string]ExpectedQueue{ - "root": {[]string{"user3-app-01", "user3-app-02"}, twoPods}, + "root": {[]string{"user3-app-01", "user3-app-02"}, twoPods}, "root.group1_resources": {[]string{"user3-app-02"}, onePod}, "root.group2_resources": {[]string{"user3-app-01"}, onePod}, }, @@ -159,20 +159,20 @@ var _ = Describe("QuotaTracking: Two leaf queues for two groups", func() { func deploySleepPod(appID, queue, userName string, groupsName []string, ns string) { By(fmt.Sprintf("Deploy the sleep app %s to the %s namespace", appID, ns)) - sleepObj, podErr := k8s.InitSleepPod(k8s.SleepPodConfig{ - AppID: appID, - NS: ns, - Labels: map[string]string{"queue": queue}, - }) - sleepObj.ObjectMeta.Annotations["yunikorn.apache.org/user.info"] = fmt.Sprintf("{username:%s, groups:{%s}}", userName, groupsName) - Ω(podErr).NotTo(HaveOccurred()) - sleepRespPod, podErr := kClient.CreatePod(sleepObj, ns) - Ω(podErr).NotTo(HaveOccurred()) + sleepObj, podErr := k8s.InitSleepPod(k8s.SleepPodConfig{ + AppID: appID, + NS: ns, + Labels: map[string]string{"queue": queue}, + }) + sleepObj.ObjectMeta.Annotations["yunikorn.apache.org/user.info"] = fmt.Sprintf("{username:%s, groups:{%s}}", userName, groupsName) + Ω(podErr).NotTo(HaveOccurred()) + sleepRespPod, podErr := kClient.CreatePod(sleepObj, ns) + Ω(podErr).NotTo(HaveOccurred()) - // Wait for pod to move to running state - podErr = kClient.WaitForPodBySelectorRunning(ns, - fmt.Sprintf("app=%s", sleepRespPod.ObjectMeta.Labels["app"]), WAIT_INTERVAL) - Ω(podErr).NotTo(HaveOccurred()) + // Wait for pod to move to running state + podErr = kClient.WaitForPodBySelectorRunning(ns, + fmt.Sprintf("app=%s", sleepRespPod.ObjectMeta.Labels["app"]), WAIT_INTERVAL) + Ω(podErr).NotTo(HaveOccurred()) } type ExpectedQueue struct { @@ -251,7 +251,7 @@ func checkUserUsage(FromREST bool, usersUsage []*dao.UserResourceUsageDAOInfo, e type ExpectedUsersQuota struct { NumberOfUsers int - Users map[string]ExpectedUserQuota + Users map[string]ExpectedUserQuota } func checkUsersUsage(expectedUsersUsage ExpectedUsersQuota) { @@ -260,4 +260,4 @@ func checkUsersUsage(expectedUsersUsage ExpectedUsersQuota) { Ω(err).NotTo(HaveOccurred()) Ω(len(usersUsage)).To(Equal(expectedUsersUsage.NumberOfUsers), "Total number of users is not correct") checkUserUsage(false, usersUsage, expectedUsersUsage.Users) -} \ No newline at end of file +} From 2b0c731c7b27b4c3d35082e8085f7c71a30d947d Mon Sep 17 00:00:00 2001 From: 0yukali0 Date: Sun, 10 Sep 2023 08:09:26 +0800 Subject: [PATCH 12/16] fix lint problem --- test/e2e/user_quota_tracing/user_quota_tracing_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/e2e/user_quota_tracing/user_quota_tracing_test.go b/test/e2e/user_quota_tracing/user_quota_tracing_test.go index b786b7bcc..7522ec23d 100644 --- a/test/e2e/user_quota_tracing/user_quota_tracing_test.go +++ b/test/e2e/user_quota_tracing/user_quota_tracing_test.go @@ -196,12 +196,12 @@ type ExpectedGroupQuota struct { Queues map[string]ExpectedQueue } -func checkGroupUsage(FromREST bool, groupsUsage []*dao.GroupResourceUsageDAOInfo, expectedGroupUsage map[string]ExpectedGroupQuota) { +func checkGroupUsage(fromREST bool, groupsUsage []*dao.GroupResourceUsageDAOInfo, expectedGroupUsage map[string]ExpectedGroupQuota) { var groupUsage *dao.GroupResourceUsageDAOInfo var err error for groupName, expectedGroup := range expectedGroupUsage { By(fmt.Sprintf("GroupUsage: Check group resource usage of %s in each queue", groupName)) - if FromREST { + if fromREST { groupUsage, err = RestClient.GetGroupUsage(DEFAULT_PARTITION, groupName) Ω(err).NotTo(HaveOccurred()) } else { @@ -231,12 +231,12 @@ type ExpectedUserQuota struct { Queues map[string]ExpectedQueue } -func checkUserUsage(FromREST bool, usersUsage []*dao.UserResourceUsageDAOInfo, expectedUserUsage map[string]ExpectedUserQuota) { +func checkUserUsage(fromREST bool, usersUsage []*dao.UserResourceUsageDAOInfo, expectedUserUsage map[string]ExpectedUserQuota) { var userUsage *dao.UserResourceUsageDAOInfo var err error for userName, expectedUserUsage := range expectedUserUsage { By(fmt.Sprintf("UserUsage: Check group resource usage of %s in each queue", userName)) - if FromREST { + if fromREST { userUsage, err = RestClient.GetUserUsage(DEFAULT_PARTITION, userName) Ω(err).NotTo(HaveOccurred()) } else { From 79841fc344645d7afe220e51ecdb29f33ddc71e4 Mon Sep 17 00:00:00 2001 From: 0yukali0 Date: Sun, 10 Sep 2023 09:21:09 +0800 Subject: [PATCH 13/16] message is wrong --- test/e2e/user_quota_tracing/user_quota_tracing_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/user_quota_tracing/user_quota_tracing_test.go b/test/e2e/user_quota_tracing/user_quota_tracing_test.go index 7522ec23d..548ee4b09 100644 --- a/test/e2e/user_quota_tracing/user_quota_tracing_test.go +++ b/test/e2e/user_quota_tracing/user_quota_tracing_test.go @@ -235,7 +235,7 @@ func checkUserUsage(fromREST bool, usersUsage []*dao.UserResourceUsageDAOInfo, e var userUsage *dao.UserResourceUsageDAOInfo var err error for userName, expectedUserUsage := range expectedUserUsage { - By(fmt.Sprintf("UserUsage: Check group resource usage of %s in each queue", userName)) + By(fmt.Sprintf("UserUsage: Check user resource usage of %s in each queue", userName)) if fromREST { userUsage, err = RestClient.GetUserUsage(DEFAULT_PARTITION, userName) Ω(err).NotTo(HaveOccurred()) From bd9c39b0b0f7ca21f74d77d49e10e55b673926df Mon Sep 17 00:00:00 2001 From: 0yukali0 Date: Sun, 17 Sep 2023 20:15:31 +0800 Subject: [PATCH 14/16] yunikorn parse the queue.yaml in the e2e test --- .../user_quota_tracing_test.go | 45 ++++++++++--------- 1 file changed, 24 insertions(+), 21 deletions(-) diff --git a/test/e2e/user_quota_tracing/user_quota_tracing_test.go b/test/e2e/user_quota_tracing/user_quota_tracing_test.go index 548ee4b09..0b9dc81dc 100644 --- a/test/e2e/user_quota_tracing/user_quota_tracing_test.go +++ b/test/e2e/user_quota_tracing/user_quota_tracing_test.go @@ -22,17 +22,19 @@ import ( "fmt" v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "github.com/apache/yunikorn-core/pkg/common/configs" + "github.com/apache/yunikorn-k8shim/pkg/common/constants" "github.com/apache/yunikorn-core/pkg/webservice/dao" tests "github.com/apache/yunikorn-k8shim/test/e2e" "github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/common" "github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/k8s" "github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/yunikorn" siCommon "github.com/apache/yunikorn-scheduler-interface/lib/go/common" + "github.com/apache/yunikorn-k8shim/test/e2e/framework/configmanager" ) const ( @@ -55,26 +57,27 @@ var _ = Describe("QuotaTracking: Two leaf queues for two groups", func() { zeroPod := map[string]int64{siCommon.CPU: 0, siCommon.Memory: 0} onePod := map[string]int64{siCommon.CPU: 100, siCommon.Memory: 50} twoPods := map[string]int64{siCommon.CPU: 200, siCommon.Memory: 100} - yunikorn.UpdateCustomConfigMapWrapper(oldConfigMap, "", annotation, func(sc *configs.SchedulerConfig) error { - // remove placement rules so we can control queue - sc.Partitions[0].PlacementRules = nil - queuesConfigs := []struct { - partition, parentQueue, QueueName string - }{ - {DEFAULT_PARTITION, "root", "group1_resources"}, - {DEFAULT_PARTITION, "root", "group2_resources"}, - } - for _, queueConfig := range queuesConfigs { - By(fmt.Sprintf("Add child queue %s to the parent queue %s", queueConfig.QueueName, queueConfig.parentQueue)) - config := configs.QueueConfig{ - Name: queueConfig.QueueName, - } - if err := common.AddQueue(sc, queueConfig.partition, queueConfig.parentQueue, config); err != nil { - return err - } - } - return nil - }) + configMap := v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: constants.ConfigMapName, + Namespace: configmanager.YuniKornTestConfig.YkNamespace, + }, + Data: map[string]string{ + "queue.yaml": ` + partitions: + - name: default + queues: + - name: root + submitacl: "*" + queues: + - name: group1_resources + - name: group2_resources + `, + }, + } + cm, err := kClient.UpdateConfigMap(&configMap, configmanager.YuniKornTestConfig.YkNamespace) + Ω(err).ShouldNot(HaveOccurred()) + Ω(cm).ShouldNot(BeNil()) /* * groups: group1, group2 From 8c7b17eee53fa694483e5a61c6c448bbd5563c87 Mon Sep 17 00:00:00 2001 From: 0yukali0 Date: Sun, 17 Sep 2023 20:21:07 +0800 Subject: [PATCH 15/16] yunikorn parse the queue.yaml in the e2e test --- test/e2e/user_quota_tracing/user_quota_tracing_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/e2e/user_quota_tracing/user_quota_tracing_test.go b/test/e2e/user_quota_tracing/user_quota_tracing_test.go index 0b9dc81dc..c0ba74433 100644 --- a/test/e2e/user_quota_tracing/user_quota_tracing_test.go +++ b/test/e2e/user_quota_tracing/user_quota_tracing_test.go @@ -27,14 +27,14 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "github.com/apache/yunikorn-k8shim/pkg/common/constants" "github.com/apache/yunikorn-core/pkg/webservice/dao" + "github.com/apache/yunikorn-k8shim/pkg/common/constants" tests "github.com/apache/yunikorn-k8shim/test/e2e" + "github.com/apache/yunikorn-k8shim/test/e2e/framework/configmanager" "github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/common" "github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/k8s" "github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/yunikorn" siCommon "github.com/apache/yunikorn-scheduler-interface/lib/go/common" - "github.com/apache/yunikorn-k8shim/test/e2e/framework/configmanager" ) const ( From dfe16e5c6327fdfaa26b0d93ae716034514d1565 Mon Sep 17 00:00:00 2001 From: 0yukali0 Date: Sun, 17 Sep 2023 21:29:32 +0800 Subject: [PATCH 16/16] [YUNIKORN-1901] A basic example for the user tracing and the group tracing --- test/e2e/user_quota_tracing/user_quota_tracing_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/e2e/user_quota_tracing/user_quota_tracing_test.go b/test/e2e/user_quota_tracing/user_quota_tracing_test.go index c0ba74433..a5f309f8c 100644 --- a/test/e2e/user_quota_tracing/user_quota_tracing_test.go +++ b/test/e2e/user_quota_tracing/user_quota_tracing_test.go @@ -28,6 +28,7 @@ import ( . "github.com/onsi/gomega" "github.com/apache/yunikorn-core/pkg/webservice/dao" + amConf "github.com/apache/yunikorn-k8shim/pkg/admission/conf" "github.com/apache/yunikorn-k8shim/pkg/common/constants" tests "github.com/apache/yunikorn-k8shim/test/e2e" "github.com/apache/yunikorn-k8shim/test/e2e/framework/configmanager" @@ -73,6 +74,8 @@ var _ = Describe("QuotaTracking: Two leaf queues for two groups", func() { - name: group1_resources - name: group2_resources `, + amConf.AMAccessControlExternalUsers: "^user1$,^user2$,^user3$", + amConf.AMAccessControlExternalGroups: "^group1$,^group2$", }, } cm, err := kClient.UpdateConfigMap(&configMap, configmanager.YuniKornTestConfig.YkNamespace)