From 3df5ce89d14b3649aad3cfa6f83524e4ede0db07 Mon Sep 17 00:00:00 2001 From: Monokaix <changxuzheng@huawei.com> Date: Sat, 11 Jan 2025 21:44:54 +0800 Subject: [PATCH] add node event Signed-off-by: Monokaix <changxuzheng@huawei.com> --- pkg/scheduler/api/hyper_node_info.go | 131 ++++++- pkg/scheduler/api/hyper_node_info_test.go | 312 ++++++++++++++- pkg/scheduler/api/test_utils.go | 30 +- pkg/scheduler/api/test_utils_test.go | 22 +- pkg/scheduler/cache/cache.go | 40 +- pkg/scheduler/cache/cache_mock.go | 1 + pkg/scheduler/cache/dumper.go | 2 +- pkg/scheduler/cache/event_handlers.go | 119 ++++-- pkg/scheduler/cache/event_handlers_test.go | 421 ++++++++++++++++++--- pkg/scheduler/cache/util.go | 15 + 10 files changed, 962 insertions(+), 131 deletions(-) diff --git a/pkg/scheduler/api/hyper_node_info.go b/pkg/scheduler/api/hyper_node_info.go index 863d0ed216..3eead32c04 100644 --- a/pkg/scheduler/api/hyper_node_info.go +++ b/pkg/scheduler/api/hyper_node_info.go @@ -45,7 +45,8 @@ type HyperNodesInfo struct { // s0->node0, node1, s1->node2, node3, s4->node0, node1, s1->node2, node3 realNodesSet map[string]sets.Set[string] // nodeLister Lister to list Kubernetes nodes. - nodeLister listerv1.NodeLister + nodeLister listerv1.NodeLister + builtErrHyperNode string // ready indicates whether the HyperNodesInfo is ready (build process is complete). ready *atomic.Bool @@ -107,6 +108,15 @@ func (hni *HyperNodesInfo) HyperNodes() map[string]*HyperNodeInfo { return hni.hyperNodes } +// HyperNode returns a hyperNode by name. +func (hni *HyperNodesInfo) HyperNode(name string) *topologyv1alpha1.HyperNode { + hn := hni.hyperNodes[name] + if hn == nil { + return nil + } + return hn.HyperNode +} + // HyperNodesSetByTier returns a deep copy of the map that groups HyperNode names by their tier. // This ensures that the returned map is independent of the original, preventing unintended modifications. func (hni *HyperNodesInfo) HyperNodesSetByTier() map[int]sets.Set[string] { @@ -324,6 +334,67 @@ func (hni *HyperNodesInfo) getChildren(hyperNodeName string) sets.Set[string] { return children } +// GetLeafNodes returns the leaf hyperNodes set for a given hyperNode. +func (hni *HyperNodesInfo) GetLeafNodes(hyperNodeName string) sets.Set[string] { + leafNodes := sets.New[string]() + ancestorsChain := sets.New[string]() + hni.findLeafNodesWithCycleCheck(hyperNodeName, leafNodes, ancestorsChain) + return leafNodes +} + +func (hni *HyperNodesInfo) findLeafNodesWithCycleCheck(hyperNodeName string, leafNodes sets.Set[string], ancestorsChain sets.Set[string]) { + if ancestorsChain.Has(hyperNodeName) { + klog.ErrorS(nil, "Cycle detected in HyperNode hierarchy", "hyperNode", hyperNodeName) + return + } + + hn, ok := hni.hyperNodes[hyperNodeName] + if !ok { + return + } + + ancestorsChain.Insert(hyperNodeName) + defer ancestorsChain.Delete(hyperNodeName) + + isLeaf := true + for _, member := range hn.HyperNode.Spec.Members { + if member.Type == topologyv1alpha1.MemberTypeHyperNode { + isLeaf = false + hni.findLeafNodesWithCycleCheck(member.Selector.ExactMatch.Name, leafNodes, ancestorsChain) + } + } + + if isLeaf { + leafNodes.Insert(hyperNodeName) + } +} + +// GetRegexSelectorLeafHyperNodes returns leaf hyperNodes whose member's selector is regex match. +func (hni *HyperNodesInfo) GetRegexSelectorLeafHyperNodes() sets.Set[string] { + leaf := sets.New[string]() + for name, hnInfo := range hni.hyperNodes { + if hnInfo == nil || hnInfo.HyperNode == nil { + continue + } + + isLeaf := true + hasRegexMatch := false + for _, member := range hnInfo.HyperNode.Spec.Members { + if member.Type == topologyv1alpha1.MemberTypeHyperNode { + isLeaf = false + break + } + if member.Selector.RegexMatch != nil { + hasRegexMatch = true + } + } + if isLeaf && hasRegexMatch { + leaf.Insert(name) + } + } + return leaf +} + // setParent sets the parent of a HyperNode member. func (hni *HyperNodesInfo) setParent(member, parent string) error { hn, ok := hni.hyperNodes[member] @@ -343,6 +414,7 @@ func (hni *HyperNodesInfo) setParent(member, parent string) error { } if currentParent != parent { + hni.builtErrHyperNode = parent return fmt.Errorf("HyperNode %s already has a parent %s, and cannot set another parent %s", member, currentParent, parent) } @@ -410,8 +482,29 @@ func (hni *HyperNodesInfo) exactMatchMember(selector topologyv1alpha1.MemberSele } func (hni *HyperNodesInfo) updateAncestors(name string) error { + if err := hni.rebuildCache(name); err != nil { + hni.setReady(false) + return err + } + // When last time BuildHyperNodeCache has an err that a hyperNode has multi parents, after the parent is updated + // we should find the parent hyperNode to rebuild the correct cache. + if hni.builtErrHyperNode == name { + klog.InfoS("Rebuilt parent hyperNode", "name", hni.builtErrHyperNode) + leafHyperNods := hni.GetLeafNodes(name) + for leaf := range leafHyperNods { + if err := hni.rebuildCache(leaf); err != nil { + return err + } + } + + } + hni.builtErrHyperNode = "" + hni.setReady(true) + return nil +} + +func (hni *HyperNodesInfo) rebuildCache(name string) error { ancestors := hni.GetAncestors(name) - fmt.Println("acxx", ancestors) // Clear realNodesSet of hyperNode before rebuild hyperNode cache. for ancestor := range ancestors { delete(hni.realNodesSet, ancestor) @@ -429,12 +522,11 @@ func (hni *HyperNodesInfo) updateAncestors(name string) error { for ancestor := range ancestors { if hn, ok := hni.hyperNodes[ancestor]; ok { if err := hni.BuildHyperNodeCache(hn, processed, ancestorsChain, ancestors, nodes); err != nil { - hni.setReady(false) return err } } } - hni.setReady(true) + return nil } @@ -517,3 +609,34 @@ func (hni *HyperNodesInfo) HyperNodesInfo() map[string]string { } return actualHyperNodes } + +// NodeRegexMatchLeafHyperNode checks if a given node regex matches the MemberSelector of a HyperNode. +func (hni *HyperNodesInfo) NodeRegexMatchLeafHyperNode(nodeName string, hyperNodeName string) (bool, error) { + hn, ok := hni.hyperNodes[hyperNodeName] + if !ok { + return false, fmt.Errorf("HyperNode %s not found in cache", hyperNodeName) + } + + for _, member := range hn.HyperNode.Spec.Members { + if member.Type == topologyv1alpha1.MemberTypeNode { + if hni.nodeMatchRegexSelector(nodeName, member.Selector) { + return true, nil + } + } + } + return false, nil +} + +// nodeMatchRegexSelector checks if a node matches a MemberSelector. +func (hni *HyperNodesInfo) nodeMatchRegexSelector(nodeName string, selector topologyv1alpha1.MemberSelector) bool { + if selector.RegexMatch == nil { + return false + } + + reg, err := regexp.Compile(selector.RegexMatch.Pattern) + if err != nil { + klog.ErrorS(err, "Failed to compile regex pattern", "pattern", selector.RegexMatch.Pattern) + return false + } + return reg.MatchString(nodeName) +} diff --git a/pkg/scheduler/api/hyper_node_info_test.go b/pkg/scheduler/api/hyper_node_info_test.go index 802cc886a1..e4e4f17eed 100644 --- a/pkg/scheduler/api/hyper_node_info_test.go +++ b/pkg/scheduler/api/hyper_node_info_test.go @@ -17,7 +17,6 @@ limitations under the License. package api import ( - "fmt" "log" "testing" @@ -31,9 +30,18 @@ import ( func TestHyperNodesInfo_UpdateHyperNode_Normal(t *testing.T) { selector := "exact" - s0 := BuildHyperNode("s0", 1, topologyv1alpha1.MemberTypeNode, []string{"node-0", "node-1"}, selector) - s1 := BuildHyperNode("s1", 1, topologyv1alpha1.MemberTypeNode, []string{"node-2", "node-3"}, selector) - s2 := BuildHyperNode("s2", 2, topologyv1alpha1.MemberTypeHyperNode, []string{"s1", "s0"}, selector) + s0 := BuildHyperNode("s0", 1, []MemberConfig{ + {"node-0", topologyv1alpha1.MemberTypeNode, selector}, + {"node-1", topologyv1alpha1.MemberTypeNode, selector}, + }) + s1 := BuildHyperNode("s1", 1, []MemberConfig{ + {"node-2", topologyv1alpha1.MemberTypeNode, selector}, + {"node-3", topologyv1alpha1.MemberTypeNode, selector}, + }) + s2 := BuildHyperNode("s2", 2, []MemberConfig{ + {"s1", topologyv1alpha1.MemberTypeHyperNode, selector}, + {"s0", topologyv1alpha1.MemberTypeHyperNode, selector}, + }) initialHyperNodes := []*topologyv1alpha1.HyperNode{s2, s0, s1} tests := []struct { @@ -87,11 +95,21 @@ func TestHyperNodesInfo_UpdateHyperNode_Normal(t *testing.T) { func TestHyperNodesInfo_UpdateHyperNode_WithCycle(t *testing.T) { selector := "exact" - s0 := BuildHyperNode("s0", 1, topologyv1alpha1.MemberTypeNode, []string{"node-0", "node-1"}, selector) - s1 := BuildHyperNode("s1", 2, topologyv1alpha1.MemberTypeHyperNode, []string{"s0", "s3"}, selector) - s11 := BuildHyperNode("s1", 2, topologyv1alpha1.MemberTypeHyperNode, []string{"s0"}, selector) - s2 := BuildHyperNode("s2", 3, topologyv1alpha1.MemberTypeHyperNode, []string{"s1"}, selector) - s3 := BuildHyperNode("s3", 4, topologyv1alpha1.MemberTypeHyperNode, []string{"s2"}, selector) + s0 := BuildHyperNode("s0", 1, []MemberConfig{ + {"node-0", topologyv1alpha1.MemberTypeNode, selector}, + {"node-1", topologyv1alpha1.MemberTypeNode, selector}}) + s1 := BuildHyperNode("s1", 2, []MemberConfig{ + {"s0", topologyv1alpha1.MemberTypeHyperNode, selector}, + {"s3", topologyv1alpha1.MemberTypeHyperNode, selector}}) + s11 := BuildHyperNode("s1", 2, []MemberConfig{ + {"s0", topologyv1alpha1.MemberTypeHyperNode, selector}, + }) + s2 := BuildHyperNode("s2", 3, []MemberConfig{ + {"s1", topologyv1alpha1.MemberTypeHyperNode, selector}, + }) + s3 := BuildHyperNode("s3", 4, []MemberConfig{ + {"s2", topologyv1alpha1.MemberTypeHyperNode, selector}, + }) initialHyperNodes := []*topologyv1alpha1.HyperNode{s2, s0, s1, s3} tests := []struct { @@ -139,10 +157,10 @@ func TestHyperNodesInfo_UpdateHyperNode_WithCycle(t *testing.T) { } assert.Equal(t, false, hni.Ready()) - fmt.Println("begin updatexxx") + + log.Println("begin update...") // update and resolve dependency. err := hni.UpdateHyperNode(s11) - assert.NoError(t, err) actualHyperNodes := hni.HyperNodesInfo() assert.Equal(t, tt.expectedHyperNodesInfo, actualHyperNodes) @@ -161,10 +179,19 @@ func TestHyperNodesInfo_UpdateHyperNode_WithCycle(t *testing.T) { func TestHyperNodesInfo_UpdateHyperNode_MultipleParents(t *testing.T) { selector := "exact" - s0 := BuildHyperNode("s0", 1, topologyv1alpha1.MemberTypeNode, []string{"node-0", "node-1"}, selector) - s1 := BuildHyperNode("s1", 2, topologyv1alpha1.MemberTypeHyperNode, []string{"s0"}, selector) - s2 := BuildHyperNode("s2", 3, topologyv1alpha1.MemberTypeHyperNode, []string{"s0"}, selector) - s22 := BuildHyperNode("s2", 3, topologyv1alpha1.MemberTypeHyperNode, []string{"s1"}, selector) + s0 := BuildHyperNode("s0", 1, []MemberConfig{ + {Name: "node-0", Type: topologyv1alpha1.MemberTypeNode, Selector: selector}, + {Name: "node-1", Type: topologyv1alpha1.MemberTypeNode, Selector: selector}, + }) + s1 := BuildHyperNode("s1", 2, []MemberConfig{ + {Name: "s0", Type: topologyv1alpha1.MemberTypeHyperNode, Selector: selector}, + }) + s2 := BuildHyperNode("s2", 3, []MemberConfig{ + {Name: "s0", Type: topologyv1alpha1.MemberTypeHyperNode, Selector: selector}, + }) + s22 := BuildHyperNode("s2", 3, []MemberConfig{ + {Name: "s1", Type: topologyv1alpha1.MemberTypeHyperNode, Selector: selector}, + }) initialHyperNodes := []*topologyv1alpha1.HyperNode{s0, s1, s2} tests := []struct { @@ -228,3 +255,258 @@ func TestHyperNodesInfo_UpdateHyperNode_MultipleParents(t *testing.T) { }) } } + +func TestHyperNodesInfo_GetRegexSelectorLeafHyperNodes(t *testing.T) { + exactSelector := "exact" + regexSelector := "regex" + s0 := BuildHyperNode("s0", 1, []MemberConfig{ + {"node-1", topologyv1alpha1.MemberTypeNode, exactSelector}, + {"node-1", topologyv1alpha1.MemberTypeNode, exactSelector}, + }) + s1 := BuildHyperNode("s1", 1, []MemberConfig{ + {"node-2", topologyv1alpha1.MemberTypeNode, regexSelector}, + {"node-3", topologyv1alpha1.MemberTypeNode, regexSelector}, + }) + s2 := BuildHyperNode("s2", 1, []MemberConfig{ + {"node-4", topologyv1alpha1.MemberTypeNode, regexSelector}, + {"node-5", topologyv1alpha1.MemberTypeHyperNode, exactSelector}, + }) + s3 := BuildHyperNode("s3", 1, []MemberConfig{ + {"node-6", topologyv1alpha1.MemberTypeNode, regexSelector}, + {"node-7", topologyv1alpha1.MemberTypeNode, exactSelector}, + }) + s4 := BuildHyperNode("s4", 2, []MemberConfig{ + {"s0", topologyv1alpha1.MemberTypeHyperNode, exactSelector}, + {"s1", topologyv1alpha1.MemberTypeHyperNode, exactSelector}, + }) + s5 := BuildHyperNode("s5", 2, []MemberConfig{ + {"s2", topologyv1alpha1.MemberTypeHyperNode, regexSelector}, + {"s3", topologyv1alpha1.MemberTypeHyperNode, exactSelector}, + }) + s6 := BuildHyperNode("s6", 3, []MemberConfig{ + {"s4", topologyv1alpha1.MemberTypeHyperNode, exactSelector}, + {"s5", topologyv1alpha1.MemberTypeHyperNode, exactSelector}, + }) + tests := []struct { + name string + hyperNods []*topologyv1alpha1.HyperNode + want sets.Set[string] + }{ + { + name: "get all leaf hyperNodes correctly", + hyperNods: []*topologyv1alpha1.HyperNode{s0, s1, s2, s3, s4, s5, s6}, + want: sets.New[string]("s1", "s3"), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + hni := NewHyperNodesInfo(nil) + for _, hn := range tt.hyperNods { + hni.hyperNodes[hn.Name] = NewHyperNodeInfo(hn) + } + assert.Equalf(t, tt.want, hni.GetRegexSelectorLeafHyperNodes(), "GetRegexSelcectorLeafHyperNodes()") + }) + } +} + +func TestHyperNodesInfo_GetLeafNodes(t *testing.T) { + selector := "exact" + s0 := BuildHyperNode("s0", 1, []MemberConfig{ + {"node-0", topologyv1alpha1.MemberTypeNode, selector}, + {"node-1", topologyv1alpha1.MemberTypeNode, selector}, + }) + s1 := BuildHyperNode("s1", 1, []MemberConfig{ + {"node-2", topologyv1alpha1.MemberTypeNode, selector}, + {"node-3", topologyv1alpha1.MemberTypeNode, selector}, + }) + s2 := BuildHyperNode("s2", 2, []MemberConfig{ + {"s1", topologyv1alpha1.MemberTypeHyperNode, selector}, + {"s0", topologyv1alpha1.MemberTypeHyperNode, selector}, + }) + s3 := BuildHyperNode("s3", 3, []MemberConfig{ + {"s2", topologyv1alpha1.MemberTypeHyperNode, selector}, + }) + initialHyperNodes := []*topologyv1alpha1.HyperNode{s2, s3, s0, s1} + + tests := []struct { + name string + hyperNodeName string + initialHyperNodes []*topologyv1alpha1.HyperNode + expectedLeafNodes sets.Set[string] + }{ + { + name: "Get correct leaf hyperNodes", + hyperNodeName: "s3", + initialHyperNodes: initialHyperNodes, + expectedLeafNodes: sets.New[string]("s0", "s1"), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + informerFactory := informers.NewSharedInformerFactory(fakeclientset.NewClientset(), 0) + nodeLister := informerFactory.Core().V1().Nodes().Lister() + hni := NewHyperNodesInfo(nodeLister) + + for _, node := range tt.initialHyperNodes { + err := hni.UpdateHyperNode(node) + assert.NoError(t, err) + } + + assert.Equal(t, tt.expectedLeafNodes, hni.GetLeafNodes(tt.hyperNodeName)) + }) + } +} + +func TestHyperNodesInfo_NodeRegexMatchHyperNode(t *testing.T) { + exactSelector := "exact" + regexSelector := "regex" + s0 := BuildHyperNode("s0", 1, []MemberConfig{ + {"node-0", topologyv1alpha1.MemberTypeNode, exactSelector}, + {"node-1", topologyv1alpha1.MemberTypeNode, exactSelector}, + }) + s1 := BuildHyperNode("s1", 1, []MemberConfig{ + {"^prefix", topologyv1alpha1.MemberTypeNode, regexSelector}, + {"node-2", topologyv1alpha1.MemberTypeNode, regexSelector}, + }) + s2 := BuildHyperNode("s2", 1, []MemberConfig{ + {"^not-match-prefix", topologyv1alpha1.MemberTypeHyperNode, regexSelector}, + {"node-3", topologyv1alpha1.MemberTypeNode, regexSelector}, + }) + s3 := BuildHyperNode("s3", 1, []MemberConfig{ + {"node-4", topologyv1alpha1.MemberTypeHyperNode, regexSelector}, + }) + initialHyperNodes := []*topologyv1alpha1.HyperNode{s2, s3, s0, s1} + + tests := []struct { + name string + nodeName string + hyperNodeName string + expectedMatch bool + expectedErr bool + }{ + { + name: "not match", + nodeName: "node-0", + hyperNodeName: "s0", + expectedMatch: false, + expectedErr: false, + }, + { + name: "not match", + nodeName: "node-1", + hyperNodeName: "s0", + expectedMatch: false, + expectedErr: false, + }, + { + name: "match", + nodeName: "prefix-1", + hyperNodeName: "s1", + expectedMatch: true, + expectedErr: false, + }, + { + name: "match", + nodeName: "node-2", + hyperNodeName: "s1", + expectedMatch: true, + expectedErr: false, + }, + { + name: "not match", + nodeName: "not-match-prefix", + hyperNodeName: "s2", + expectedMatch: false, + expectedErr: false, + }, + { + name: "not match", + nodeName: "node-4", + hyperNodeName: "s3", + expectedMatch: false, + expectedErr: false, + }, + } + + informerFactory := informers.NewSharedInformerFactory(fakeclientset.NewClientset(), 0) + nodeLister := informerFactory.Core().V1().Nodes().Lister() + hni := NewHyperNodesInfo(nodeLister) + for _, node := range initialHyperNodes { + err := hni.UpdateHyperNode(node) + assert.NoError(t, err) + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + match, err := hni.NodeRegexMatchLeafHyperNode(tt.nodeName, tt.hyperNodeName) + assert.Equal(t, tt.expectedMatch, match) + assert.Equal(t, tt.expectedErr, err != nil) + }) + } +} + +func TestHyperNodesInfo_GetAncestors(t *testing.T) { + selector := "exact" + s0 := BuildHyperNode("s0", 1, []MemberConfig{ + {"node-0", topologyv1alpha1.MemberTypeNode, selector}, + {"node-1", topologyv1alpha1.MemberTypeNode, selector}, + }) + s1 := BuildHyperNode("s1", 1, []MemberConfig{ + {"node-0", topologyv1alpha1.MemberTypeNode, selector}, + {"node-1", topologyv1alpha1.MemberTypeNode, selector}, + }) + s2 := BuildHyperNode("s2", 2, []MemberConfig{ + {"s0", topologyv1alpha1.MemberTypeHyperNode, selector}, + {"s1", topologyv1alpha1.MemberTypeHyperNode, selector}, + }) + s3 := BuildHyperNode("s3", 3, []MemberConfig{ + {"s2", topologyv1alpha1.MemberTypeHyperNode, selector}, + }) + s4 := BuildHyperNode("s4", 4, []MemberConfig{ + {"s3", topologyv1alpha1.MemberTypeHyperNode, selector}, + }) + initialHyperNodes := []*topologyv1alpha1.HyperNode{s0, s4, s1, s2, s3} + + tests := []struct { + name string + hyperNodeName string + initialHyperNodes []*topologyv1alpha1.HyperNode + expectedAncestors sets.Set[string] + }{ + { + name: "Get ancestors for leaf node", + hyperNodeName: "s0", + initialHyperNodes: initialHyperNodes, + expectedAncestors: sets.New[string]("s0", "s2", "s3", "s4"), + }, + { + name: "Get ancestors for intermediate node", + hyperNodeName: "s2", + initialHyperNodes: initialHyperNodes, + expectedAncestors: sets.New[string]("s2", "s3", "s4"), + }, + { + name: "Get ancestors for root node", + hyperNodeName: "s4", + initialHyperNodes: initialHyperNodes, + expectedAncestors: sets.New[string]("s4"), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + informerFactory := informers.NewSharedInformerFactory(fakeclientset.NewClientset(), 0) + nodeLister := informerFactory.Core().V1().Nodes().Lister() + hni := NewHyperNodesInfo(nodeLister) + + for _, node := range tt.initialHyperNodes { + err := hni.UpdateHyperNode(node) + assert.NoError(t, err) + } + + actualAncestors := hni.GetAncestors(tt.hyperNodeName) + assert.Equal(t, tt.expectedAncestors, actualAncestors) + }) + } +} diff --git a/pkg/scheduler/api/test_utils.go b/pkg/scheduler/api/test_utils.go index 9e997c62c9..a017ad96e9 100644 --- a/pkg/scheduler/api/test_utils.go +++ b/pkg/scheduler/api/test_utils.go @@ -134,8 +134,13 @@ func BuildPodgroup(name, ns string, minMember int32, minResource v1.ResourceList } } -// BuildHyperNode builds a hyperNode. -func BuildHyperNode(name string, tier int, memberType topologyv1alpha1.MemberType, members []string, selector string) *topologyv1alpha1.HyperNode { +type MemberConfig struct { + Name string + Type topologyv1alpha1.MemberType + Selector string +} + +func BuildHyperNode(name string, tier int, members []MemberConfig) *topologyv1alpha1.HyperNode { hn := &topologyv1alpha1.HyperNode{ ObjectMeta: metav1.ObjectMeta{ Name: name, @@ -147,17 +152,20 @@ func BuildHyperNode(name string, tier int, memberType topologyv1alpha1.MemberTyp } for i, member := range members { - hn.Spec.Members[i] = topologyv1alpha1.MemberSpec{ - Type: memberType, - } - if selector == "exact" { - hn.Spec.Members[i].Selector.ExactMatch = &topologyv1alpha1.ExactMatch{Name: member} - continue + memberSpec := topologyv1alpha1.MemberSpec{ + Type: member.Type, } - if selector == "regex" { - hn.Spec.Members[i].Selector.RegexMatch = &topologyv1alpha1.RegexMatch{Pattern: member} - continue + switch member.Selector { + case "exact": + memberSpec.Selector.ExactMatch = &topologyv1alpha1.ExactMatch{Name: member.Name} + case "regex": + memberSpec.Selector.RegexMatch = &topologyv1alpha1.RegexMatch{Pattern: member.Name} + default: + return nil } + + hn.Spec.Members[i] = memberSpec } + return hn } diff --git a/pkg/scheduler/api/test_utils_test.go b/pkg/scheduler/api/test_utils_test.go index e7a47b4aa8..7dc15e7656 100644 --- a/pkg/scheduler/api/test_utils_test.go +++ b/pkg/scheduler/api/test_utils_test.go @@ -30,18 +30,17 @@ func TestBuildHyperNode(t *testing.T) { name string hyperNodeName string tier int - memberType topologyv1alpha1.MemberType - members []string - selector string + members []MemberConfig want *topologyv1alpha1.HyperNode }{ { name: "build leaf hyperNode", hyperNodeName: "s0", tier: 1, - memberType: topologyv1alpha1.MemberTypeNode, - members: []string{"node-1", "node-2"}, - selector: "exact", + members: []MemberConfig{ + {"node-1", topologyv1alpha1.MemberTypeNode, "regex"}, + {"node-2", topologyv1alpha1.MemberTypeNode, "exact"}, + }, want: &topologyv1alpha1.HyperNode{ ObjectMeta: metav1.ObjectMeta{ Name: "s0", @@ -49,7 +48,7 @@ func TestBuildHyperNode(t *testing.T) { Spec: topologyv1alpha1.HyperNodeSpec{ Tier: 1, Members: []topologyv1alpha1.MemberSpec{ - {Type: topologyv1alpha1.MemberTypeNode, Selector: topologyv1alpha1.MemberSelector{ExactMatch: &topologyv1alpha1.ExactMatch{Name: "node-1"}}}, + {Type: topologyv1alpha1.MemberTypeNode, Selector: topologyv1alpha1.MemberSelector{RegexMatch: &topologyv1alpha1.RegexMatch{Pattern: "node-1"}}}, {Type: topologyv1alpha1.MemberTypeNode, Selector: topologyv1alpha1.MemberSelector{ExactMatch: &topologyv1alpha1.ExactMatch{Name: "node-2"}}}, }, }, @@ -59,9 +58,10 @@ func TestBuildHyperNode(t *testing.T) { name: "build non-leaf hyperNode", hyperNodeName: "s4", tier: 2, - selector: "exact", - memberType: topologyv1alpha1.MemberTypeHyperNode, - members: []string{"s0", "s1"}, + members: []MemberConfig{ + {"s0", topologyv1alpha1.MemberTypeHyperNode, "exact"}, + {"s1", topologyv1alpha1.MemberTypeHyperNode, "exact"}, + }, want: &topologyv1alpha1.HyperNode{ ObjectMeta: metav1.ObjectMeta{ Name: "s4", @@ -78,7 +78,7 @@ func TestBuildHyperNode(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - assert.Equalf(t, tt.want, BuildHyperNode(tt.hyperNodeName, tt.tier, tt.memberType, tt.members, tt.selector), "BuildHyperNode(%v, %v, %v, %v)", tt.hyperNodeName, tt.tier, tt.memberType, tt.members) + assert.Equalf(t, tt.want, BuildHyperNode(tt.hyperNodeName, tt.tier, tt.members), "BuildHyperNode(%v, %v, %v)", tt.hyperNodeName, tt.tier, tt.members) }) } } diff --git a/pkg/scheduler/cache/cache.go b/pkg/scheduler/cache/cache.go index 27f7b5b2d0..382eebda0a 100644 --- a/pkg/scheduler/cache/cache.go +++ b/pkg/scheduler/cache/cache.go @@ -62,6 +62,7 @@ import ( vcinformer "volcano.sh/apis/pkg/client/informers/externalversions" cpuinformerv1 "volcano.sh/apis/pkg/client/informers/externalversions/nodeinfo/v1alpha1" vcinformerv1 "volcano.sh/apis/pkg/client/informers/externalversions/scheduling/v1beta1" + topologyinformerv1alpha1 "volcano.sh/apis/pkg/client/informers/externalversions/topology/v1alpha1" "volcano.sh/volcano/cmd/scheduler/app/options" "volcano.sh/volcano/pkg/features" schedulingapi "volcano.sh/volcano/pkg/scheduler/api" @@ -107,6 +108,7 @@ type SchedulerCache struct { podInformer infov1.PodInformer nodeInformer infov1.NodeInformer + hyperNodeInformer topologyinformerv1alpha1.HyperNodeInformer podGroupInformerV1beta1 vcinformerv1.PodGroupInformer queueInformerV1beta1 vcinformerv1.QueueInformer pvInformer infov1.PersistentVolumeInformer @@ -139,9 +141,10 @@ type SchedulerCache struct { NamespaceCollection map[string]*schedulingapi.NamespaceCollection - errTasks workqueue.RateLimitingInterface - nodeQueue workqueue.RateLimitingInterface - DeletedJobs workqueue.RateLimitingInterface + errTasks workqueue.RateLimitingInterface + nodeQueue workqueue.RateLimitingInterface + DeletedJobs workqueue.RateLimitingInterface + hyperNodesQueue workqueue.TypedRateLimitingInterface[string] informerFactory informers.SharedInformerFactory vcInformerFactory vcinformer.SharedInformerFactory @@ -572,6 +575,7 @@ func newSchedulerCache(config *rest.Config, schedulerNames []string, defaultQueu errTasks: workqueue.NewRateLimitingQueue(errTaskRateLimiter), nodeQueue: workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter()), DeletedJobs: workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter()), + hyperNodesQueue: workqueue.NewTypedRateLimitingQueue[string](workqueue.DefaultTypedControllerRateLimiter[string]()), kubeClient: kubeClient, vcClient: vcClient, restConfig: config, @@ -798,8 +802,8 @@ func (sc *SchedulerCache) addEventHandler() { }) } - hyperNodeInformer := sc.vcInformerFactory.Topology().V1alpha1().HyperNodes() - hyperNodeInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ + sc.hyperNodeInformer = sc.vcInformerFactory.Topology().V1alpha1().HyperNodes() + sc.hyperNodeInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ AddFunc: sc.AddHyperNode, UpdateFunc: sc.UpdateHyperNode, DeleteFunc: sc.DeleteHyperNode, @@ -815,6 +819,9 @@ func (sc *SchedulerCache) Run(stopCh <-chan struct{}) { go wait.Until(sc.runNodeWorker, 0, stopCh) } + // Sync hyperNode. + go wait.Until(sc.processSyncHyperNode, 0, stopCh) + // Re-sync error tasks. go wait.Until(sc.processResyncTask, 0, stopCh) @@ -1235,6 +1242,29 @@ func (sc *SchedulerCache) processSyncNode() bool { return true } +func (sc *SchedulerCache) processSyncHyperNode() { + worker := func() bool { + name, shutdown := sc.hyperNodesQueue.Get() + if shutdown { + return false + } + defer sc.hyperNodesQueue.Done(name) + + klog.V(5).Infof("started sync hyperNode %s", name) + err := sc.SyncHyperNode(name) + if err == nil { + sc.hyperNodesQueue.Forget(name) + return true + } + + klog.ErrorS(err, "Failed to sync hyperNode, retry it.", "name", name) + sc.hyperNodesQueue.AddRateLimited(name) + return true + } + for worker() { + } +} + // AddBindTask add task to be bind to a cache which consumes by go runtime func (sc *SchedulerCache) AddBindTask(taskInfo *schedulingapi.TaskInfo) error { klog.V(5).Infof("add bind task %v/%v", taskInfo.Namespace, taskInfo.Name) diff --git a/pkg/scheduler/cache/cache_mock.go b/pkg/scheduler/cache/cache_mock.go index be92bef3c3..689021ec88 100644 --- a/pkg/scheduler/cache/cache_mock.go +++ b/pkg/scheduler/cache/cache_mock.go @@ -107,6 +107,7 @@ func newMockSchedulerCache(schedulerName string) *SchedulerCache { errTasks: workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter()), nodeQueue: workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter()), DeletedJobs: workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter()), + hyperNodesQueue: workqueue.NewTypedRateLimitingQueue[string](workqueue.DefaultTypedControllerRateLimiter[string]()), kubeClient: fake.NewSimpleClientset(), vcClient: fakevcClient.NewSimpleClientset(), restConfig: nil, diff --git a/pkg/scheduler/cache/dumper.go b/pkg/scheduler/cache/dumper.go index 6281aace5d..38b94781dd 100644 --- a/pkg/scheduler/cache/dumper.go +++ b/pkg/scheduler/cache/dumper.go @@ -118,7 +118,7 @@ func (d *Dumper) printHyperNodeInfo(HyperNodesSetByTier map[int]sets.Set[string] data.WriteString("\n") for tier, hyperNodes := range HyperNodesSetByTier { for hyperNode := range hyperNodes { - data.WriteString(fmt.Sprintf("tier: %d, HyperNodeName: %s, Nodes: %s\n", tier, hyperNode, HyperNodes[hyperNode])) + data.WriteString(fmt.Sprintf("Tier: %d, HyperNodeName: %s, Nodes: %s\n", tier, hyperNode, HyperNodes[hyperNode])) } } data.WriteString("\n") diff --git a/pkg/scheduler/cache/event_handlers.go b/pkg/scheduler/cache/event_handlers.go index a03755464c..ab5b767b18 100644 --- a/pkg/scheduler/cache/event_handlers.go +++ b/pkg/scheduler/cache/event_handlers.go @@ -552,6 +552,7 @@ func (sc *SchedulerCache) AddNode(obj interface{}) { return } sc.nodeQueue.Add(node.Name) + sc.hyperNodesQueue.Add(string(hyperNodeEventSourceNode) + "/" + node.Name) } // UpdateNode update node to scheduler cache @@ -587,6 +588,7 @@ func (sc *SchedulerCache) DeleteNode(obj interface{}) { return } sc.nodeQueue.Add(node.Name) + sc.hyperNodesQueue.Add(string(hyperNodeEventSourceNode) + "/" + node.Name) } func (sc *SchedulerCache) SyncNode(nodeName string) error { @@ -695,6 +697,72 @@ func (sc *SchedulerCache) DeleteCSINode(obj interface{}) { sc.nodeQueue.Add(csiNode.Name) } +func (sc *SchedulerCache) SyncHyperNode(name string) error { + eventSource := getHyperNodeEventSource(name) + if eventSource == nil { + klog.ErrorS(nil, "Failed to get event source for node <%s>", "eventSource", name) + // just return nil and not process again as it's an invalid event. + return nil + } + switch eventSource[0] { + case string(hyperNodeEventSourceNode): + return sc.triggerUpdateHyperNode(eventSource[1]) + case string(hyperNodeEventSourceHyperNode): + return sc.processHyperNode(eventSource[1]) + default: + // should never happen. + klog.ErrorS(nil, "Unknown event source", "name", eventSource) + } + return nil +} + +func (sc *SchedulerCache) triggerUpdateHyperNode(name string) error { + sc.HyperNodesInfo.Lock() + defer sc.HyperNodesInfo.Unlock() + + leafNodes := sc.HyperNodesInfo.GetRegexSelectorLeafHyperNodes() + if len(leafNodes) == 0 { + klog.V(3).InfoS("No need to update hyperNode cache when node added or deleted") + return nil + } + + for leafNode := range leafNodes { + hn := sc.HyperNodesInfo.HyperNode(leafNode) + if hn == nil { + klog.ErrorS(nil, "Get empty hyperNode", "hyperNodeName", leafNode) + continue + } + match, err := sc.HyperNodesInfo.NodeRegexMatchLeafHyperNode(name, hn.Name) + if err != nil { + klog.ErrorS(err, "Failed to get node regex match leaf hyperNode", "nodeName", name, "hyperNodeName", hn.Name) + continue + } + if !match { + continue + } + + klog.V(3).InfoS("Update leaf hyperNode and its ancestors when node added or deleted", "nodeName", name, "hyperNodeName", hn.Name) + if err := sc.updateHyperNode(hn); err != nil { + return fmt.Errorf("faield to update hyperNode: %v", err) + } + } + return nil +} + +func (sc *SchedulerCache) processHyperNode(name string) error { + sc.HyperNodesInfo.Lock() + defer sc.HyperNodesInfo.Unlock() + + hn, err := sc.hyperNodeInformer.Lister().Get(name) + if err != nil { + if !errors.IsNotFound(err) { + return fmt.Errorf("failed to get hyperNode <%s>: %v", name, err) + } + return sc.deleteHyperNode(name) + } + return sc.updateHyperNode(hn) +} + func getJobID(pg *schedulingapi.PodGroup) schedulingapi.JobID { return schedulingapi.JobID(fmt.Sprintf("%s/%s", pg.Namespace, pg.Name)) } @@ -1263,46 +1331,27 @@ func (sc *SchedulerCache) setCSIResourceOnNode(csiNode *sv1.CSINode, node *v1.No } } -// AddHyperNode adds HyperNode and rebuild HyperNodesInfo cache. +// AddHyperNode adds hyperNode name to the hyperNodesQueue. func (sc *SchedulerCache) AddHyperNode(obj interface{}) { - hyperNode, ok := obj.(*topologyv1alpha1.HyperNode) + hn, ok := obj.(*topologyv1alpha1.HyperNode) if !ok { klog.ErrorS(nil, "Cannot convert to *topologyv1alpha1.HyperNode", "type", reflect.TypeOf(obj)) return } - - sc.HyperNodesInfo.Lock() - defer sc.HyperNodesInfo.Unlock() - - if err := sc.updateHyperNode(hyperNode); err != nil { - klog.ErrorS(err, "Failed to update HyperNode", "type", reflect.TypeOf(hyperNode)) - return - } - klog.V(3).InfoS("Added HyperNode in cache", "name", hyperNode.Name) + sc.hyperNodesQueue.Add(string(hyperNodeEventSourceHyperNode) + "/" + hn.Name) } -// UpdateHyperNode updates HyperNode and rebuild HyperNodesInfo cache, it does three things in order: -// 1.update parent map if members reduced. -// 2.reset current hyperNode and its ancestors' cache. +// UpdateHyperNode adds hyperNode name to the hyperNodesQueue. func (sc *SchedulerCache) UpdateHyperNode(oldObj, newObj interface{}) { newHyperNode, ok := newObj.(*topologyv1alpha1.HyperNode) if !ok { klog.ErrorS(nil, "Cannot convert newObj to *topologyv1alpha1.HyperNode: %v", reflect.TypeOf(newObj)) return } - - sc.HyperNodesInfo.Lock() - defer sc.HyperNodesInfo.Unlock() - - if err := sc.updateHyperNode(newHyperNode); err != nil { - klog.ErrorS(err, "Failed to update HyperNode", "name", newHyperNode.Name) - return - } - klog.V(3).InfoS("Updated HyperNode in cache", "name", newHyperNode.Name) + sc.hyperNodesQueue.Add(string(hyperNodeEventSourceHyperNode) + "/" + newHyperNode.Name) } -// DeleteHyperNode deletes HyperNode and rebuild HyperNodesInfo cache. -// It clears current hyperNode and update ancestors' cache. +// DeleteHyperNode adds hyperNode name to the hyperNodesQueue. func (sc *SchedulerCache) DeleteHyperNode(obj interface{}) { var hn *topologyv1alpha1.HyperNode switch t := obj.(type) { @@ -1319,16 +1368,18 @@ func (sc *SchedulerCache) DeleteHyperNode(obj interface{}) { klog.ErrorS(nil, "Cannot convert to HyperNode", "type", reflect.TypeOf(t)) return } + sc.hyperNodesQueue.Add(string(hyperNodeEventSourceHyperNode) + "/" + hn.Name) +} - sc.HyperNodesInfo.Lock() - defer sc.HyperNodesInfo.Unlock() - - if err := sc.HyperNodesInfo.DeleteHyperNode(hn.Name); err != nil { - klog.ErrorS(err, "Failed to delete HyperNode", "name", hn.Name) - } - klog.V(3).InfoS("Deleted HyperNode from cache", "name", hn.Name) +// UpdateHyperNode updates HyperNode and rebuild HyperNodesInfo cache, it does three things in order: +// 1.update parent map if members reduced. +// 2.reset current hyperNode and its ancestors' cache. +func (sc *SchedulerCache) updateHyperNode(hn *topologyv1alpha1.HyperNode) error { + return sc.HyperNodesInfo.UpdateHyperNode(hn) } -func (sc *SchedulerCache) updateHyperNode(hyperNode *topologyv1alpha1.HyperNode) error { - return sc.HyperNodesInfo.UpdateHyperNode(hyperNode) +// deleteHyperNode deletes HyperNode and rebuild HyperNodesInfo cache. +// It clears current hyperNode and update ancestors' cache. +func (sc *SchedulerCache) deleteHyperNode(name string) error { + return sc.HyperNodesInfo.DeleteHyperNode(name) } diff --git a/pkg/scheduler/cache/event_handlers_test.go b/pkg/scheduler/cache/event_handlers_test.go index 189746c912..8553918d11 100644 --- a/pkg/scheduler/cache/event_handlers_test.go +++ b/pkg/scheduler/cache/event_handlers_test.go @@ -715,20 +715,48 @@ func TestSchedulerCache_SyncNode(t *testing.T) { func TestSchedulerCache_AddHyperNode(t *testing.T) { exactSelector := "exact" - s5 := schedulingapi.BuildHyperNode("s5", 2, topologyv1alpha1.MemberTypeHyperNode, []string{"s2", "s3"}, exactSelector) - s4 := schedulingapi.BuildHyperNode("s4", 2, topologyv1alpha1.MemberTypeHyperNode, []string{"s0", "s1"}, exactSelector) - s3 := schedulingapi.BuildHyperNode("s3", 1, topologyv1alpha1.MemberTypeNode, []string{"node-6", "node-7"}, exactSelector) - s1 := schedulingapi.BuildHyperNode("s1", 1, topologyv1alpha1.MemberTypeNode, []string{"node-2", "node-3"}, exactSelector) - s2 := schedulingapi.BuildHyperNode("s2", 1, topologyv1alpha1.MemberTypeNode, []string{"node-4", "node-5"}, exactSelector) - s0 := schedulingapi.BuildHyperNode("s0", 1, topologyv1alpha1.MemberTypeNode, []string{"node-0", "node-1"}, exactSelector) + s5 := schedulingapi.BuildHyperNode("s5", 2, []schedulingapi.MemberConfig{ + {"s2", topologyv1alpha1.MemberTypeHyperNode, exactSelector}, + {"s3", topologyv1alpha1.MemberTypeHyperNode, exactSelector}, + }) + s4 := schedulingapi.BuildHyperNode("s4", 2, []schedulingapi.MemberConfig{ + {"s0", topologyv1alpha1.MemberTypeHyperNode, exactSelector}, + {"s1", topologyv1alpha1.MemberTypeHyperNode, exactSelector}, + }) + s3 := schedulingapi.BuildHyperNode("s3", 1, []schedulingapi.MemberConfig{ + {"node-6", topologyv1alpha1.MemberTypeNode, exactSelector}, + {"node-7", topologyv1alpha1.MemberTypeNode, exactSelector}, + }) + s1 := schedulingapi.BuildHyperNode("s1", 1, []schedulingapi.MemberConfig{ + {"node-2", topologyv1alpha1.MemberTypeNode, exactSelector}, + {"node-3", topologyv1alpha1.MemberTypeNode, exactSelector}, + }) + s2 := schedulingapi.BuildHyperNode("s2", 1, []schedulingapi.MemberConfig{ + {"node-4", topologyv1alpha1.MemberTypeNode, exactSelector}, + {"node-5", topologyv1alpha1.MemberTypeNode, exactSelector}, + }) + s0 := schedulingapi.BuildHyperNode("s0", 1, []schedulingapi.MemberConfig{ + {"node-0", topologyv1alpha1.MemberTypeNode, exactSelector}, + {"node-1", topologyv1alpha1.MemberTypeNode, exactSelector}, + }) initialHyperNodes0 := []*topologyv1alpha1.HyperNode{s5, s0, s4, s3, s1, s2} regexSelector := "regex" - s00 := schedulingapi.BuildHyperNode("s0", 1, topologyv1alpha1.MemberTypeNode, []string{"node-0", "node-1"}, regexSelector) - s10 := schedulingapi.BuildHyperNode("s1", 1, topologyv1alpha1.MemberTypeNode, []string{"node-[2-3]"}, regexSelector) - s20 := schedulingapi.BuildHyperNode("s2", 1, topologyv1alpha1.MemberTypeNode, []string{"^prefix"}, regexSelector) + s00 := schedulingapi.BuildHyperNode("s0", 1, []schedulingapi.MemberConfig{ + {"node-0", topologyv1alpha1.MemberTypeNode, exactSelector}, + {"node-1", topologyv1alpha1.MemberTypeNode, exactSelector}, + }) + s10 := schedulingapi.BuildHyperNode("s1", 1, []schedulingapi.MemberConfig{ + {"node-[2-3]", topologyv1alpha1.MemberTypeNode, regexSelector}, + }) + s20 := schedulingapi.BuildHyperNode("s2", 1, []schedulingapi.MemberConfig{ + {"^prefix", topologyv1alpha1.MemberTypeNode, regexSelector}, + }) initialHyperNodes1 := []*topologyv1alpha1.HyperNode{s5, s00, s4, s3, s10, s20} - s6 := schedulingapi.BuildHyperNode("s6", 3, topologyv1alpha1.MemberTypeHyperNode, []string{"s4", "s5"}, exactSelector) + s6 := schedulingapi.BuildHyperNode("s6", 3, []schedulingapi.MemberConfig{ + {"s4", topologyv1alpha1.MemberTypeHyperNode, exactSelector}, + {"s5", topologyv1alpha1.MemberTypeHyperNode, exactSelector}, + }) tests := []struct { name string nodes []*v1.Node @@ -836,15 +864,15 @@ func TestSchedulerCache_AddHyperNode(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { sc := NewDefaultMockSchedulerCache("volcano") - // Add some nodes to match regex selector. + // Add some initialNodes to match regex selector. for _, node := range tt.nodes { sc.nodeInformer.Informer().GetIndexer().Add(node) } for _, hyperNode := range tt.initialHyperNodes { - sc.AddHyperNode(hyperNode) + assert.NoError(t, sc.updateHyperNode(hyperNode)) } for _, hyperNode := range tt.hypeNodesToAdd { - sc.AddHyperNode(hyperNode) + assert.NoError(t, sc.updateHyperNode(hyperNode)) } assert.Equal(t, tt.expectedHyperNodesSetByTier, sc.HyperNodesInfo.HyperNodesSetByTier()) assert.Equal(t, tt.expectedRealNodesSet, sc.HyperNodesInfo.RealNodesSet()) @@ -857,14 +885,37 @@ func TestSchedulerCache_AddHyperNode(t *testing.T) { func TestSchedulerCache_Delete_Then_AddBack(t *testing.T) { exactSelector := "exact" - s5 := schedulingapi.BuildHyperNode("s5", 2, topologyv1alpha1.MemberTypeHyperNode, []string{"s2", "s3"}, exactSelector) - s4 := schedulingapi.BuildHyperNode("s4", 2, topologyv1alpha1.MemberTypeHyperNode, []string{"s0", "s1"}, exactSelector) - s3 := schedulingapi.BuildHyperNode("s3", 1, topologyv1alpha1.MemberTypeNode, []string{"node-6", "node-7"}, exactSelector) - s1 := schedulingapi.BuildHyperNode("s1", 1, topologyv1alpha1.MemberTypeNode, []string{"node-2", "node-3"}, exactSelector) - s2 := schedulingapi.BuildHyperNode("s2", 1, topologyv1alpha1.MemberTypeNode, []string{"node-4", "node-5"}, exactSelector) - s0 := schedulingapi.BuildHyperNode("s0", 1, topologyv1alpha1.MemberTypeNode, []string{"node-0", "node-1"}, exactSelector) - s6 := schedulingapi.BuildHyperNode("s6", 3, topologyv1alpha1.MemberTypeHyperNode, []string{"s4", "s5"}, exactSelector) + + s5 := schedulingapi.BuildHyperNode("s5", 2, []schedulingapi.MemberConfig{ + {"s2", topologyv1alpha1.MemberTypeHyperNode, exactSelector}, + {"s3", topologyv1alpha1.MemberTypeHyperNode, exactSelector}, + }) + s4 := schedulingapi.BuildHyperNode("s4", 2, []schedulingapi.MemberConfig{ + {"s0", topologyv1alpha1.MemberTypeHyperNode, exactSelector}, + {"s1", topologyv1alpha1.MemberTypeHyperNode, exactSelector}, + }) + s3 := schedulingapi.BuildHyperNode("s3", 1, []schedulingapi.MemberConfig{ + {"node-6", topologyv1alpha1.MemberTypeNode, exactSelector}, + {"node-7", topologyv1alpha1.MemberTypeNode, exactSelector}, + }) + s1 := schedulingapi.BuildHyperNode("s1", 1, []schedulingapi.MemberConfig{ + {"node-2", topologyv1alpha1.MemberTypeNode, exactSelector}, + {"node-3", topologyv1alpha1.MemberTypeNode, exactSelector}, + }) + s2 := schedulingapi.BuildHyperNode("s2", 1, []schedulingapi.MemberConfig{ + {"node-4", topologyv1alpha1.MemberTypeNode, exactSelector}, + {"node-5", topologyv1alpha1.MemberTypeNode, exactSelector}, + }) + s0 := schedulingapi.BuildHyperNode("s0", 1, []schedulingapi.MemberConfig{ + {"node-0", topologyv1alpha1.MemberTypeNode, exactSelector}, + {"node-1", topologyv1alpha1.MemberTypeNode, exactSelector}, + }) + s6 := schedulingapi.BuildHyperNode("s6", 3, []schedulingapi.MemberConfig{ + {"s4", topologyv1alpha1.MemberTypeHyperNode, exactSelector}, + {"s5", topologyv1alpha1.MemberTypeHyperNode, exactSelector}, + }) initialHyperNodes := []*topologyv1alpha1.HyperNode{s6, s5, s0, s4, s3, s1, s2} + tests := []struct { name string nodes []*v1.Node @@ -923,15 +974,15 @@ func TestSchedulerCache_Delete_Then_AddBack(t *testing.T) { assert.NoError(t, sc.nodeInformer.Informer().GetIndexer().Add(node)) } for _, hyperNode := range tt.initialHyperNodes { - sc.AddHyperNode(hyperNode) + assert.NoError(t, sc.updateHyperNode(hyperNode)) } for _, hyperNode := range tt.hypeNodesToDelete { - sc.DeleteHyperNode(hyperNode) + assert.NoError(t, sc.deleteHyperNode(hyperNode.Name)) } log.Println("begin add...") // add it back. for _, hyperNode := range tt.hypeNodesToDelete { - sc.AddHyperNode(hyperNode) + assert.NoError(t, sc.updateHyperNode(hyperNode)) } assert.Equal(t, tt.expectedHyperNodesSetByTier, sc.HyperNodesInfo.HyperNodesSetByTier()) assert.Equal(t, tt.expectedRealNodesSet, sc.HyperNodesInfo.RealNodesSet()) @@ -946,13 +997,36 @@ func TestSchedulerCache_UpdateHyperNode(t *testing.T) { exactSelector := "exact" regexSelector := "regex" - s0 := schedulingapi.BuildHyperNode("s0", 1, topologyv1alpha1.MemberTypeNode, []string{"node-0", "node-1"}, exactSelector) - s1 := schedulingapi.BuildHyperNode("s1", 1, topologyv1alpha1.MemberTypeNode, []string{"node-2", "node-3"}, exactSelector) - s2 := schedulingapi.BuildHyperNode("s2", 1, topologyv1alpha1.MemberTypeNode, []string{"node-4", "node-5"}, exactSelector) - s3 := schedulingapi.BuildHyperNode("s3", 1, topologyv1alpha1.MemberTypeNode, []string{"node-6", "node-7"}, exactSelector) - s4 := schedulingapi.BuildHyperNode("s4", 2, topologyv1alpha1.MemberTypeHyperNode, []string{"s0", "s1"}, exactSelector) - s5 := schedulingapi.BuildHyperNode("s5", 2, topologyv1alpha1.MemberTypeHyperNode, []string{"s2", "s3"}, exactSelector) - s6 := schedulingapi.BuildHyperNode("s6", 3, topologyv1alpha1.MemberTypeHyperNode, []string{"s4", "s5"}, exactSelector) + s0 := schedulingapi.BuildHyperNode("s0", 1, []schedulingapi.MemberConfig{ + {"node-0", topologyv1alpha1.MemberTypeNode, exactSelector}, + {"node-1", topologyv1alpha1.MemberTypeNode, exactSelector}, + }) + s1 := schedulingapi.BuildHyperNode("s1", 1, []schedulingapi.MemberConfig{ + {"node-2", topologyv1alpha1.MemberTypeNode, exactSelector}, + {"node-3", topologyv1alpha1.MemberTypeNode, exactSelector}, + }) + s2 := schedulingapi.BuildHyperNode("s2", 1, []schedulingapi.MemberConfig{ + {"node-4", topologyv1alpha1.MemberTypeNode, exactSelector}, + {"node-5", topologyv1alpha1.MemberTypeNode, exactSelector}, + }) + s3 := schedulingapi.BuildHyperNode("s3", 1, []schedulingapi.MemberConfig{ + {"node-6", topologyv1alpha1.MemberTypeNode, exactSelector}, + {"node-7", topologyv1alpha1.MemberTypeNode, exactSelector}, + }) + s4 := schedulingapi.BuildHyperNode("s4", 2, []schedulingapi.MemberConfig{ + {"s0", topologyv1alpha1.MemberTypeHyperNode, exactSelector}, + {"s1", topologyv1alpha1.MemberTypeHyperNode, exactSelector}, + }) + + s5 := schedulingapi.BuildHyperNode("s5", 2, []schedulingapi.MemberConfig{ + {"s2", topologyv1alpha1.MemberTypeHyperNode, exactSelector}, + {"s3", topologyv1alpha1.MemberTypeHyperNode, exactSelector}, + }) + s6 := schedulingapi.BuildHyperNode("s6", 3, + []schedulingapi.MemberConfig{ + {"s4", topologyv1alpha1.MemberTypeHyperNode, exactSelector}, + {"s5", topologyv1alpha1.MemberTypeHyperNode, exactSelector}, + }) initialHyperNodes := []*topologyv1alpha1.HyperNode{s0, s1, s2, s3, s4, s5, s6} tests := []struct { @@ -970,9 +1044,15 @@ func TestSchedulerCache_UpdateHyperNode(t *testing.T) { initialHyperNodes: initialHyperNodes, hyperNodesToUpdated: []*topologyv1alpha1.HyperNode{ // first remove s2 from s5. - schedulingapi.BuildHyperNode("s5", 2, topologyv1alpha1.MemberTypeHyperNode, []string{"s3"}, exactSelector), + schedulingapi.BuildHyperNode("s5", 2, []schedulingapi.MemberConfig{ + {"s3", topologyv1alpha1.MemberTypeHyperNode, exactSelector}, + }), // second add s2 to s4. - schedulingapi.BuildHyperNode("s4", 2, topologyv1alpha1.MemberTypeHyperNode, []string{"s0", "s1", "s2"}, exactSelector), + schedulingapi.BuildHyperNode("s4", 2, []schedulingapi.MemberConfig{ + {"s0", topologyv1alpha1.MemberTypeHyperNode, exactSelector}, + {"s1", topologyv1alpha1.MemberTypeHyperNode, exactSelector}, + {"s2", topologyv1alpha1.MemberTypeHyperNode, exactSelector}, + }), }, expectedHyperNodesSetByTier: []map[int]sets.Set[string]{ { @@ -1032,7 +1112,7 @@ func TestSchedulerCache_UpdateHyperNode(t *testing.T) { name: "Remove hyperNode s3's members node-6 and node-7.", initialHyperNodes: initialHyperNodes, hyperNodesToUpdated: []*topologyv1alpha1.HyperNode{ - schedulingapi.BuildHyperNode("s3", 1, topologyv1alpha1.MemberTypeNode, []string{}, exactSelector), + schedulingapi.BuildHyperNode("s3", 1, []schedulingapi.MemberConfig{}), }, expectedHyperNodesSetByTier: []map[int]sets.Set[string]{ { @@ -1073,7 +1153,9 @@ func TestSchedulerCache_UpdateHyperNode(t *testing.T) { }, initialHyperNodes: initialHyperNodes, hyperNodesToUpdated: []*topologyv1alpha1.HyperNode{ - schedulingapi.BuildHyperNode("s2", 1, topologyv1alpha1.MemberTypeNode, []string{"-suffix"}, regexSelector), + schedulingapi.BuildHyperNode("s2", 1, []schedulingapi.MemberConfig{ + {"-suffix", topologyv1alpha1.MemberTypeNode, regexSelector}, + }), }, expectedHyperNodesSetByTier: []map[int]sets.Set[string]{ { @@ -1111,18 +1193,18 @@ func TestSchedulerCache_UpdateHyperNode(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { sc := NewDefaultMockSchedulerCache("volcano") - // Add some nodes to match regex selector. + // Add some initialNodes to match regex selector. for _, node := range tt.nodes { sc.nodeInformer.Informer().GetIndexer().Add(node) } for _, hyperNode := range tt.initialHyperNodes { - sc.AddHyperNode(hyperNode) + assert.NoError(t, sc.updateHyperNode(hyperNode)) } log.Println("begin update...") // compare the result by index as we have updated multi hyperNodes. for i, hyperNode := range tt.hyperNodesToUpdated { - sc.UpdateHyperNode(nil, hyperNode) + assert.NoError(t, sc.updateHyperNode(hyperNode)) assert.Equal(t, tt.expectedHyperNodesSetByTier[i], sc.HyperNodesInfo.HyperNodesSetByTier()) assert.Equal(t, tt.expectedRealNodesSet[i], sc.HyperNodesInfo.RealNodesSet(), "RealNodesSet mismatch, index %d", i) actualHyperNodes := sc.HyperNodesInfo.HyperNodesInfo() @@ -1135,19 +1217,40 @@ func TestSchedulerCache_UpdateHyperNode(t *testing.T) { func TestSchedulerCache_DeleteHyperNode(t *testing.T) { selector := "exact" - s0 := schedulingapi.BuildHyperNode("s0", 1, topologyv1alpha1.MemberTypeNode, []string{"node-0", "node-1"}, selector) - s1 := schedulingapi.BuildHyperNode("s1", 1, topologyv1alpha1.MemberTypeNode, []string{"node-2", "node-3"}, selector) - s2 := schedulingapi.BuildHyperNode("s2", 1, topologyv1alpha1.MemberTypeNode, []string{"node-4", "node-5"}, selector) - s3 := schedulingapi.BuildHyperNode("s3", 1, topologyv1alpha1.MemberTypeNode, []string{"node-6", "node-7"}, selector) - s4 := schedulingapi.BuildHyperNode("s4", 2, topologyv1alpha1.MemberTypeHyperNode, []string{"s0", "s1"}, selector) - s5 := schedulingapi.BuildHyperNode("s5", 2, topologyv1alpha1.MemberTypeHyperNode, []string{"s2", "s3"}, selector) - s6 := schedulingapi.BuildHyperNode("s6", 3, topologyv1alpha1.MemberTypeHyperNode, []string{"s4", "s5"}, selector) + s0 := schedulingapi.BuildHyperNode("s0", 1, []schedulingapi.MemberConfig{ + {"node-0", topologyv1alpha1.MemberTypeNode, selector}, + {"node-1", topologyv1alpha1.MemberTypeNode, selector}, + }) + s1 := schedulingapi.BuildHyperNode("s1", 1, []schedulingapi.MemberConfig{ + {"node-2", topologyv1alpha1.MemberTypeNode, selector}, + {"node-3", topologyv1alpha1.MemberTypeNode, selector}, + }) + s2 := schedulingapi.BuildHyperNode("s2", 1, []schedulingapi.MemberConfig{ + {"node-4", topologyv1alpha1.MemberTypeNode, selector}, + {"node-5", topologyv1alpha1.MemberTypeNode, selector}, + }) + s3 := schedulingapi.BuildHyperNode("s3", 1, []schedulingapi.MemberConfig{ + {"node-6", topologyv1alpha1.MemberTypeNode, selector}, + {"node-7", topologyv1alpha1.MemberTypeNode, selector}, + }) + s4 := schedulingapi.BuildHyperNode("s4", 2, []schedulingapi.MemberConfig{ + {"s0", topologyv1alpha1.MemberTypeHyperNode, selector}, + {"s1", topologyv1alpha1.MemberTypeHyperNode, selector}, + }) + s5 := schedulingapi.BuildHyperNode("s5", 2, []schedulingapi.MemberConfig{ + {"s2", topologyv1alpha1.MemberTypeHyperNode, selector}, + {"s3", topologyv1alpha1.MemberTypeHyperNode, selector}, + }) + s6 := schedulingapi.BuildHyperNode("s6", 3, []schedulingapi.MemberConfig{ + {"s4", topologyv1alpha1.MemberTypeHyperNode, selector}, + {"s5", topologyv1alpha1.MemberTypeHyperNode, selector}, + }) initialHyperNodes := []*topologyv1alpha1.HyperNode{s0, s1, s2, s3, s4, s5, s6} tests := []struct { name string initialHyperNodes []*topologyv1alpha1.HyperNode - typerNodesToDelete *topologyv1alpha1.HyperNode + hyperNodesToDelete *topologyv1alpha1.HyperNode expectedHyperNodesSetByTier map[int]sets.Set[string] expectedRealNodesSet map[string]sets.Set[string] expectedHyperNodesInfo map[string]string @@ -1156,7 +1259,7 @@ func TestSchedulerCache_DeleteHyperNode(t *testing.T) { { name: "Delete non-leaf hyperNode s4", initialHyperNodes: initialHyperNodes, - typerNodesToDelete: s4, + hyperNodesToDelete: s4, expectedHyperNodesSetByTier: map[int]sets.Set[string]{ 1: sets.New[string]("s0", "s1", "s2", "s3"), 2: sets.New[string]("s5"), @@ -1183,7 +1286,7 @@ func TestSchedulerCache_DeleteHyperNode(t *testing.T) { { name: "Delete leaf hyperNode s0", initialHyperNodes: initialHyperNodes, - typerNodesToDelete: s0, + hyperNodesToDelete: s0, expectedHyperNodesSetByTier: map[int]sets.Set[string]{ 1: sets.New[string]("s1", "s2", "s3"), 2: sets.New[string]("s4", "s5"), @@ -1210,7 +1313,7 @@ func TestSchedulerCache_DeleteHyperNode(t *testing.T) { { name: "Delete root hyperNode s6", initialHyperNodes: initialHyperNodes, - typerNodesToDelete: s6, + hyperNodesToDelete: s6, expectedHyperNodesSetByTier: map[int]sets.Set[string]{ 1: sets.New[string]("s0", "s1", "s2", "s3"), 2: sets.New[string]("s4", "s5"), @@ -1239,11 +1342,11 @@ func TestSchedulerCache_DeleteHyperNode(t *testing.T) { t.Run(tt.name, func(t *testing.T) { sc := NewDefaultMockSchedulerCache("volcano") for _, hyperNode := range tt.initialHyperNodes { - sc.AddHyperNode(hyperNode) + assert.NoError(t, sc.updateHyperNode(hyperNode)) } log.Println("begin delete...") - sc.DeleteHyperNode(tt.typerNodesToDelete) + assert.NoError(t, sc.deleteHyperNode(tt.hyperNodesToDelete.Name)) assert.Equal(t, tt.expectedHyperNodesSetByTier, sc.HyperNodesInfo.HyperNodesSetByTier()) assert.Equal(t, tt.expectedRealNodesSet, sc.HyperNodesInfo.RealNodesSet(), "RealNodesSet mismatch") actualHyperNodes := sc.HyperNodesInfo.HyperNodesInfo() @@ -1252,3 +1355,221 @@ func TestSchedulerCache_DeleteHyperNode(t *testing.T) { }) } } + +func TestSchedulerCache_SyncHyperNode(t *testing.T) { + exactSelector := "exact" + regexSelector := "regex" + s6 := schedulingapi.BuildHyperNode("s6", 3, []schedulingapi.MemberConfig{ + {"s4", topologyv1alpha1.MemberTypeHyperNode, exactSelector}, + {"s5", topologyv1alpha1.MemberTypeHyperNode, exactSelector}, + }) + s5 := schedulingapi.BuildHyperNode("s5", 2, []schedulingapi.MemberConfig{ + {"s2", topologyv1alpha1.MemberTypeHyperNode, exactSelector}, + {"s3", topologyv1alpha1.MemberTypeHyperNode, exactSelector}, + }) + s4 := schedulingapi.BuildHyperNode("s4", 2, []schedulingapi.MemberConfig{ + {"s0", topologyv1alpha1.MemberTypeHyperNode, exactSelector}, + {"s1", topologyv1alpha1.MemberTypeHyperNode, exactSelector}, + }) + s3 := schedulingapi.BuildHyperNode("s3", 1, []schedulingapi.MemberConfig{ + {"node-6", topologyv1alpha1.MemberTypeNode, exactSelector}, + {"node-7", topologyv1alpha1.MemberTypeNode, exactSelector}, + }) + s2 := schedulingapi.BuildHyperNode("s2", 1, []schedulingapi.MemberConfig{ + {"node-4", topologyv1alpha1.MemberTypeNode, exactSelector}, + {"node-5", topologyv1alpha1.MemberTypeNode, exactSelector}, + }) + s20 := schedulingapi.BuildHyperNode("s2", 1, []schedulingapi.MemberConfig{ + {"node-4", topologyv1alpha1.MemberTypeNode, exactSelector}, + {"node-5", topologyv1alpha1.MemberTypeNode, exactSelector}, + {"node-9", topologyv1alpha1.MemberTypeNode, regexSelector}, + }) + s21 := schedulingapi.BuildHyperNode("s2", 1, []schedulingapi.MemberConfig{ + {"node-4", topologyv1alpha1.MemberTypeNode, exactSelector}, + {"node-5", topologyv1alpha1.MemberTypeNode, regexSelector}, + }) + s1 := schedulingapi.BuildHyperNode("s1", 1, []schedulingapi.MemberConfig{ + {"node-2", topologyv1alpha1.MemberTypeNode, exactSelector}, + {"node-3", topologyv1alpha1.MemberTypeNode, exactSelector}, + }) + s0 := schedulingapi.BuildHyperNode("s0", 1, []schedulingapi.MemberConfig{ + {"node-0", topologyv1alpha1.MemberTypeNode, exactSelector}, + {"node-1", topologyv1alpha1.MemberTypeNode, exactSelector}, + }) + s00 := schedulingapi.BuildHyperNode("s0", 1, []schedulingapi.MemberConfig{ + {"node-0", topologyv1alpha1.MemberTypeNode, exactSelector}, + {"node-1", topologyv1alpha1.MemberTypeNode, exactSelector}, + {"^prefix", topologyv1alpha1.MemberTypeNode, regexSelector}, + }) + s01 := schedulingapi.BuildHyperNode("s0", 1, []schedulingapi.MemberConfig{ + {"node-[01]", topologyv1alpha1.MemberTypeNode, regexSelector}, + }) + initialHyperNodes0 := []*topologyv1alpha1.HyperNode{s5, s0, s4, s3, s1, s2, s6} + initialHyperNodes1 := []*topologyv1alpha1.HyperNode{s5, s00, s4, s3, s1, s20, s6} + initialHyperNodes2 := []*topologyv1alpha1.HyperNode{s5, s01, s4, s3, s1, s21, s6} + tests := []struct { + name string + initialNodes []*v1.Node + nodeToAdd []*v1.Node + nodeToDelete []*v1.Node + initialHyperNodes []*topologyv1alpha1.HyperNode + expectedHyperNodesListByTier map[int]sets.Set[string] + expectedRealNodesSet map[string]sets.Set[string] + expectedHyperNodesInfo map[string]string + ready bool + }{ + { + name: "leaf HyperNode member with exact selector, no need to update", + initialNodes: []*v1.Node{ + buildNode("node-0", nil), + buildNode("node-1", nil), + buildNode("node-2", nil), + buildNode("node-3", nil), + buildNode("node-4", nil), + buildNode("node-5", nil), + buildNode("node-6", nil), + buildNode("node-7", nil), + }, + initialHyperNodes: initialHyperNodes0, + nodeToAdd: []*v1.Node{buildNode("node-8", nil)}, + expectedHyperNodesListByTier: map[int]sets.Set[string]{ + 1: sets.New[string]("s0", "s1", "s2", "s3"), + 2: sets.New[string]("s4", "s5"), + 3: sets.New[string]("s6"), + }, + expectedRealNodesSet: map[string]sets.Set[string]{ + "s0": sets.New[string]("node-0", "node-1"), + "s1": sets.New[string]("node-2", "node-3"), + "s2": sets.New[string]("node-4", "node-5"), + "s3": sets.New[string]("node-6", "node-7"), + "s4": sets.New[string]("node-0", "node-1", "node-2", "node-3"), + "s5": sets.New[string]("node-4", "node-5", "node-6", "node-7"), + "s6": sets.New[string]("node-0", "node-1", "node-2", "node-3", "node-4", "node-5", "node-6", "node-7"), + }, + expectedHyperNodesInfo: map[string]string{ + "s0": "Name: s0, Tier: 1, Parent: s4", + "s1": "Name: s1, Tier: 1, Parent: s4", + "s2": "Name: s2, Tier: 1, Parent: s5", + "s3": "Name: s3, Tier: 1, Parent: s5", + "s4": "Name: s4, Tier: 2, Parent: s6", + "s5": "Name: s5, Tier: 2, Parent: s6", + "s6": "Name: s6, Tier: 3, Parent: ", + }, + ready: true, + }, + { + name: "update hyperNode when new node added and matched", + initialNodes: []*v1.Node{ + buildNode("node-0", nil), + buildNode("node-1", nil), + buildNode("node-2", nil), + buildNode("node-3", nil), + buildNode("node-4", nil), + buildNode("node-5", nil), + buildNode("node-6", nil), + buildNode("node-7", nil), + }, + initialHyperNodes: initialHyperNodes1, + nodeToAdd: []*v1.Node{ + buildNode("prefix-node-8", nil), + buildNode("node-9", nil), + }, + expectedHyperNodesListByTier: map[int]sets.Set[string]{ + 1: sets.New[string]("s0", "s1", "s2", "s3"), + 2: sets.New[string]("s4", "s5"), + 3: sets.New[string]("s6"), + }, + expectedRealNodesSet: map[string]sets.Set[string]{ + "s0": sets.New[string]("node-0", "node-1", "prefix-node-8"), + "s1": sets.New[string]("node-2", "node-3"), + "s2": sets.New[string]("node-4", "node-5", "node-9"), + "s3": sets.New[string]("node-6", "node-7"), + "s4": sets.New[string]("node-0", "node-1", "node-2", "node-3", "prefix-node-8"), + "s5": sets.New[string]("node-4", "node-5", "node-6", "node-7", "node-9"), + "s6": sets.New[string]("node-0", "node-1", "node-2", "node-3", "node-4", "node-5", "node-6", "node-7", "prefix-node-8", "node-9"), + }, + expectedHyperNodesInfo: map[string]string{ + "s0": "Name: s0, Tier: 1, Parent: s4", + "s1": "Name: s1, Tier: 1, Parent: s4", + "s2": "Name: s2, Tier: 1, Parent: s5", + "s3": "Name: s3, Tier: 1, Parent: s5", + "s4": "Name: s4, Tier: 2, Parent: s6", + "s5": "Name: s5, Tier: 2, Parent: s6", + "s6": "Name: s6, Tier: 3, Parent: ", + }, + ready: true, + }, + { + name: "update hyperNode when new node deleted", + initialNodes: []*v1.Node{ + buildNode("node-0", nil), + buildNode("node-1", nil), + buildNode("node-2", nil), + buildNode("node-3", nil), + buildNode("node-4", nil), + buildNode("node-5", nil), + buildNode("node-6", nil), + buildNode("node-7", nil), + }, + initialHyperNodes: initialHyperNodes2, + nodeToDelete: []*v1.Node{ + buildNode("node-0", nil), + buildNode("node-5", nil), + }, + expectedHyperNodesListByTier: map[int]sets.Set[string]{ + 1: sets.New[string]("s0", "s1", "s2", "s3"), + 2: sets.New[string]("s4", "s5"), + 3: sets.New[string]("s6"), + }, + expectedRealNodesSet: map[string]sets.Set[string]{ + "s0": sets.New[string]("node-1"), + "s1": sets.New[string]("node-2", "node-3"), + "s2": sets.New[string]("node-4"), + "s3": sets.New[string]("node-6", "node-7"), + "s4": sets.New[string]("node-1", "node-2", "node-3"), + "s5": sets.New[string]("node-4", "node-6", "node-7"), + "s6": sets.New[string]("node-1", "node-2", "node-3", "node-4", "node-6", "node-7"), + }, + expectedHyperNodesInfo: map[string]string{ + "s0": "Name: s0, Tier: 1, Parent: s4", + "s1": "Name: s1, Tier: 1, Parent: s4", + "s2": "Name: s2, Tier: 1, Parent: s5", + "s3": "Name: s3, Tier: 1, Parent: s5", + "s4": "Name: s4, Tier: 2, Parent: s6", + "s5": "Name: s5, Tier: 2, Parent: s6", + "s6": "Name: s6, Tier: 3, Parent: ", + }, + ready: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + sc := NewDefaultMockSchedulerCache("volcano") + // Add some initialNodes to match regex selector. + for _, node := range tt.initialNodes { + assert.NoError(t, sc.nodeInformer.Informer().GetIndexer().Add(node)) + } + for _, hyperNode := range tt.initialHyperNodes { + assert.NoError(t, sc.updateHyperNode(hyperNode)) + } + + for _, node := range tt.nodeToAdd { + assert.NoError(t, sc.nodeInformer.Informer().GetIndexer().Add(node)) + name := "node/" + node.Name + assert.NoError(t, sc.SyncHyperNode(name)) + } + for _, node := range tt.nodeToDelete { + assert.NoError(t, sc.nodeInformer.Informer().GetIndexer().Delete(node)) + name := "node/" + node.Name + assert.NoError(t, sc.SyncHyperNode(name)) + } + + assert.Equal(t, tt.expectedHyperNodesListByTier, sc.HyperNodesInfo.HyperNodesSetByTier()) + assert.Equal(t, tt.expectedRealNodesSet, sc.HyperNodesInfo.RealNodesSet()) + actualHyperNodes := sc.HyperNodesInfo.HyperNodesInfo() + assert.Equal(t, tt.expectedHyperNodesInfo, actualHyperNodes) + assert.Equal(t, tt.ready, sc.HyperNodesInfo.Ready()) + }) + } +} diff --git a/pkg/scheduler/cache/util.go b/pkg/scheduler/cache/util.go index 48415bee4a..f6aae15147 100644 --- a/pkg/scheduler/cache/util.go +++ b/pkg/scheduler/cache/util.go @@ -30,6 +30,13 @@ import ( scheduling "volcano.sh/apis/pkg/apis/scheduling/v1beta1" ) +type hyperNodeEventSource string + +const ( + hyperNodeEventSourceNode hyperNodeEventSource = "node" + hyperNodeEventSourceHyperNode hyperNodeEventSource = "hyperNode" +) + // responsibleForPod returns false at following conditions: // 1. The current scheduler is not specified scheduler in Pod's spec. // 2. The Job which the Pod belongs is not assigned to current scheduler based on the hash algorithm in multi-schedulers scenario @@ -117,3 +124,11 @@ func getMultiSchedulerInfo() (schedulerPodName string, c *consistent.Consistent) } return mySchedulerPodName, c } + +func getHyperNodeEventSource(source string) []string { + parts := strings.Split(source, "/") + if len(parts) != 2 { + return nil + } + return parts +}