From 3df5ce89d14b3649aad3cfa6f83524e4ede0db07 Mon Sep 17 00:00:00 2001
From: Monokaix <changxuzheng@huawei.com>
Date: Sat, 11 Jan 2025 21:44:54 +0800
Subject: [PATCH] add node event
Signed-off-by: Monokaix <changxuzheng@huawei.com>
---
pkg/scheduler/api/hyper_node_info.go | 131 ++++++-
pkg/scheduler/api/hyper_node_info_test.go | 312 ++++++++++++++-
pkg/scheduler/api/test_utils.go | 30 +-
pkg/scheduler/api/test_utils_test.go | 22 +-
pkg/scheduler/cache/cache.go | 40 +-
pkg/scheduler/cache/cache_mock.go | 1 +
pkg/scheduler/cache/dumper.go | 2 +-
pkg/scheduler/cache/event_handlers.go | 119 ++++--
pkg/scheduler/cache/event_handlers_test.go | 421 ++++++++++++++++++---
pkg/scheduler/cache/util.go | 15 +
10 files changed, 962 insertions(+), 131 deletions(-)
diff --git a/pkg/scheduler/api/hyper_node_info.go b/pkg/scheduler/api/hyper_node_info.go
index 863d0ed216..3eead32c04 100644
--- a/pkg/scheduler/api/hyper_node_info.go
+++ b/pkg/scheduler/api/hyper_node_info.go
@@ -45,7 +45,8 @@ type HyperNodesInfo struct {
// s0->node0, node1, s1->node2, node3, s4->node0, node1, s1->node2, node3
realNodesSet map[string]sets.Set[string]
// nodeLister Lister to list Kubernetes nodes.
- nodeLister listerv1.NodeLister
+ nodeLister listerv1.NodeLister
+ builtErrHyperNode string
// ready indicates whether the HyperNodesInfo is ready (build process is complete).
ready *atomic.Bool
@@ -107,6 +108,15 @@ func (hni *HyperNodesInfo) HyperNodes() map[string]*HyperNodeInfo {
return hni.hyperNodes
}
+// HyperNode returns a hyperNode by name.
+func (hni *HyperNodesInfo) HyperNode(name string) *topologyv1alpha1.HyperNode {
+ hn := hni.hyperNodes[name]
+ if hn == nil {
+ return nil
+ }
+ return hn.HyperNode
+}
+
// HyperNodesSetByTier returns a deep copy of the map that groups HyperNode names by their tier.
// This ensures that the returned map is independent of the original, preventing unintended modifications.
func (hni *HyperNodesInfo) HyperNodesSetByTier() map[int]sets.Set[string] {
@@ -324,6 +334,67 @@ func (hni *HyperNodesInfo) getChildren(hyperNodeName string) sets.Set[string] {
return children
}
+// GetLeafNodes returns the leaf hyperNodes set for a given hyperNode.
+func (hni *HyperNodesInfo) GetLeafNodes(hyperNodeName string) sets.Set[string] {
+ leafNodes := sets.New[string]()
+ ancestorsChain := sets.New[string]()
+ hni.findLeafNodesWithCycleCheck(hyperNodeName, leafNodes, ancestorsChain)
+ return leafNodes
+}
+
+func (hni *HyperNodesInfo) findLeafNodesWithCycleCheck(hyperNodeName string, leafNodes sets.Set[string], ancestorsChain sets.Set[string]) {
+ if ancestorsChain.Has(hyperNodeName) {
+ klog.ErrorS(nil, "Cycle detected in HyperNode hierarchy", "hyperNode", hyperNodeName)
+ return
+ }
+
+ hn, ok := hni.hyperNodes[hyperNodeName]
+ if !ok {
+ return
+ }
+
+ ancestorsChain.Insert(hyperNodeName)
+ defer ancestorsChain.Delete(hyperNodeName)
+
+ isLeaf := true
+ for _, member := range hn.HyperNode.Spec.Members {
+ if member.Type == topologyv1alpha1.MemberTypeHyperNode {
+ isLeaf = false
+ hni.findLeafNodesWithCycleCheck(member.Selector.ExactMatch.Name, leafNodes, ancestorsChain)
+ }
+ }
+
+ if isLeaf {
+ leafNodes.Insert(hyperNodeName)
+ }
+}
+
+// GetRegexSelectorLeafHyperNodes returns leaf hyperNodes whose member's selector is regex match.
+func (hni *HyperNodesInfo) GetRegexSelectorLeafHyperNodes() sets.Set[string] {
+ leaf := sets.New[string]()
+ for name, hnInfo := range hni.hyperNodes {
+ if hnInfo == nil || hnInfo.HyperNode == nil {
+ continue
+ }
+
+ isLeaf := true
+ hasRegexMatch := false
+ for _, member := range hnInfo.HyperNode.Spec.Members {
+ if member.Type == topologyv1alpha1.MemberTypeHyperNode {
+ isLeaf = false
+ break
+ }
+ if member.Selector.RegexMatch != nil {
+ hasRegexMatch = true
+ }
+ }
+ if isLeaf && hasRegexMatch {
+ leaf.Insert(name)
+ }
+ }
+ return leaf
+}
+
// setParent sets the parent of a HyperNode member.
func (hni *HyperNodesInfo) setParent(member, parent string) error {
hn, ok := hni.hyperNodes[member]
@@ -343,6 +414,7 @@ func (hni *HyperNodesInfo) setParent(member, parent string) error {
}
if currentParent != parent {
+ hni.builtErrHyperNode = parent
return fmt.Errorf("HyperNode %s already has a parent %s, and cannot set another parent %s", member, currentParent, parent)
}
@@ -410,8 +482,29 @@ func (hni *HyperNodesInfo) exactMatchMember(selector topologyv1alpha1.MemberSele
}
func (hni *HyperNodesInfo) updateAncestors(name string) error {
+ if err := hni.rebuildCache(name); err != nil {
+ hni.setReady(false)
+ return err
+ }
+ // When last time BuildHyperNodeCache has an err that a hyperNode has multi parents, after the parent is updated
+ // we should find the parent hyperNode to rebuild the correct cache.
+ if hni.builtErrHyperNode == name {
+ klog.InfoS("Rebuilt parent hyperNode", "name", hni.builtErrHyperNode)
+ leafHyperNods := hni.GetLeafNodes(name)
+ for leaf := range leafHyperNods {
+ if err := hni.rebuildCache(leaf); err != nil {
+ return err
+ }
+ }
+
+ }
+ hni.builtErrHyperNode = ""
+ hni.setReady(true)
+ return nil
+}
+
+func (hni *HyperNodesInfo) rebuildCache(name string) error {
ancestors := hni.GetAncestors(name)
- fmt.Println("acxx", ancestors)
// Clear realNodesSet of hyperNode before rebuild hyperNode cache.
for ancestor := range ancestors {
delete(hni.realNodesSet, ancestor)
@@ -429,12 +522,11 @@ func (hni *HyperNodesInfo) updateAncestors(name string) error {
for ancestor := range ancestors {
if hn, ok := hni.hyperNodes[ancestor]; ok {
if err := hni.BuildHyperNodeCache(hn, processed, ancestorsChain, ancestors, nodes); err != nil {
- hni.setReady(false)
return err
}
}
}
- hni.setReady(true)
+
return nil
}
@@ -517,3 +609,34 @@ func (hni *HyperNodesInfo) HyperNodesInfo() map[string]string {
}
return actualHyperNodes
}
+
+// NodeRegexMatchLeafHyperNode checks if a given node regex matches the MemberSelector of a HyperNode.
+func (hni *HyperNodesInfo) NodeRegexMatchLeafHyperNode(nodeName string, hyperNodeName string) (bool, error) {
+ hn, ok := hni.hyperNodes[hyperNodeName]
+ if !ok {
+ return false, fmt.Errorf("HyperNode %s not found in cache", hyperNodeName)
+ }
+
+ for _, member := range hn.HyperNode.Spec.Members {
+ if member.Type == topologyv1alpha1.MemberTypeNode {
+ if hni.nodeMatchRegexSelector(nodeName, member.Selector) {
+ return true, nil
+ }
+ }
+ }
+ return false, nil
+}
+
+// nodeMatchRegexSelector checks if a node matches a MemberSelector.
+func (hni *HyperNodesInfo) nodeMatchRegexSelector(nodeName string, selector topologyv1alpha1.MemberSelector) bool {
+ if selector.RegexMatch == nil {
+ return false
+ }
+
+ reg, err := regexp.Compile(selector.RegexMatch.Pattern)
+ if err != nil {
+ klog.ErrorS(err, "Failed to compile regex pattern", "pattern", selector.RegexMatch.Pattern)
+ return false
+ }
+ return reg.MatchString(nodeName)
+}
diff --git a/pkg/scheduler/api/hyper_node_info_test.go b/pkg/scheduler/api/hyper_node_info_test.go
index 802cc886a1..e4e4f17eed 100644
--- a/pkg/scheduler/api/hyper_node_info_test.go
+++ b/pkg/scheduler/api/hyper_node_info_test.go
@@ -17,7 +17,6 @@ limitations under the License.
package api
import (
- "fmt"
"log"
"testing"
@@ -31,9 +30,18 @@ import (
func TestHyperNodesInfo_UpdateHyperNode_Normal(t *testing.T) {
selector := "exact"
- s0 := BuildHyperNode("s0", 1, topologyv1alpha1.MemberTypeNode, []string{"node-0", "node-1"}, selector)
- s1 := BuildHyperNode("s1", 1, topologyv1alpha1.MemberTypeNode, []string{"node-2", "node-3"}, selector)
- s2 := BuildHyperNode("s2", 2, topologyv1alpha1.MemberTypeHyperNode, []string{"s1", "s0"}, selector)
+ s0 := BuildHyperNode("s0", 1, []MemberConfig{
+ {"node-0", topologyv1alpha1.MemberTypeNode, selector},
+ {"node-1", topologyv1alpha1.MemberTypeNode, selector},
+ })
+ s1 := BuildHyperNode("s1", 1, []MemberConfig{
+ {"node-2", topologyv1alpha1.MemberTypeNode, selector},
+ {"node-3", topologyv1alpha1.MemberTypeNode, selector},
+ })
+ s2 := BuildHyperNode("s2", 2, []MemberConfig{
+ {"s1", topologyv1alpha1.MemberTypeHyperNode, selector},
+ {"s0", topologyv1alpha1.MemberTypeHyperNode, selector},
+ })
initialHyperNodes := []*topologyv1alpha1.HyperNode{s2, s0, s1}
tests := []struct {
@@ -87,11 +95,21 @@ func TestHyperNodesInfo_UpdateHyperNode_Normal(t *testing.T) {
func TestHyperNodesInfo_UpdateHyperNode_WithCycle(t *testing.T) {
selector := "exact"
- s0 := BuildHyperNode("s0", 1, topologyv1alpha1.MemberTypeNode, []string{"node-0", "node-1"}, selector)
- s1 := BuildHyperNode("s1", 2, topologyv1alpha1.MemberTypeHyperNode, []string{"s0", "s3"}, selector)
- s11 := BuildHyperNode("s1", 2, topologyv1alpha1.MemberTypeHyperNode, []string{"s0"}, selector)
- s2 := BuildHyperNode("s2", 3, topologyv1alpha1.MemberTypeHyperNode, []string{"s1"}, selector)
- s3 := BuildHyperNode("s3", 4, topologyv1alpha1.MemberTypeHyperNode, []string{"s2"}, selector)
+ s0 := BuildHyperNode("s0", 1, []MemberConfig{
+ {"node-0", topologyv1alpha1.MemberTypeNode, selector},
+ {"node-1", topologyv1alpha1.MemberTypeNode, selector}})
+ s1 := BuildHyperNode("s1", 2, []MemberConfig{
+ {"s0", topologyv1alpha1.MemberTypeHyperNode, selector},
+ {"s3", topologyv1alpha1.MemberTypeHyperNode, selector}})
+ s11 := BuildHyperNode("s1", 2, []MemberConfig{
+ {"s0", topologyv1alpha1.MemberTypeHyperNode, selector},
+ })
+ s2 := BuildHyperNode("s2", 3, []MemberConfig{
+ {"s1", topologyv1alpha1.MemberTypeHyperNode, selector},
+ })
+ s3 := BuildHyperNode("s3", 4, []MemberConfig{
+ {"s2", topologyv1alpha1.MemberTypeHyperNode, selector},
+ })
initialHyperNodes := []*topologyv1alpha1.HyperNode{s2, s0, s1, s3}
tests := []struct {
@@ -139,10 +157,10 @@ func TestHyperNodesInfo_UpdateHyperNode_WithCycle(t *testing.T) {
}
assert.Equal(t, false, hni.Ready())
- fmt.Println("begin updatexxx")
+
+ log.Println("begin update...")
// update and resolve dependency.
err := hni.UpdateHyperNode(s11)
-
assert.NoError(t, err)
actualHyperNodes := hni.HyperNodesInfo()
assert.Equal(t, tt.expectedHyperNodesInfo, actualHyperNodes)
@@ -161,10 +179,19 @@ func TestHyperNodesInfo_UpdateHyperNode_WithCycle(t *testing.T) {
func TestHyperNodesInfo_UpdateHyperNode_MultipleParents(t *testing.T) {
selector := "exact"
- s0 := BuildHyperNode("s0", 1, topologyv1alpha1.MemberTypeNode, []string{"node-0", "node-1"}, selector)
- s1 := BuildHyperNode("s1", 2, topologyv1alpha1.MemberTypeHyperNode, []string{"s0"}, selector)
- s2 := BuildHyperNode("s2", 3, topologyv1alpha1.MemberTypeHyperNode, []string{"s0"}, selector)
- s22 := BuildHyperNode("s2", 3, topologyv1alpha1.MemberTypeHyperNode, []string{"s1"}, selector)
+ s0 := BuildHyperNode("s0", 1, []MemberConfig{
+ {Name: "node-0", Type: topologyv1alpha1.MemberTypeNode, Selector: selector},
+ {Name: "node-1", Type: topologyv1alpha1.MemberTypeNode, Selector: selector},
+ })
+ s1 := BuildHyperNode("s1", 2, []MemberConfig{
+ {Name: "s0", Type: topologyv1alpha1.MemberTypeHyperNode, Selector: selector},
+ })
+ s2 := BuildHyperNode("s2", 3, []MemberConfig{
+ {Name: "s0", Type: topologyv1alpha1.MemberTypeHyperNode, Selector: selector},
+ })
+ s22 := BuildHyperNode("s2", 3, []MemberConfig{
+ {Name: "s1", Type: topologyv1alpha1.MemberTypeHyperNode, Selector: selector},
+ })
initialHyperNodes := []*topologyv1alpha1.HyperNode{s0, s1, s2}
tests := []struct {
@@ -228,3 +255,258 @@ func TestHyperNodesInfo_UpdateHyperNode_MultipleParents(t *testing.T) {
})
}
}
+
+func TestHyperNodesInfo_GetRegexSelectorLeafHyperNodes(t *testing.T) {
+ exactSelector := "exact"
+ regexSelector := "regex"
+ s0 := BuildHyperNode("s0", 1, []MemberConfig{
+ {"node-1", topologyv1alpha1.MemberTypeNode, exactSelector},
+ {"node-1", topologyv1alpha1.MemberTypeNode, exactSelector},
+ })
+ s1 := BuildHyperNode("s1", 1, []MemberConfig{
+ {"node-2", topologyv1alpha1.MemberTypeNode, regexSelector},
+ {"node-3", topologyv1alpha1.MemberTypeNode, regexSelector},
+ })
+ s2 := BuildHyperNode("s2", 1, []MemberConfig{
+ {"node-4", topologyv1alpha1.MemberTypeNode, regexSelector},
+ {"node-5", topologyv1alpha1.MemberTypeHyperNode, exactSelector},
+ })
+ s3 := BuildHyperNode("s3", 1, []MemberConfig{
+ {"node-6", topologyv1alpha1.MemberTypeNode, regexSelector},
+ {"node-7", topologyv1alpha1.MemberTypeNode, exactSelector},
+ })
+ s4 := BuildHyperNode("s4", 2, []MemberConfig{
+ {"s0", topologyv1alpha1.MemberTypeHyperNode, exactSelector},
+ {"s1", topologyv1alpha1.MemberTypeHyperNode, exactSelector},
+ })
+ s5 := BuildHyperNode("s5", 2, []MemberConfig{
+ {"s2", topologyv1alpha1.MemberTypeHyperNode, regexSelector},
+ {"s3", topologyv1alpha1.MemberTypeHyperNode, exactSelector},
+ })
+ s6 := BuildHyperNode("s6", 3, []MemberConfig{
+ {"s4", topologyv1alpha1.MemberTypeHyperNode, exactSelector},
+ {"s5", topologyv1alpha1.MemberTypeHyperNode, exactSelector},
+ })
+ tests := []struct {
+ name string
+ hyperNods []*topologyv1alpha1.HyperNode
+ want sets.Set[string]
+ }{
+ {
+ name: "get all leaf hyperNodes correctly",
+ hyperNods: []*topologyv1alpha1.HyperNode{s0, s1, s2, s3, s4, s5, s6},
+ want: sets.New[string]("s1", "s3"),
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ hni := NewHyperNodesInfo(nil)
+ for _, hn := range tt.hyperNods {
+ hni.hyperNodes[hn.Name] = NewHyperNodeInfo(hn)
+ }
+ assert.Equalf(t, tt.want, hni.GetRegexSelectorLeafHyperNodes(), "GetRegexSelcectorLeafHyperNodes()")
+ })
+ }
+}
+
+func TestHyperNodesInfo_GetLeafNodes(t *testing.T) {
+ selector := "exact"
+ s0 := BuildHyperNode("s0", 1, []MemberConfig{
+ {"node-0", topologyv1alpha1.MemberTypeNode, selector},
+ {"node-1", topologyv1alpha1.MemberTypeNode, selector},
+ })
+ s1 := BuildHyperNode("s1", 1, []MemberConfig{
+ {"node-2", topologyv1alpha1.MemberTypeNode, selector},
+ {"node-3", topologyv1alpha1.MemberTypeNode, selector},
+ })
+ s2 := BuildHyperNode("s2", 2, []MemberConfig{
+ {"s1", topologyv1alpha1.MemberTypeHyperNode, selector},
+ {"s0", topologyv1alpha1.MemberTypeHyperNode, selector},
+ })
+ s3 := BuildHyperNode("s3", 3, []MemberConfig{
+ {"s2", topologyv1alpha1.MemberTypeHyperNode, selector},
+ })
+ initialHyperNodes := []*topologyv1alpha1.HyperNode{s2, s3, s0, s1}
+
+ tests := []struct {
+ name string
+ hyperNodeName string
+ initialHyperNodes []*topologyv1alpha1.HyperNode
+ expectedLeafNodes sets.Set[string]
+ }{
+ {
+ name: "Get correct leaf hyperNodes",
+ hyperNodeName: "s3",
+ initialHyperNodes: initialHyperNodes,
+ expectedLeafNodes: sets.New[string]("s0", "s1"),
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ informerFactory := informers.NewSharedInformerFactory(fakeclientset.NewClientset(), 0)
+ nodeLister := informerFactory.Core().V1().Nodes().Lister()
+ hni := NewHyperNodesInfo(nodeLister)
+
+ for _, node := range tt.initialHyperNodes {
+ err := hni.UpdateHyperNode(node)
+ assert.NoError(t, err)
+ }
+
+ assert.Equal(t, tt.expectedLeafNodes, hni.GetLeafNodes(tt.hyperNodeName))
+ })
+ }
+}
+
+func TestHyperNodesInfo_NodeRegexMatchHyperNode(t *testing.T) {
+ exactSelector := "exact"
+ regexSelector := "regex"
+ s0 := BuildHyperNode("s0", 1, []MemberConfig{
+ {"node-0", topologyv1alpha1.MemberTypeNode, exactSelector},
+ {"node-1", topologyv1alpha1.MemberTypeNode, exactSelector},
+ })
+ s1 := BuildHyperNode("s1", 1, []MemberConfig{
+ {"^prefix", topologyv1alpha1.MemberTypeNode, regexSelector},
+ {"node-2", topologyv1alpha1.MemberTypeNode, regexSelector},
+ })
+ s2 := BuildHyperNode("s2", 1, []MemberConfig{
+ {"^not-match-prefix", topologyv1alpha1.MemberTypeHyperNode, regexSelector},
+ {"node-3", topologyv1alpha1.MemberTypeNode, regexSelector},
+ })
+ s3 := BuildHyperNode("s3", 1, []MemberConfig{
+ {"node-4", topologyv1alpha1.MemberTypeHyperNode, regexSelector},
+ })
+ initialHyperNodes := []*topologyv1alpha1.HyperNode{s2, s3, s0, s1}
+
+ tests := []struct {
+ name string
+ nodeName string
+ hyperNodeName string
+ expectedMatch bool
+ expectedErr bool
+ }{
+ {
+ name: "not match",
+ nodeName: "node-0",
+ hyperNodeName: "s0",
+ expectedMatch: false,
+ expectedErr: false,
+ },
+ {
+ name: "not match",
+ nodeName: "node-1",
+ hyperNodeName: "s0",
+ expectedMatch: false,
+ expectedErr: false,
+ },
+ {
+ name: "match",
+ nodeName: "prefix-1",
+ hyperNodeName: "s1",
+ expectedMatch: true,
+ expectedErr: false,
+ },
+ {
+ name: "match",
+ nodeName: "node-2",
+ hyperNodeName: "s1",
+ expectedMatch: true,
+ expectedErr: false,
+ },
+ {
+ name: "not match",
+ nodeName: "not-match-prefix",
+ hyperNodeName: "s2",
+ expectedMatch: false,
+ expectedErr: false,
+ },
+ {
+ name: "not match",
+ nodeName: "node-4",
+ hyperNodeName: "s3",
+ expectedMatch: false,
+ expectedErr: false,
+ },
+ }
+
+ informerFactory := informers.NewSharedInformerFactory(fakeclientset.NewClientset(), 0)
+ nodeLister := informerFactory.Core().V1().Nodes().Lister()
+ hni := NewHyperNodesInfo(nodeLister)
+ for _, node := range initialHyperNodes {
+ err := hni.UpdateHyperNode(node)
+ assert.NoError(t, err)
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ match, err := hni.NodeRegexMatchLeafHyperNode(tt.nodeName, tt.hyperNodeName)
+ assert.Equal(t, tt.expectedMatch, match)
+ assert.Equal(t, tt.expectedErr, err != nil)
+ })
+ }
+}
+
+func TestHyperNodesInfo_GetAncestors(t *testing.T) {
+ selector := "exact"
+ s0 := BuildHyperNode("s0", 1, []MemberConfig{
+ {"node-0", topologyv1alpha1.MemberTypeNode, selector},
+ {"node-1", topologyv1alpha1.MemberTypeNode, selector},
+ })
+ s1 := BuildHyperNode("s1", 1, []MemberConfig{
+ {"node-0", topologyv1alpha1.MemberTypeNode, selector},
+ {"node-1", topologyv1alpha1.MemberTypeNode, selector},
+ })
+ s2 := BuildHyperNode("s2", 2, []MemberConfig{
+ {"s0", topologyv1alpha1.MemberTypeHyperNode, selector},
+ {"s1", topologyv1alpha1.MemberTypeHyperNode, selector},
+ })
+ s3 := BuildHyperNode("s3", 3, []MemberConfig{
+ {"s2", topologyv1alpha1.MemberTypeHyperNode, selector},
+ })
+ s4 := BuildHyperNode("s4", 4, []MemberConfig{
+ {"s3", topologyv1alpha1.MemberTypeHyperNode, selector},
+ })
+ initialHyperNodes := []*topologyv1alpha1.HyperNode{s0, s4, s1, s2, s3}
+
+ tests := []struct {
+ name string
+ hyperNodeName string
+ initialHyperNodes []*topologyv1alpha1.HyperNode
+ expectedAncestors sets.Set[string]
+ }{
+ {
+ name: "Get ancestors for leaf node",
+ hyperNodeName: "s0",
+ initialHyperNodes: initialHyperNodes,
+ expectedAncestors: sets.New[string]("s0", "s2", "s3", "s4"),
+ },
+ {
+ name: "Get ancestors for intermediate node",
+ hyperNodeName: "s2",
+ initialHyperNodes: initialHyperNodes,
+ expectedAncestors: sets.New[string]("s2", "s3", "s4"),
+ },
+ {
+ name: "Get ancestors for root node",
+ hyperNodeName: "s4",
+ initialHyperNodes: initialHyperNodes,
+ expectedAncestors: sets.New[string]("s4"),
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ informerFactory := informers.NewSharedInformerFactory(fakeclientset.NewClientset(), 0)
+ nodeLister := informerFactory.Core().V1().Nodes().Lister()
+ hni := NewHyperNodesInfo(nodeLister)
+
+ for _, node := range tt.initialHyperNodes {
+ err := hni.UpdateHyperNode(node)
+ assert.NoError(t, err)
+ }
+
+ actualAncestors := hni.GetAncestors(tt.hyperNodeName)
+ assert.Equal(t, tt.expectedAncestors, actualAncestors)
+ })
+ }
+}
diff --git a/pkg/scheduler/api/test_utils.go b/pkg/scheduler/api/test_utils.go
index 9e997c62c9..a017ad96e9 100644
--- a/pkg/scheduler/api/test_utils.go
+++ b/pkg/scheduler/api/test_utils.go
@@ -134,8 +134,13 @@ func BuildPodgroup(name, ns string, minMember int32, minResource v1.ResourceList
}
}
-// BuildHyperNode builds a hyperNode.
-func BuildHyperNode(name string, tier int, memberType topologyv1alpha1.MemberType, members []string, selector string) *topologyv1alpha1.HyperNode {
+type MemberConfig struct {
+ Name string
+ Type topologyv1alpha1.MemberType
+ Selector string
+}
+
+func BuildHyperNode(name string, tier int, members []MemberConfig) *topologyv1alpha1.HyperNode {
hn := &topologyv1alpha1.HyperNode{
ObjectMeta: metav1.ObjectMeta{
Name: name,
@@ -147,17 +152,20 @@ func BuildHyperNode(name string, tier int, memberType topologyv1alpha1.MemberTyp
}
for i, member := range members {
- hn.Spec.Members[i] = topologyv1alpha1.MemberSpec{
- Type: memberType,
- }
- if selector == "exact" {
- hn.Spec.Members[i].Selector.ExactMatch = &topologyv1alpha1.ExactMatch{Name: member}
- continue
+ memberSpec := topologyv1alpha1.MemberSpec{
+ Type: member.Type,
}
- if selector == "regex" {
- hn.Spec.Members[i].Selector.RegexMatch = &topologyv1alpha1.RegexMatch{Pattern: member}
- continue
+ switch member.Selector {
+ case "exact":
+ memberSpec.Selector.ExactMatch = &topologyv1alpha1.ExactMatch{Name: member.Name}
+ case "regex":
+ memberSpec.Selector.RegexMatch = &topologyv1alpha1.RegexMatch{Pattern: member.Name}
+ default:
+ return nil
}
+
+ hn.Spec.Members[i] = memberSpec
}
+
return hn
}
diff --git a/pkg/scheduler/api/test_utils_test.go b/pkg/scheduler/api/test_utils_test.go
index e7a47b4aa8..7dc15e7656 100644
--- a/pkg/scheduler/api/test_utils_test.go
+++ b/pkg/scheduler/api/test_utils_test.go
@@ -30,18 +30,17 @@ func TestBuildHyperNode(t *testing.T) {
name string
hyperNodeName string
tier int
- memberType topologyv1alpha1.MemberType
- members []string
- selector string
+ members []MemberConfig
want *topologyv1alpha1.HyperNode
}{
{
name: "build leaf hyperNode",
hyperNodeName: "s0",
tier: 1,
- memberType: topologyv1alpha1.MemberTypeNode,
- members: []string{"node-1", "node-2"},
- selector: "exact",
+ members: []MemberConfig{
+ {"node-1", topologyv1alpha1.MemberTypeNode, "regex"},
+ {"node-2", topologyv1alpha1.MemberTypeNode, "exact"},
+ },
want: &topologyv1alpha1.HyperNode{
ObjectMeta: metav1.ObjectMeta{
Name: "s0",
@@ -49,7 +48,7 @@ func TestBuildHyperNode(t *testing.T) {
Spec: topologyv1alpha1.HyperNodeSpec{
Tier: 1,
Members: []topologyv1alpha1.MemberSpec{
- {Type: topologyv1alpha1.MemberTypeNode, Selector: topologyv1alpha1.MemberSelector{ExactMatch: &topologyv1alpha1.ExactMatch{Name: "node-1"}}},
+ {Type: topologyv1alpha1.MemberTypeNode, Selector: topologyv1alpha1.MemberSelector{RegexMatch: &topologyv1alpha1.RegexMatch{Pattern: "node-1"}}},
{Type: topologyv1alpha1.MemberTypeNode, Selector: topologyv1alpha1.MemberSelector{ExactMatch: &topologyv1alpha1.ExactMatch{Name: "node-2"}}},
},
},
@@ -59,9 +58,10 @@ func TestBuildHyperNode(t *testing.T) {
name: "build non-leaf hyperNode",
hyperNodeName: "s4",
tier: 2,
- selector: "exact",
- memberType: topologyv1alpha1.MemberTypeHyperNode,
- members: []string{"s0", "s1"},
+ members: []MemberConfig{
+ {"s0", topologyv1alpha1.MemberTypeHyperNode, "exact"},
+ {"s1", topologyv1alpha1.MemberTypeHyperNode, "exact"},
+ },
want: &topologyv1alpha1.HyperNode{
ObjectMeta: metav1.ObjectMeta{
Name: "s4",
@@ -78,7 +78,7 @@ func TestBuildHyperNode(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
- assert.Equalf(t, tt.want, BuildHyperNode(tt.hyperNodeName, tt.tier, tt.memberType, tt.members, tt.selector), "BuildHyperNode(%v, %v, %v, %v)", tt.hyperNodeName, tt.tier, tt.memberType, tt.members)
+ assert.Equalf(t, tt.want, BuildHyperNode(tt.hyperNodeName, tt.tier, tt.members), "BuildHyperNode(%v, %v, %v)", tt.hyperNodeName, tt.tier, tt.members)
})
}
}
diff --git a/pkg/scheduler/cache/cache.go b/pkg/scheduler/cache/cache.go
index 27f7b5b2d0..382eebda0a 100644
--- a/pkg/scheduler/cache/cache.go
+++ b/pkg/scheduler/cache/cache.go
@@ -62,6 +62,7 @@ import (
vcinformer "volcano.sh/apis/pkg/client/informers/externalversions"
cpuinformerv1 "volcano.sh/apis/pkg/client/informers/externalversions/nodeinfo/v1alpha1"
vcinformerv1 "volcano.sh/apis/pkg/client/informers/externalversions/scheduling/v1beta1"
+ topologyinformerv1alpha1 "volcano.sh/apis/pkg/client/informers/externalversions/topology/v1alpha1"
"volcano.sh/volcano/cmd/scheduler/app/options"
"volcano.sh/volcano/pkg/features"
schedulingapi "volcano.sh/volcano/pkg/scheduler/api"
@@ -107,6 +108,7 @@ type SchedulerCache struct {
podInformer infov1.PodInformer
nodeInformer infov1.NodeInformer
+ hyperNodeInformer topologyinformerv1alpha1.HyperNodeInformer
podGroupInformerV1beta1 vcinformerv1.PodGroupInformer
queueInformerV1beta1 vcinformerv1.QueueInformer
pvInformer infov1.PersistentVolumeInformer
@@ -139,9 +141,10 @@ type SchedulerCache struct {
NamespaceCollection map[string]*schedulingapi.NamespaceCollection
- errTasks workqueue.RateLimitingInterface
- nodeQueue workqueue.RateLimitingInterface
- DeletedJobs workqueue.RateLimitingInterface
+ errTasks workqueue.RateLimitingInterface
+ nodeQueue workqueue.RateLimitingInterface
+ DeletedJobs workqueue.RateLimitingInterface
+ hyperNodesQueue workqueue.TypedRateLimitingInterface[string]
informerFactory informers.SharedInformerFactory
vcInformerFactory vcinformer.SharedInformerFactory
@@ -572,6 +575,7 @@ func newSchedulerCache(config *rest.Config, schedulerNames []string, defaultQueu
errTasks: workqueue.NewRateLimitingQueue(errTaskRateLimiter),
nodeQueue: workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter()),
DeletedJobs: workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter()),
+ hyperNodesQueue: workqueue.NewTypedRateLimitingQueue[string](workqueue.DefaultTypedControllerRateLimiter[string]()),
kubeClient: kubeClient,
vcClient: vcClient,
restConfig: config,
@@ -798,8 +802,8 @@ func (sc *SchedulerCache) addEventHandler() {
})
}
- hyperNodeInformer := sc.vcInformerFactory.Topology().V1alpha1().HyperNodes()
- hyperNodeInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
+ sc.hyperNodeInformer = sc.vcInformerFactory.Topology().V1alpha1().HyperNodes()
+ sc.hyperNodeInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
AddFunc: sc.AddHyperNode,
UpdateFunc: sc.UpdateHyperNode,
DeleteFunc: sc.DeleteHyperNode,
@@ -815,6 +819,9 @@ func (sc *SchedulerCache) Run(stopCh <-chan struct{}) {
go wait.Until(sc.runNodeWorker, 0, stopCh)
}
+ // Sync hyperNode.
+ go wait.Until(sc.processSyncHyperNode, 0, stopCh)
+
// Re-sync error tasks.
go wait.Until(sc.processResyncTask, 0, stopCh)
@@ -1235,6 +1242,29 @@ func (sc *SchedulerCache) processSyncNode() bool {
return true
}
+func (sc *SchedulerCache) processSyncHyperNode() {
+ worker := func() bool {
+ name, shutdown := sc.hyperNodesQueue.Get()
+ if shutdown {
+ return false
+ }
+ defer sc.hyperNodesQueue.Done(name)
+
+ klog.V(5).Infof("started sync hyperNode %s", name)
+ err := sc.SyncHyperNode(name)
+ if err == nil {
+ sc.hyperNodesQueue.Forget(name)
+ return true
+ }
+
+ klog.ErrorS(err, "Failed to sync hyperNode, retry it.", "name", name)
+ sc.hyperNodesQueue.AddRateLimited(name)
+ return true
+ }
+ for worker() {
+ }
+}
+
// AddBindTask add task to be bind to a cache which consumes by go runtime
func (sc *SchedulerCache) AddBindTask(taskInfo *schedulingapi.TaskInfo) error {
klog.V(5).Infof("add bind task %v/%v", taskInfo.Namespace, taskInfo.Name)
diff --git a/pkg/scheduler/cache/cache_mock.go b/pkg/scheduler/cache/cache_mock.go
index be92bef3c3..689021ec88 100644
--- a/pkg/scheduler/cache/cache_mock.go
+++ b/pkg/scheduler/cache/cache_mock.go
@@ -107,6 +107,7 @@ func newMockSchedulerCache(schedulerName string) *SchedulerCache {
errTasks: workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter()),
nodeQueue: workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter()),
DeletedJobs: workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter()),
+ hyperNodesQueue: workqueue.NewTypedRateLimitingQueue[string](workqueue.DefaultTypedControllerRateLimiter[string]()),
kubeClient: fake.NewSimpleClientset(),
vcClient: fakevcClient.NewSimpleClientset(),
restConfig: nil,
diff --git a/pkg/scheduler/cache/dumper.go b/pkg/scheduler/cache/dumper.go
index 6281aace5d..38b94781dd 100644
--- a/pkg/scheduler/cache/dumper.go
+++ b/pkg/scheduler/cache/dumper.go
@@ -118,7 +118,7 @@ func (d *Dumper) printHyperNodeInfo(HyperNodesSetByTier map[int]sets.Set[string]
data.WriteString("\n")
for tier, hyperNodes := range HyperNodesSetByTier {
for hyperNode := range hyperNodes {
- data.WriteString(fmt.Sprintf("tier: %d, HyperNodeName: %s, Nodes: %s\n", tier, hyperNode, HyperNodes[hyperNode]))
+ data.WriteString(fmt.Sprintf("Tier: %d, HyperNodeName: %s, Nodes: %s\n", tier, hyperNode, HyperNodes[hyperNode]))
}
}
data.WriteString("\n")
diff --git a/pkg/scheduler/cache/event_handlers.go b/pkg/scheduler/cache/event_handlers.go
index a03755464c..ab5b767b18 100644
--- a/pkg/scheduler/cache/event_handlers.go
+++ b/pkg/scheduler/cache/event_handlers.go
@@ -552,6 +552,7 @@ func (sc *SchedulerCache) AddNode(obj interface{}) {
return
}
sc.nodeQueue.Add(node.Name)
+ sc.hyperNodesQueue.Add(string(hyperNodeEventSourceNode) + "/" + node.Name)
}
// UpdateNode update node to scheduler cache
@@ -587,6 +588,7 @@ func (sc *SchedulerCache) DeleteNode(obj interface{}) {
return
}
sc.nodeQueue.Add(node.Name)
+ sc.hyperNodesQueue.Add(string(hyperNodeEventSourceNode) + "/" + node.Name)
}
func (sc *SchedulerCache) SyncNode(nodeName string) error {
@@ -695,6 +697,72 @@ func (sc *SchedulerCache) DeleteCSINode(obj interface{}) {
sc.nodeQueue.Add(csiNode.Name)
}
+func (sc *SchedulerCache) SyncHyperNode(name string) error {
+ eventSource := getHyperNodeEventSource(name)
+ if eventSource == nil {
+ klog.ErrorS(nil, "Failed to get event source for node <%s>", "eventSource", name)
+ // just return nil and not process again as it's an invalid event.
+ return nil
+ }
+ switch eventSource[0] {
+ case string(hyperNodeEventSourceNode):
+ return sc.triggerUpdateHyperNode(eventSource[1])
+ case string(hyperNodeEventSourceHyperNode):
+ return sc.processHyperNode(eventSource[1])
+ default:
+ // should never happen.
+ klog.ErrorS(nil, "Unknown event source", "name", eventSource)
+ }
+ return nil
+}
+
+func (sc *SchedulerCache) triggerUpdateHyperNode(name string) error {
+ sc.HyperNodesInfo.Lock()
+ defer sc.HyperNodesInfo.Unlock()
+
+ leafNodes := sc.HyperNodesInfo.GetRegexSelectorLeafHyperNodes()
+ if len(leafNodes) == 0 {
+ klog.V(3).InfoS("No need to update hyperNode cache when node added or deleted")
+ return nil
+ }
+
+ for leafNode := range leafNodes {
+ hn := sc.HyperNodesInfo.HyperNode(leafNode)
+ if hn == nil {
+ klog.ErrorS(nil, "Get empty hyperNode", "hyperNodeName", leafNode)
+ continue
+ }
+ match, err := sc.HyperNodesInfo.NodeRegexMatchLeafHyperNode(name, hn.Name)
+ if err != nil {
+ klog.ErrorS(err, "Failed to get node regex match leaf hyperNode", "nodeName", name, "hyperNodeName", hn.Name)
+ continue
+ }
+ if !match {
+ continue
+ }
+
+ klog.V(3).InfoS("Update leaf hyperNode and its ancestors when node added or deleted", "nodeName", name, "hyperNodeName", hn.Name)
+ if err := sc.updateHyperNode(hn); err != nil {
+ return fmt.Errorf("faield to update hyperNode: %v", err)
+ }
+ }
+ return nil
+}
+
+func (sc *SchedulerCache) processHyperNode(name string) error {
+ sc.HyperNodesInfo.Lock()
+ defer sc.HyperNodesInfo.Unlock()
+
+ hn, err := sc.hyperNodeInformer.Lister().Get(name)
+ if err != nil {
+ if !errors.IsNotFound(err) {
+ return fmt.Errorf("failed to get hyperNode <%s>: %v", name, err)
+ }
+ return sc.deleteHyperNode(name)
+ }
+ return sc.updateHyperNode(hn)
+}
+
func getJobID(pg *schedulingapi.PodGroup) schedulingapi.JobID {
return schedulingapi.JobID(fmt.Sprintf("%s/%s", pg.Namespace, pg.Name))
}
@@ -1263,46 +1331,27 @@ func (sc *SchedulerCache) setCSIResourceOnNode(csiNode *sv1.CSINode, node *v1.No
}
}
-// AddHyperNode adds HyperNode and rebuild HyperNodesInfo cache.
+// AddHyperNode adds hyperNode name to the hyperNodesQueue.
func (sc *SchedulerCache) AddHyperNode(obj interface{}) {
- hyperNode, ok := obj.(*topologyv1alpha1.HyperNode)
+ hn, ok := obj.(*topologyv1alpha1.HyperNode)
if !ok {
klog.ErrorS(nil, "Cannot convert to *topologyv1alpha1.HyperNode", "type", reflect.TypeOf(obj))
return
}
-
- sc.HyperNodesInfo.Lock()
- defer sc.HyperNodesInfo.Unlock()
-
- if err := sc.updateHyperNode(hyperNode); err != nil {
- klog.ErrorS(err, "Failed to update HyperNode", "type", reflect.TypeOf(hyperNode))
- return
- }
- klog.V(3).InfoS("Added HyperNode in cache", "name", hyperNode.Name)
+ sc.hyperNodesQueue.Add(string(hyperNodeEventSourceHyperNode) + "/" + hn.Name)
}
-// UpdateHyperNode updates HyperNode and rebuild HyperNodesInfo cache, it does three things in order:
-// 1.update parent map if members reduced.
-// 2.reset current hyperNode and its ancestors' cache.
+// UpdateHyperNode adds hyperNode name to the hyperNodesQueue.
func (sc *SchedulerCache) UpdateHyperNode(oldObj, newObj interface{}) {
newHyperNode, ok := newObj.(*topologyv1alpha1.HyperNode)
if !ok {
klog.ErrorS(nil, "Cannot convert newObj to *topologyv1alpha1.HyperNode: %v", reflect.TypeOf(newObj))
return
}
-
- sc.HyperNodesInfo.Lock()
- defer sc.HyperNodesInfo.Unlock()
-
- if err := sc.updateHyperNode(newHyperNode); err != nil {
- klog.ErrorS(err, "Failed to update HyperNode", "name", newHyperNode.Name)
- return
- }
- klog.V(3).InfoS("Updated HyperNode in cache", "name", newHyperNode.Name)
+ sc.hyperNodesQueue.Add(string(hyperNodeEventSourceHyperNode) + "/" + newHyperNode.Name)
}
-// DeleteHyperNode deletes HyperNode and rebuild HyperNodesInfo cache.
-// It clears current hyperNode and update ancestors' cache.
+// DeleteHyperNode adds hyperNode name to the hyperNodesQueue.
func (sc *SchedulerCache) DeleteHyperNode(obj interface{}) {
var hn *topologyv1alpha1.HyperNode
switch t := obj.(type) {
@@ -1319,16 +1368,18 @@ func (sc *SchedulerCache) DeleteHyperNode(obj interface{}) {
klog.ErrorS(nil, "Cannot convert to HyperNode", "type", reflect.TypeOf(t))
return
}
+ sc.hyperNodesQueue.Add(string(hyperNodeEventSourceHyperNode) + "/" + hn.Name)
+}
- sc.HyperNodesInfo.Lock()
- defer sc.HyperNodesInfo.Unlock()
-
- if err := sc.HyperNodesInfo.DeleteHyperNode(hn.Name); err != nil {
- klog.ErrorS(err, "Failed to delete HyperNode", "name", hn.Name)
- }
- klog.V(3).InfoS("Deleted HyperNode from cache", "name", hn.Name)
+// UpdateHyperNode updates HyperNode and rebuild HyperNodesInfo cache, it does three things in order:
+// 1.update parent map if members reduced.
+// 2.reset current hyperNode and its ancestors' cache.
+func (sc *SchedulerCache) updateHyperNode(hn *topologyv1alpha1.HyperNode) error {
+ return sc.HyperNodesInfo.UpdateHyperNode(hn)
}
-func (sc *SchedulerCache) updateHyperNode(hyperNode *topologyv1alpha1.HyperNode) error {
- return sc.HyperNodesInfo.UpdateHyperNode(hyperNode)
+// deleteHyperNode deletes HyperNode and rebuild HyperNodesInfo cache.
+// It clears current hyperNode and update ancestors' cache.
+func (sc *SchedulerCache) deleteHyperNode(name string) error {
+ return sc.HyperNodesInfo.DeleteHyperNode(name)
}
diff --git a/pkg/scheduler/cache/event_handlers_test.go b/pkg/scheduler/cache/event_handlers_test.go
index 189746c912..8553918d11 100644
--- a/pkg/scheduler/cache/event_handlers_test.go
+++ b/pkg/scheduler/cache/event_handlers_test.go
@@ -715,20 +715,48 @@ func TestSchedulerCache_SyncNode(t *testing.T) {
func TestSchedulerCache_AddHyperNode(t *testing.T) {
exactSelector := "exact"
- s5 := schedulingapi.BuildHyperNode("s5", 2, topologyv1alpha1.MemberTypeHyperNode, []string{"s2", "s3"}, exactSelector)
- s4 := schedulingapi.BuildHyperNode("s4", 2, topologyv1alpha1.MemberTypeHyperNode, []string{"s0", "s1"}, exactSelector)
- s3 := schedulingapi.BuildHyperNode("s3", 1, topologyv1alpha1.MemberTypeNode, []string{"node-6", "node-7"}, exactSelector)
- s1 := schedulingapi.BuildHyperNode("s1", 1, topologyv1alpha1.MemberTypeNode, []string{"node-2", "node-3"}, exactSelector)
- s2 := schedulingapi.BuildHyperNode("s2", 1, topologyv1alpha1.MemberTypeNode, []string{"node-4", "node-5"}, exactSelector)
- s0 := schedulingapi.BuildHyperNode("s0", 1, topologyv1alpha1.MemberTypeNode, []string{"node-0", "node-1"}, exactSelector)
+ s5 := schedulingapi.BuildHyperNode("s5", 2, []schedulingapi.MemberConfig{
+ {"s2", topologyv1alpha1.MemberTypeHyperNode, exactSelector},
+ {"s3", topologyv1alpha1.MemberTypeHyperNode, exactSelector},
+ })
+ s4 := schedulingapi.BuildHyperNode("s4", 2, []schedulingapi.MemberConfig{
+ {"s0", topologyv1alpha1.MemberTypeHyperNode, exactSelector},
+ {"s1", topologyv1alpha1.MemberTypeHyperNode, exactSelector},
+ })
+ s3 := schedulingapi.BuildHyperNode("s3", 1, []schedulingapi.MemberConfig{
+ {"node-6", topologyv1alpha1.MemberTypeNode, exactSelector},
+ {"node-7", topologyv1alpha1.MemberTypeNode, exactSelector},
+ })
+ s1 := schedulingapi.BuildHyperNode("s1", 1, []schedulingapi.MemberConfig{
+ {"node-2", topologyv1alpha1.MemberTypeNode, exactSelector},
+ {"node-3", topologyv1alpha1.MemberTypeNode, exactSelector},
+ })
+ s2 := schedulingapi.BuildHyperNode("s2", 1, []schedulingapi.MemberConfig{
+ {"node-4", topologyv1alpha1.MemberTypeNode, exactSelector},
+ {"node-5", topologyv1alpha1.MemberTypeNode, exactSelector},
+ })
+ s0 := schedulingapi.BuildHyperNode("s0", 1, []schedulingapi.MemberConfig{
+ {"node-0", topologyv1alpha1.MemberTypeNode, exactSelector},
+ {"node-1", topologyv1alpha1.MemberTypeNode, exactSelector},
+ })
initialHyperNodes0 := []*topologyv1alpha1.HyperNode{s5, s0, s4, s3, s1, s2}
regexSelector := "regex"
- s00 := schedulingapi.BuildHyperNode("s0", 1, topologyv1alpha1.MemberTypeNode, []string{"node-0", "node-1"}, regexSelector)
- s10 := schedulingapi.BuildHyperNode("s1", 1, topologyv1alpha1.MemberTypeNode, []string{"node-[2-3]"}, regexSelector)
- s20 := schedulingapi.BuildHyperNode("s2", 1, topologyv1alpha1.MemberTypeNode, []string{"^prefix"}, regexSelector)
+ s00 := schedulingapi.BuildHyperNode("s0", 1, []schedulingapi.MemberConfig{
+ {"node-0", topologyv1alpha1.MemberTypeNode, exactSelector},
+ {"node-1", topologyv1alpha1.MemberTypeNode, exactSelector},
+ })
+ s10 := schedulingapi.BuildHyperNode("s1", 1, []schedulingapi.MemberConfig{
+ {"node-[2-3]", topologyv1alpha1.MemberTypeNode, regexSelector},
+ })
+ s20 := schedulingapi.BuildHyperNode("s2", 1, []schedulingapi.MemberConfig{
+ {"^prefix", topologyv1alpha1.MemberTypeNode, regexSelector},
+ })
initialHyperNodes1 := []*topologyv1alpha1.HyperNode{s5, s00, s4, s3, s10, s20}
- s6 := schedulingapi.BuildHyperNode("s6", 3, topologyv1alpha1.MemberTypeHyperNode, []string{"s4", "s5"}, exactSelector)
+ s6 := schedulingapi.BuildHyperNode("s6", 3, []schedulingapi.MemberConfig{
+ {"s4", topologyv1alpha1.MemberTypeHyperNode, exactSelector},
+ {"s5", topologyv1alpha1.MemberTypeHyperNode, exactSelector},
+ })
tests := []struct {
name string
nodes []*v1.Node
@@ -836,15 +864,15 @@ func TestSchedulerCache_AddHyperNode(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
sc := NewDefaultMockSchedulerCache("volcano")
- // Add some nodes to match regex selector.
+ // Add some initialNodes to match regex selector.
for _, node := range tt.nodes {
sc.nodeInformer.Informer().GetIndexer().Add(node)
}
for _, hyperNode := range tt.initialHyperNodes {
- sc.AddHyperNode(hyperNode)
+ assert.NoError(t, sc.updateHyperNode(hyperNode))
}
for _, hyperNode := range tt.hypeNodesToAdd {
- sc.AddHyperNode(hyperNode)
+ assert.NoError(t, sc.updateHyperNode(hyperNode))
}
assert.Equal(t, tt.expectedHyperNodesSetByTier, sc.HyperNodesInfo.HyperNodesSetByTier())
assert.Equal(t, tt.expectedRealNodesSet, sc.HyperNodesInfo.RealNodesSet())
@@ -857,14 +885,37 @@ func TestSchedulerCache_AddHyperNode(t *testing.T) {
func TestSchedulerCache_Delete_Then_AddBack(t *testing.T) {
exactSelector := "exact"
- s5 := schedulingapi.BuildHyperNode("s5", 2, topologyv1alpha1.MemberTypeHyperNode, []string{"s2", "s3"}, exactSelector)
- s4 := schedulingapi.BuildHyperNode("s4", 2, topologyv1alpha1.MemberTypeHyperNode, []string{"s0", "s1"}, exactSelector)
- s3 := schedulingapi.BuildHyperNode("s3", 1, topologyv1alpha1.MemberTypeNode, []string{"node-6", "node-7"}, exactSelector)
- s1 := schedulingapi.BuildHyperNode("s1", 1, topologyv1alpha1.MemberTypeNode, []string{"node-2", "node-3"}, exactSelector)
- s2 := schedulingapi.BuildHyperNode("s2", 1, topologyv1alpha1.MemberTypeNode, []string{"node-4", "node-5"}, exactSelector)
- s0 := schedulingapi.BuildHyperNode("s0", 1, topologyv1alpha1.MemberTypeNode, []string{"node-0", "node-1"}, exactSelector)
- s6 := schedulingapi.BuildHyperNode("s6", 3, topologyv1alpha1.MemberTypeHyperNode, []string{"s4", "s5"}, exactSelector)
+
+ s5 := schedulingapi.BuildHyperNode("s5", 2, []schedulingapi.MemberConfig{
+ {"s2", topologyv1alpha1.MemberTypeHyperNode, exactSelector},
+ {"s3", topologyv1alpha1.MemberTypeHyperNode, exactSelector},
+ })
+ s4 := schedulingapi.BuildHyperNode("s4", 2, []schedulingapi.MemberConfig{
+ {"s0", topologyv1alpha1.MemberTypeHyperNode, exactSelector},
+ {"s1", topologyv1alpha1.MemberTypeHyperNode, exactSelector},
+ })
+ s3 := schedulingapi.BuildHyperNode("s3", 1, []schedulingapi.MemberConfig{
+ {"node-6", topologyv1alpha1.MemberTypeNode, exactSelector},
+ {"node-7", topologyv1alpha1.MemberTypeNode, exactSelector},
+ })
+ s1 := schedulingapi.BuildHyperNode("s1", 1, []schedulingapi.MemberConfig{
+ {"node-2", topologyv1alpha1.MemberTypeNode, exactSelector},
+ {"node-3", topologyv1alpha1.MemberTypeNode, exactSelector},
+ })
+ s2 := schedulingapi.BuildHyperNode("s2", 1, []schedulingapi.MemberConfig{
+ {"node-4", topologyv1alpha1.MemberTypeNode, exactSelector},
+ {"node-5", topologyv1alpha1.MemberTypeNode, exactSelector},
+ })
+ s0 := schedulingapi.BuildHyperNode("s0", 1, []schedulingapi.MemberConfig{
+ {"node-0", topologyv1alpha1.MemberTypeNode, exactSelector},
+ {"node-1", topologyv1alpha1.MemberTypeNode, exactSelector},
+ })
+ s6 := schedulingapi.BuildHyperNode("s6", 3, []schedulingapi.MemberConfig{
+ {"s4", topologyv1alpha1.MemberTypeHyperNode, exactSelector},
+ {"s5", topologyv1alpha1.MemberTypeHyperNode, exactSelector},
+ })
initialHyperNodes := []*topologyv1alpha1.HyperNode{s6, s5, s0, s4, s3, s1, s2}
+
tests := []struct {
name string
nodes []*v1.Node
@@ -923,15 +974,15 @@ func TestSchedulerCache_Delete_Then_AddBack(t *testing.T) {
assert.NoError(t, sc.nodeInformer.Informer().GetIndexer().Add(node))
}
for _, hyperNode := range tt.initialHyperNodes {
- sc.AddHyperNode(hyperNode)
+ assert.NoError(t, sc.updateHyperNode(hyperNode))
}
for _, hyperNode := range tt.hypeNodesToDelete {
- sc.DeleteHyperNode(hyperNode)
+ assert.NoError(t, sc.deleteHyperNode(hyperNode.Name))
}
log.Println("begin add...")
// add it back.
for _, hyperNode := range tt.hypeNodesToDelete {
- sc.AddHyperNode(hyperNode)
+ assert.NoError(t, sc.updateHyperNode(hyperNode))
}
assert.Equal(t, tt.expectedHyperNodesSetByTier, sc.HyperNodesInfo.HyperNodesSetByTier())
assert.Equal(t, tt.expectedRealNodesSet, sc.HyperNodesInfo.RealNodesSet())
@@ -946,13 +997,36 @@ func TestSchedulerCache_UpdateHyperNode(t *testing.T) {
exactSelector := "exact"
regexSelector := "regex"
- s0 := schedulingapi.BuildHyperNode("s0", 1, topologyv1alpha1.MemberTypeNode, []string{"node-0", "node-1"}, exactSelector)
- s1 := schedulingapi.BuildHyperNode("s1", 1, topologyv1alpha1.MemberTypeNode, []string{"node-2", "node-3"}, exactSelector)
- s2 := schedulingapi.BuildHyperNode("s2", 1, topologyv1alpha1.MemberTypeNode, []string{"node-4", "node-5"}, exactSelector)
- s3 := schedulingapi.BuildHyperNode("s3", 1, topologyv1alpha1.MemberTypeNode, []string{"node-6", "node-7"}, exactSelector)
- s4 := schedulingapi.BuildHyperNode("s4", 2, topologyv1alpha1.MemberTypeHyperNode, []string{"s0", "s1"}, exactSelector)
- s5 := schedulingapi.BuildHyperNode("s5", 2, topologyv1alpha1.MemberTypeHyperNode, []string{"s2", "s3"}, exactSelector)
- s6 := schedulingapi.BuildHyperNode("s6", 3, topologyv1alpha1.MemberTypeHyperNode, []string{"s4", "s5"}, exactSelector)
+ s0 := schedulingapi.BuildHyperNode("s0", 1, []schedulingapi.MemberConfig{
+ {"node-0", topologyv1alpha1.MemberTypeNode, exactSelector},
+ {"node-1", topologyv1alpha1.MemberTypeNode, exactSelector},
+ })
+ s1 := schedulingapi.BuildHyperNode("s1", 1, []schedulingapi.MemberConfig{
+ {"node-2", topologyv1alpha1.MemberTypeNode, exactSelector},
+ {"node-3", topologyv1alpha1.MemberTypeNode, exactSelector},
+ })
+ s2 := schedulingapi.BuildHyperNode("s2", 1, []schedulingapi.MemberConfig{
+ {"node-4", topologyv1alpha1.MemberTypeNode, exactSelector},
+ {"node-5", topologyv1alpha1.MemberTypeNode, exactSelector},
+ })
+ s3 := schedulingapi.BuildHyperNode("s3", 1, []schedulingapi.MemberConfig{
+ {"node-6", topologyv1alpha1.MemberTypeNode, exactSelector},
+ {"node-7", topologyv1alpha1.MemberTypeNode, exactSelector},
+ })
+ s4 := schedulingapi.BuildHyperNode("s4", 2, []schedulingapi.MemberConfig{
+ {"s0", topologyv1alpha1.MemberTypeHyperNode, exactSelector},
+ {"s1", topologyv1alpha1.MemberTypeHyperNode, exactSelector},
+ })
+
+ s5 := schedulingapi.BuildHyperNode("s5", 2, []schedulingapi.MemberConfig{
+ {"s2", topologyv1alpha1.MemberTypeHyperNode, exactSelector},
+ {"s3", topologyv1alpha1.MemberTypeHyperNode, exactSelector},
+ })
+ s6 := schedulingapi.BuildHyperNode("s6", 3,
+ []schedulingapi.MemberConfig{
+ {"s4", topologyv1alpha1.MemberTypeHyperNode, exactSelector},
+ {"s5", topologyv1alpha1.MemberTypeHyperNode, exactSelector},
+ })
initialHyperNodes := []*topologyv1alpha1.HyperNode{s0, s1, s2, s3, s4, s5, s6}
tests := []struct {
@@ -970,9 +1044,15 @@ func TestSchedulerCache_UpdateHyperNode(t *testing.T) {
initialHyperNodes: initialHyperNodes,
hyperNodesToUpdated: []*topologyv1alpha1.HyperNode{
// first remove s2 from s5.
- schedulingapi.BuildHyperNode("s5", 2, topologyv1alpha1.MemberTypeHyperNode, []string{"s3"}, exactSelector),
+ schedulingapi.BuildHyperNode("s5", 2, []schedulingapi.MemberConfig{
+ {"s3", topologyv1alpha1.MemberTypeHyperNode, exactSelector},
+ }),
// second add s2 to s4.
- schedulingapi.BuildHyperNode("s4", 2, topologyv1alpha1.MemberTypeHyperNode, []string{"s0", "s1", "s2"}, exactSelector),
+ schedulingapi.BuildHyperNode("s4", 2, []schedulingapi.MemberConfig{
+ {"s0", topologyv1alpha1.MemberTypeHyperNode, exactSelector},
+ {"s1", topologyv1alpha1.MemberTypeHyperNode, exactSelector},
+ {"s2", topologyv1alpha1.MemberTypeHyperNode, exactSelector},
+ }),
},
expectedHyperNodesSetByTier: []map[int]sets.Set[string]{
{
@@ -1032,7 +1112,7 @@ func TestSchedulerCache_UpdateHyperNode(t *testing.T) {
name: "Remove hyperNode s3's members node-6 and node-7.",
initialHyperNodes: initialHyperNodes,
hyperNodesToUpdated: []*topologyv1alpha1.HyperNode{
- schedulingapi.BuildHyperNode("s3", 1, topologyv1alpha1.MemberTypeNode, []string{}, exactSelector),
+ schedulingapi.BuildHyperNode("s3", 1, []schedulingapi.MemberConfig{}),
},
expectedHyperNodesSetByTier: []map[int]sets.Set[string]{
{
@@ -1073,7 +1153,9 @@ func TestSchedulerCache_UpdateHyperNode(t *testing.T) {
},
initialHyperNodes: initialHyperNodes,
hyperNodesToUpdated: []*topologyv1alpha1.HyperNode{
- schedulingapi.BuildHyperNode("s2", 1, topologyv1alpha1.MemberTypeNode, []string{"-suffix"}, regexSelector),
+ schedulingapi.BuildHyperNode("s2", 1, []schedulingapi.MemberConfig{
+ {"-suffix", topologyv1alpha1.MemberTypeNode, regexSelector},
+ }),
},
expectedHyperNodesSetByTier: []map[int]sets.Set[string]{
{
@@ -1111,18 +1193,18 @@ func TestSchedulerCache_UpdateHyperNode(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
sc := NewDefaultMockSchedulerCache("volcano")
- // Add some nodes to match regex selector.
+ // Add some initialNodes to match regex selector.
for _, node := range tt.nodes {
sc.nodeInformer.Informer().GetIndexer().Add(node)
}
for _, hyperNode := range tt.initialHyperNodes {
- sc.AddHyperNode(hyperNode)
+ assert.NoError(t, sc.updateHyperNode(hyperNode))
}
log.Println("begin update...")
// compare the result by index as we have updated multi hyperNodes.
for i, hyperNode := range tt.hyperNodesToUpdated {
- sc.UpdateHyperNode(nil, hyperNode)
+ assert.NoError(t, sc.updateHyperNode(hyperNode))
assert.Equal(t, tt.expectedHyperNodesSetByTier[i], sc.HyperNodesInfo.HyperNodesSetByTier())
assert.Equal(t, tt.expectedRealNodesSet[i], sc.HyperNodesInfo.RealNodesSet(), "RealNodesSet mismatch, index %d", i)
actualHyperNodes := sc.HyperNodesInfo.HyperNodesInfo()
@@ -1135,19 +1217,40 @@ func TestSchedulerCache_UpdateHyperNode(t *testing.T) {
func TestSchedulerCache_DeleteHyperNode(t *testing.T) {
selector := "exact"
- s0 := schedulingapi.BuildHyperNode("s0", 1, topologyv1alpha1.MemberTypeNode, []string{"node-0", "node-1"}, selector)
- s1 := schedulingapi.BuildHyperNode("s1", 1, topologyv1alpha1.MemberTypeNode, []string{"node-2", "node-3"}, selector)
- s2 := schedulingapi.BuildHyperNode("s2", 1, topologyv1alpha1.MemberTypeNode, []string{"node-4", "node-5"}, selector)
- s3 := schedulingapi.BuildHyperNode("s3", 1, topologyv1alpha1.MemberTypeNode, []string{"node-6", "node-7"}, selector)
- s4 := schedulingapi.BuildHyperNode("s4", 2, topologyv1alpha1.MemberTypeHyperNode, []string{"s0", "s1"}, selector)
- s5 := schedulingapi.BuildHyperNode("s5", 2, topologyv1alpha1.MemberTypeHyperNode, []string{"s2", "s3"}, selector)
- s6 := schedulingapi.BuildHyperNode("s6", 3, topologyv1alpha1.MemberTypeHyperNode, []string{"s4", "s5"}, selector)
+ s0 := schedulingapi.BuildHyperNode("s0", 1, []schedulingapi.MemberConfig{
+ {"node-0", topologyv1alpha1.MemberTypeNode, selector},
+ {"node-1", topologyv1alpha1.MemberTypeNode, selector},
+ })
+ s1 := schedulingapi.BuildHyperNode("s1", 1, []schedulingapi.MemberConfig{
+ {"node-2", topologyv1alpha1.MemberTypeNode, selector},
+ {"node-3", topologyv1alpha1.MemberTypeNode, selector},
+ })
+ s2 := schedulingapi.BuildHyperNode("s2", 1, []schedulingapi.MemberConfig{
+ {"node-4", topologyv1alpha1.MemberTypeNode, selector},
+ {"node-5", topologyv1alpha1.MemberTypeNode, selector},
+ })
+ s3 := schedulingapi.BuildHyperNode("s3", 1, []schedulingapi.MemberConfig{
+ {"node-6", topologyv1alpha1.MemberTypeNode, selector},
+ {"node-7", topologyv1alpha1.MemberTypeNode, selector},
+ })
+ s4 := schedulingapi.BuildHyperNode("s4", 2, []schedulingapi.MemberConfig{
+ {"s0", topologyv1alpha1.MemberTypeHyperNode, selector},
+ {"s1", topologyv1alpha1.MemberTypeHyperNode, selector},
+ })
+ s5 := schedulingapi.BuildHyperNode("s5", 2, []schedulingapi.MemberConfig{
+ {"s2", topologyv1alpha1.MemberTypeHyperNode, selector},
+ {"s3", topologyv1alpha1.MemberTypeHyperNode, selector},
+ })
+ s6 := schedulingapi.BuildHyperNode("s6", 3, []schedulingapi.MemberConfig{
+ {"s4", topologyv1alpha1.MemberTypeHyperNode, selector},
+ {"s5", topologyv1alpha1.MemberTypeHyperNode, selector},
+ })
initialHyperNodes := []*topologyv1alpha1.HyperNode{s0, s1, s2, s3, s4, s5, s6}
tests := []struct {
name string
initialHyperNodes []*topologyv1alpha1.HyperNode
- typerNodesToDelete *topologyv1alpha1.HyperNode
+ hyperNodesToDelete *topologyv1alpha1.HyperNode
expectedHyperNodesSetByTier map[int]sets.Set[string]
expectedRealNodesSet map[string]sets.Set[string]
expectedHyperNodesInfo map[string]string
@@ -1156,7 +1259,7 @@ func TestSchedulerCache_DeleteHyperNode(t *testing.T) {
{
name: "Delete non-leaf hyperNode s4",
initialHyperNodes: initialHyperNodes,
- typerNodesToDelete: s4,
+ hyperNodesToDelete: s4,
expectedHyperNodesSetByTier: map[int]sets.Set[string]{
1: sets.New[string]("s0", "s1", "s2", "s3"),
2: sets.New[string]("s5"),
@@ -1183,7 +1286,7 @@ func TestSchedulerCache_DeleteHyperNode(t *testing.T) {
{
name: "Delete leaf hyperNode s0",
initialHyperNodes: initialHyperNodes,
- typerNodesToDelete: s0,
+ hyperNodesToDelete: s0,
expectedHyperNodesSetByTier: map[int]sets.Set[string]{
1: sets.New[string]("s1", "s2", "s3"),
2: sets.New[string]("s4", "s5"),
@@ -1210,7 +1313,7 @@ func TestSchedulerCache_DeleteHyperNode(t *testing.T) {
{
name: "Delete root hyperNode s6",
initialHyperNodes: initialHyperNodes,
- typerNodesToDelete: s6,
+ hyperNodesToDelete: s6,
expectedHyperNodesSetByTier: map[int]sets.Set[string]{
1: sets.New[string]("s0", "s1", "s2", "s3"),
2: sets.New[string]("s4", "s5"),
@@ -1239,11 +1342,11 @@ func TestSchedulerCache_DeleteHyperNode(t *testing.T) {
t.Run(tt.name, func(t *testing.T) {
sc := NewDefaultMockSchedulerCache("volcano")
for _, hyperNode := range tt.initialHyperNodes {
- sc.AddHyperNode(hyperNode)
+ assert.NoError(t, sc.updateHyperNode(hyperNode))
}
log.Println("begin delete...")
- sc.DeleteHyperNode(tt.typerNodesToDelete)
+ assert.NoError(t, sc.deleteHyperNode(tt.hyperNodesToDelete.Name))
assert.Equal(t, tt.expectedHyperNodesSetByTier, sc.HyperNodesInfo.HyperNodesSetByTier())
assert.Equal(t, tt.expectedRealNodesSet, sc.HyperNodesInfo.RealNodesSet(), "RealNodesSet mismatch")
actualHyperNodes := sc.HyperNodesInfo.HyperNodesInfo()
@@ -1252,3 +1355,221 @@ func TestSchedulerCache_DeleteHyperNode(t *testing.T) {
})
}
}
+
+func TestSchedulerCache_SyncHyperNode(t *testing.T) {
+ exactSelector := "exact"
+ regexSelector := "regex"
+ s6 := schedulingapi.BuildHyperNode("s6", 3, []schedulingapi.MemberConfig{
+ {"s4", topologyv1alpha1.MemberTypeHyperNode, exactSelector},
+ {"s5", topologyv1alpha1.MemberTypeHyperNode, exactSelector},
+ })
+ s5 := schedulingapi.BuildHyperNode("s5", 2, []schedulingapi.MemberConfig{
+ {"s2", topologyv1alpha1.MemberTypeHyperNode, exactSelector},
+ {"s3", topologyv1alpha1.MemberTypeHyperNode, exactSelector},
+ })
+ s4 := schedulingapi.BuildHyperNode("s4", 2, []schedulingapi.MemberConfig{
+ {"s0", topologyv1alpha1.MemberTypeHyperNode, exactSelector},
+ {"s1", topologyv1alpha1.MemberTypeHyperNode, exactSelector},
+ })
+ s3 := schedulingapi.BuildHyperNode("s3", 1, []schedulingapi.MemberConfig{
+ {"node-6", topologyv1alpha1.MemberTypeNode, exactSelector},
+ {"node-7", topologyv1alpha1.MemberTypeNode, exactSelector},
+ })
+ s2 := schedulingapi.BuildHyperNode("s2", 1, []schedulingapi.MemberConfig{
+ {"node-4", topologyv1alpha1.MemberTypeNode, exactSelector},
+ {"node-5", topologyv1alpha1.MemberTypeNode, exactSelector},
+ })
+ s20 := schedulingapi.BuildHyperNode("s2", 1, []schedulingapi.MemberConfig{
+ {"node-4", topologyv1alpha1.MemberTypeNode, exactSelector},
+ {"node-5", topologyv1alpha1.MemberTypeNode, exactSelector},
+ {"node-9", topologyv1alpha1.MemberTypeNode, regexSelector},
+ })
+ s21 := schedulingapi.BuildHyperNode("s2", 1, []schedulingapi.MemberConfig{
+ {"node-4", topologyv1alpha1.MemberTypeNode, exactSelector},
+ {"node-5", topologyv1alpha1.MemberTypeNode, regexSelector},
+ })
+ s1 := schedulingapi.BuildHyperNode("s1", 1, []schedulingapi.MemberConfig{
+ {"node-2", topologyv1alpha1.MemberTypeNode, exactSelector},
+ {"node-3", topologyv1alpha1.MemberTypeNode, exactSelector},
+ })
+ s0 := schedulingapi.BuildHyperNode("s0", 1, []schedulingapi.MemberConfig{
+ {"node-0", topologyv1alpha1.MemberTypeNode, exactSelector},
+ {"node-1", topologyv1alpha1.MemberTypeNode, exactSelector},
+ })
+ s00 := schedulingapi.BuildHyperNode("s0", 1, []schedulingapi.MemberConfig{
+ {"node-0", topologyv1alpha1.MemberTypeNode, exactSelector},
+ {"node-1", topologyv1alpha1.MemberTypeNode, exactSelector},
+ {"^prefix", topologyv1alpha1.MemberTypeNode, regexSelector},
+ })
+ s01 := schedulingapi.BuildHyperNode("s0", 1, []schedulingapi.MemberConfig{
+ {"node-[01]", topologyv1alpha1.MemberTypeNode, regexSelector},
+ })
+ initialHyperNodes0 := []*topologyv1alpha1.HyperNode{s5, s0, s4, s3, s1, s2, s6}
+ initialHyperNodes1 := []*topologyv1alpha1.HyperNode{s5, s00, s4, s3, s1, s20, s6}
+ initialHyperNodes2 := []*topologyv1alpha1.HyperNode{s5, s01, s4, s3, s1, s21, s6}
+ tests := []struct {
+ name string
+ initialNodes []*v1.Node
+ nodeToAdd []*v1.Node
+ nodeToDelete []*v1.Node
+ initialHyperNodes []*topologyv1alpha1.HyperNode
+ expectedHyperNodesListByTier map[int]sets.Set[string]
+ expectedRealNodesSet map[string]sets.Set[string]
+ expectedHyperNodesInfo map[string]string
+ ready bool
+ }{
+ {
+ name: "leaf HyperNode member with exact selector, no need to update",
+ initialNodes: []*v1.Node{
+ buildNode("node-0", nil),
+ buildNode("node-1", nil),
+ buildNode("node-2", nil),
+ buildNode("node-3", nil),
+ buildNode("node-4", nil),
+ buildNode("node-5", nil),
+ buildNode("node-6", nil),
+ buildNode("node-7", nil),
+ },
+ initialHyperNodes: initialHyperNodes0,
+ nodeToAdd: []*v1.Node{buildNode("node-8", nil)},
+ expectedHyperNodesListByTier: map[int]sets.Set[string]{
+ 1: sets.New[string]("s0", "s1", "s2", "s3"),
+ 2: sets.New[string]("s4", "s5"),
+ 3: sets.New[string]("s6"),
+ },
+ expectedRealNodesSet: map[string]sets.Set[string]{
+ "s0": sets.New[string]("node-0", "node-1"),
+ "s1": sets.New[string]("node-2", "node-3"),
+ "s2": sets.New[string]("node-4", "node-5"),
+ "s3": sets.New[string]("node-6", "node-7"),
+ "s4": sets.New[string]("node-0", "node-1", "node-2", "node-3"),
+ "s5": sets.New[string]("node-4", "node-5", "node-6", "node-7"),
+ "s6": sets.New[string]("node-0", "node-1", "node-2", "node-3", "node-4", "node-5", "node-6", "node-7"),
+ },
+ expectedHyperNodesInfo: map[string]string{
+ "s0": "Name: s0, Tier: 1, Parent: s4",
+ "s1": "Name: s1, Tier: 1, Parent: s4",
+ "s2": "Name: s2, Tier: 1, Parent: s5",
+ "s3": "Name: s3, Tier: 1, Parent: s5",
+ "s4": "Name: s4, Tier: 2, Parent: s6",
+ "s5": "Name: s5, Tier: 2, Parent: s6",
+ "s6": "Name: s6, Tier: 3, Parent: ",
+ },
+ ready: true,
+ },
+ {
+ name: "update hyperNode when new node added and matched",
+ initialNodes: []*v1.Node{
+ buildNode("node-0", nil),
+ buildNode("node-1", nil),
+ buildNode("node-2", nil),
+ buildNode("node-3", nil),
+ buildNode("node-4", nil),
+ buildNode("node-5", nil),
+ buildNode("node-6", nil),
+ buildNode("node-7", nil),
+ },
+ initialHyperNodes: initialHyperNodes1,
+ nodeToAdd: []*v1.Node{
+ buildNode("prefix-node-8", nil),
+ buildNode("node-9", nil),
+ },
+ expectedHyperNodesListByTier: map[int]sets.Set[string]{
+ 1: sets.New[string]("s0", "s1", "s2", "s3"),
+ 2: sets.New[string]("s4", "s5"),
+ 3: sets.New[string]("s6"),
+ },
+ expectedRealNodesSet: map[string]sets.Set[string]{
+ "s0": sets.New[string]("node-0", "node-1", "prefix-node-8"),
+ "s1": sets.New[string]("node-2", "node-3"),
+ "s2": sets.New[string]("node-4", "node-5", "node-9"),
+ "s3": sets.New[string]("node-6", "node-7"),
+ "s4": sets.New[string]("node-0", "node-1", "node-2", "node-3", "prefix-node-8"),
+ "s5": sets.New[string]("node-4", "node-5", "node-6", "node-7", "node-9"),
+ "s6": sets.New[string]("node-0", "node-1", "node-2", "node-3", "node-4", "node-5", "node-6", "node-7", "prefix-node-8", "node-9"),
+ },
+ expectedHyperNodesInfo: map[string]string{
+ "s0": "Name: s0, Tier: 1, Parent: s4",
+ "s1": "Name: s1, Tier: 1, Parent: s4",
+ "s2": "Name: s2, Tier: 1, Parent: s5",
+ "s3": "Name: s3, Tier: 1, Parent: s5",
+ "s4": "Name: s4, Tier: 2, Parent: s6",
+ "s5": "Name: s5, Tier: 2, Parent: s6",
+ "s6": "Name: s6, Tier: 3, Parent: ",
+ },
+ ready: true,
+ },
+ {
+ name: "update hyperNode when new node deleted",
+ initialNodes: []*v1.Node{
+ buildNode("node-0", nil),
+ buildNode("node-1", nil),
+ buildNode("node-2", nil),
+ buildNode("node-3", nil),
+ buildNode("node-4", nil),
+ buildNode("node-5", nil),
+ buildNode("node-6", nil),
+ buildNode("node-7", nil),
+ },
+ initialHyperNodes: initialHyperNodes2,
+ nodeToDelete: []*v1.Node{
+ buildNode("node-0", nil),
+ buildNode("node-5", nil),
+ },
+ expectedHyperNodesListByTier: map[int]sets.Set[string]{
+ 1: sets.New[string]("s0", "s1", "s2", "s3"),
+ 2: sets.New[string]("s4", "s5"),
+ 3: sets.New[string]("s6"),
+ },
+ expectedRealNodesSet: map[string]sets.Set[string]{
+ "s0": sets.New[string]("node-1"),
+ "s1": sets.New[string]("node-2", "node-3"),
+ "s2": sets.New[string]("node-4"),
+ "s3": sets.New[string]("node-6", "node-7"),
+ "s4": sets.New[string]("node-1", "node-2", "node-3"),
+ "s5": sets.New[string]("node-4", "node-6", "node-7"),
+ "s6": sets.New[string]("node-1", "node-2", "node-3", "node-4", "node-6", "node-7"),
+ },
+ expectedHyperNodesInfo: map[string]string{
+ "s0": "Name: s0, Tier: 1, Parent: s4",
+ "s1": "Name: s1, Tier: 1, Parent: s4",
+ "s2": "Name: s2, Tier: 1, Parent: s5",
+ "s3": "Name: s3, Tier: 1, Parent: s5",
+ "s4": "Name: s4, Tier: 2, Parent: s6",
+ "s5": "Name: s5, Tier: 2, Parent: s6",
+ "s6": "Name: s6, Tier: 3, Parent: ",
+ },
+ ready: true,
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ sc := NewDefaultMockSchedulerCache("volcano")
+ // Add some initialNodes to match regex selector.
+ for _, node := range tt.initialNodes {
+ assert.NoError(t, sc.nodeInformer.Informer().GetIndexer().Add(node))
+ }
+ for _, hyperNode := range tt.initialHyperNodes {
+ assert.NoError(t, sc.updateHyperNode(hyperNode))
+ }
+
+ for _, node := range tt.nodeToAdd {
+ assert.NoError(t, sc.nodeInformer.Informer().GetIndexer().Add(node))
+ name := "node/" + node.Name
+ assert.NoError(t, sc.SyncHyperNode(name))
+ }
+ for _, node := range tt.nodeToDelete {
+ assert.NoError(t, sc.nodeInformer.Informer().GetIndexer().Delete(node))
+ name := "node/" + node.Name
+ assert.NoError(t, sc.SyncHyperNode(name))
+ }
+
+ assert.Equal(t, tt.expectedHyperNodesListByTier, sc.HyperNodesInfo.HyperNodesSetByTier())
+ assert.Equal(t, tt.expectedRealNodesSet, sc.HyperNodesInfo.RealNodesSet())
+ actualHyperNodes := sc.HyperNodesInfo.HyperNodesInfo()
+ assert.Equal(t, tt.expectedHyperNodesInfo, actualHyperNodes)
+ assert.Equal(t, tt.ready, sc.HyperNodesInfo.Ready())
+ })
+ }
+}
diff --git a/pkg/scheduler/cache/util.go b/pkg/scheduler/cache/util.go
index 48415bee4a..f6aae15147 100644
--- a/pkg/scheduler/cache/util.go
+++ b/pkg/scheduler/cache/util.go
@@ -30,6 +30,13 @@ import (
scheduling "volcano.sh/apis/pkg/apis/scheduling/v1beta1"
)
+type hyperNodeEventSource string
+
+const (
+ hyperNodeEventSourceNode hyperNodeEventSource = "node"
+ hyperNodeEventSourceHyperNode hyperNodeEventSource = "hyperNode"
+)
+
// responsibleForPod returns false at following conditions:
// 1. The current scheduler is not specified scheduler in Pod's spec.
// 2. The Job which the Pod belongs is not assigned to current scheduler based on the hash algorithm in multi-schedulers scenario
@@ -117,3 +124,11 @@ func getMultiSchedulerInfo() (schedulerPodName string, c *consistent.Consistent)
}
return mySchedulerPodName, c
}
+
+func getHyperNodeEventSource(source string) []string {
+ parts := strings.Split(source, "/")
+ if len(parts) != 2 {
+ return nil
+ }
+ return parts
+}