Skip to content

Commit

Permalink
scheduler: add more hot scheduler comments and replace negative rank (#…
Browse files Browse the repository at this point in the history 
…8345)

ref #5691

Signed-off-by: lhy1024 <[email protected]>

Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com>
  • Loading branch information
@lhy1024 @ti-chi-bot
lhy1024 and ti-chi-bot[bot] committed Jul 26, 2024
1 parent 2baee83 commit 024656b
Show file tree
Hide file tree
Showing 6 changed files with 109 additions and 105 deletions.
11 changes: 7 additions & 4 deletions pkg/schedule/schedulers/hot_region.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ const (
HotRegionType = "hot-region"
splitHotReadBuckets = "split-hot-read-region"
splitHotWriteBuckets = "split-hot-write-region"
splitProgressiveRank = int64(-5)
splitProgressiveRank = 5
minHotScheduleInterval = time.Second
maxHotScheduleInterval = 20 * time.Second
defaultPendingAmpFactor = 2.0
Expand Down Expand Up @@ -127,6 +127,7 @@ func (h *baseHotScheduler) prepareForBalance(typ resourceType, cluster sche.Sche
switch typ {
case readLeader, readPeer:
// update read statistics
// avoid to update read statistics frequently
if time.Since(h.updateReadTime) >= statisticsInterval {
regionRead := cluster.RegionReadStats()
prepare(regionRead, utils.Read, constant.LeaderKind)
Expand All @@ -135,6 +136,7 @@ func (h *baseHotScheduler) prepareForBalance(typ resourceType, cluster sche.Sche
}
case writeLeader, writePeer:
// update write statistics
// avoid to update write statistics frequently
if time.Since(h.updateWriteTime) >= statisticsInterval {
regionWrite := cluster.RegionWriteStats()
prepare(regionWrite, utils.Write, constant.LeaderKind)
Expand Down Expand Up @@ -408,10 +410,10 @@ type solution struct {
cachedPeersRate []float64

// progressiveRank measures the contribution for balance.
// The smaller the rank, the better this solution is.
// If progressiveRank <= 0, this solution makes thing better.
// The bigger the rank, the better this solution is.
// If progressiveRank >= 0, this solution makes thing better.
// 0 indicates that this is a solution that cannot be used directly, but can be optimized.
// 1 indicates that this is a non-optimizable solution.
// -1 indicates that this is a non-optimizable solution.
// See `calcProgressiveRank` for more about progressive rank.
progressiveRank int64
// only for rank v2
Expand Down Expand Up @@ -483,6 +485,7 @@ type balanceSolver struct {
best *solution
ops []*operator.Operator

// maxSrc and minDst are used to calculate the rank.
maxSrc *statistics.StoreLoad
minDst *statistics.StoreLoad
rankStep *statistics.StoreLoad
Expand Down
1 change: 0 additions & 1 deletion pkg/schedule/schedulers/hot_region_config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,6 @@ import (
)

const (

// Scheduling has a bigger impact on TiFlash, so it needs to be corrected in configuration items
// In the default config, the TiKV difference is 1.05*1.05-1 = 0.1025, and the TiFlash difference is 1.15*1.15-1 = 0.3225
tiflashToleranceRatioCorrection = 0.1
Expand Down
68 changes: 38 additions & 30 deletions pkg/schedule/schedulers/hot_region_rank_v1.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,9 +29,10 @@ func initRankV1(r *balanceSolver) *rankV1 {
}

// isAvailable returns the solution is available.
// The solution should have no revertRegion and progressiveRank < 0.
// The solution should progressiveRank > 0.
// v1 does not support revert regions, so no need to check revertRegions.
func (*rankV1) isAvailable(s *solution) bool {
return s.progressiveRank < 0
return s.progressiveRank > 0
}

func (r *rankV1) checkByPriorityAndTolerance(loads []float64, f func(int) bool) bool {
Expand Down Expand Up @@ -66,25 +67,25 @@ func (r *rankV1) filterUniformStore() (string, bool) {
// If both dims are enough uniform, any schedule is unnecessary.
return "all-dim", true
}
if isUniformFirstPriority && (r.cur.progressiveRank == -1 || r.cur.progressiveRank == -3) {
// If first priority dim is enough uniform, -1 is unnecessary and maybe lead to worse balance for second priority dim
if isUniformFirstPriority && (r.cur.progressiveRank == 1 || r.cur.progressiveRank == 3) {
// If first priority dim is enough uniform, rank 1 is unnecessary and maybe lead to worse balance for second priority dim
return utils.DimToString(r.firstPriority), true
}
if isUniformSecondPriority && r.cur.progressiveRank == -2 {
// If second priority dim is enough uniform, -2 is unnecessary and maybe lead to worse balance for first priority dim
if isUniformSecondPriority && r.cur.progressiveRank == 2 {
// If second priority dim is enough uniform, rank 2 is unnecessary and maybe lead to worse balance for first priority dim
return utils.DimToString(r.secondPriority), true
}
return "", false
}

// calcProgressiveRank calculates `r.cur.progressiveRank`.
// See the comments of `solution.progressiveRank` for more about progressive rank.
// | ↓ firstPriority \ secondPriority → | isBetter | isNotWorsened | Worsened |
// | isBetter | -4 | -3 | -1 / 0 |
// | isNotWorsened | -2 | 1 | 1 |
// | Worsened | 0 | 1 | 1 |
// | ↓ firstPriority \ secondPriority → | isBetter | isNotWorsened | Worsened |
// | isBetter | 4 | 3 | 1 |
// | isNotWorsened | 2 | -1 | -1 |
// | Worsened | 0 | -1 | -1 |
func (r *rankV1) calcProgressiveRank() {
r.cur.progressiveRank = 1
r.cur.progressiveRank = -1
r.cur.calcPeersRate(r.firstPriority, r.secondPriority)
if r.cur.getPeersRateFromCache(r.firstPriority) < r.getMinRate(r.firstPriority) &&
r.cur.getPeersRateFromCache(r.secondPriority) < r.getMinRate(r.secondPriority) {
Expand All @@ -93,10 +94,10 @@ func (r *rankV1) calcProgressiveRank() {

if r.resourceTy == writeLeader {
// For write leader, only compare the first priority.
// If the first priority is better, the progressiveRank is -3.
// If the first priority is better, the progressiveRank is 3.
// Because it is not a solution that needs to be optimized.
if r.isBetterForWriteLeader() {
r.cur.progressiveRank = -3
r.cur.progressiveRank = 3
}
return
}
Expand All @@ -107,16 +108,16 @@ func (r *rankV1) calcProgressiveRank() {
switch {
case isFirstBetter && isSecondBetter:
// If belonging to the case, all two dim will be more balanced, the best choice.
r.cur.progressiveRank = -4
r.cur.progressiveRank = 4
case isFirstBetter && isSecondNotWorsened:
// If belonging to the case, the first priority dim will be more balanced, the second priority dim will be not worsened.
r.cur.progressiveRank = -3
r.cur.progressiveRank = 3
case isFirstNotWorsened && isSecondBetter:
// If belonging to the case, the first priority dim will be not worsened, the second priority dim will be more balanced.
r.cur.progressiveRank = -2
r.cur.progressiveRank = 2
case isFirstBetter:
// If belonging to the case, the first priority dim will be more balanced, ignore the second priority dim.
r.cur.progressiveRank = -1
r.cur.progressiveRank = 1
case isSecondBetter:
// If belonging to the case, the second priority dim will be more balanced, ignore the first priority dim.
// It's a solution that cannot be used directly, but can be optimized.
Expand All @@ -126,12 +127,12 @@ func (r *rankV1) calcProgressiveRank() {

// betterThan checks if `r.cur` is a better solution than `old`.
func (r *rankV1) betterThan(old *solution) bool {
if old == nil || r.cur.progressiveRank <= splitProgressiveRank {
if old == nil || r.cur.progressiveRank >= splitProgressiveRank {
return true
}
if r.cur.progressiveRank != old.progressiveRank {
// Smaller rank is better.
return r.cur.progressiveRank < old.progressiveRank
// Bigger rank is better.
return r.cur.progressiveRank > old.progressiveRank
}
if (r.cur.revertRegion == nil) != (old.revertRegion == nil) {
// Fewer revertRegions are better.
Expand Down Expand Up @@ -159,24 +160,28 @@ func (r *rankV1) betterThan(old *solution) bool {
// We will firstly consider ensuring converge faster, secondly reduce oscillation
firstCmp, secondCmp := r.getRkCmpPriorities(old)
switch r.cur.progressiveRank {
case -4: // isBetter(firstPriority) && isBetter(secondPriority)
case 4: // isBetter(firstPriority) && isBetter(secondPriority)
// Both are better, prefer the one with higher first priority rate.
// If the first priority rate is the similar, prefer the one with higher second priority rate.
if firstCmp != 0 {
return firstCmp > 0
}
return secondCmp > 0
case -3: // isBetter(firstPriority) && isNotWorsened(secondPriority)
case 3: // isBetter(firstPriority) && isNotWorsened(secondPriority)
// The first priority is better, prefer the one with higher first priority rate.
if firstCmp != 0 {
return firstCmp > 0
}
// prefer smaller second priority rate, to reduce oscillation
return secondCmp < 0
case -2: // isNotWorsened(firstPriority) && isBetter(secondPriority)
case 2: // isNotWorsened(firstPriority) && isBetter(secondPriority)
// The second priority is better, prefer the one with higher second priority rate.
if secondCmp != 0 {
return secondCmp > 0
}
// prefer smaller first priority rate, to reduce oscillation
return firstCmp < 0
case -1: // isBetter(firstPriority)
case 1: // isBetter(firstPriority)
return firstCmp > 0
// TODO: The smaller the difference between the value and the expectation, the better.
}
Expand All @@ -193,21 +198,24 @@ func (r *rankV1) getRkCmpPriorities(old *solution) (firstCmp int, secondCmp int)

func (r *rankV1) rankToDimString() string {
switch r.cur.progressiveRank {
case -4:
case 4:
return "all"
case -3:
case 3:
return utils.DimToString(r.firstPriority)
case -2:
case 2:
return utils.DimToString(r.secondPriority)
case -1:
case 1:
return utils.DimToString(r.firstPriority) + "-only"
default:
return "none"
}
}

func (*rankV1) needSearchRevertRegions() bool { return false }
func (*rankV1) setSearchRevertRegions() {}
func (*rankV1) needSearchRevertRegions() bool {
return false
}

func (*rankV1) setSearchRevertRegions() {}

func (r *rankV1) isBetterForWriteLeader() bool {
srcRate, dstRate := r.cur.getExtremeLoad(r.firstPriority)
Expand Down
72 changes: 36 additions & 36 deletions pkg/schedule/schedulers/hot_region_rank_v2.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -113,11 +113,11 @@ func initRankV2(bs *balanceSolver) *rankV2 {
}

// isAvailable returns the solution is available.
// If the solution has no revertRegion, progressiveRank should < 0.
// If the solution has some revertRegion, progressiveRank should equal to -4 or -3.
// If the solution has no revertRegion, progressiveRank should > 0.
// If the solution has some revertRegion, progressiveRank should equal to 4 or 3.
func (*rankV2) isAvailable(s *solution) bool {
// TODO: Test if revert region can be enabled for -1.
return s.progressiveRank <= -3 || (s.progressiveRank < 0 && s.revertRegion == nil)
// TODO: Test if revert region can be enabled for 1.
return s.progressiveRank >= 3 || (s.progressiveRank > 0 && s.revertRegion == nil)
}

func (r *rankV2) checkByPriorityAndTolerance(loads []float64, f func(int) bool) bool {
Expand Down Expand Up @@ -151,37 +151,37 @@ func (r *rankV2) filterUniformStore() (string, bool) {
// If both dims are enough uniform, any schedule is unnecessary.
return "all-dim", true
}
if isUniformFirstPriority && (r.cur.progressiveRank == -2 || r.cur.progressiveRank == -3) {
// If first priority dim is enough uniform, -2 is unnecessary and maybe lead to worse balance for second priority dim
if isUniformFirstPriority && (r.cur.progressiveRank == 2 || r.cur.progressiveRank == 3) {
// If first priority dim is enough uniform, rank 2 is unnecessary and maybe lead to worse balance for second priority dim
return utils.DimToString(r.firstPriority), true
}
if isUniformSecondPriority && r.cur.progressiveRank == -1 {
// If second priority dim is enough uniform, -1 is unnecessary and maybe lead to worse balance for first priority dim
if isUniformSecondPriority && r.cur.progressiveRank == 1 {
// If second priority dim is enough uniform, rank 1 is unnecessary and maybe lead to worse balance for first priority dim
return utils.DimToString(r.secondPriority), true
}
return "", false
}

// The search-revert-regions is performed only when the following conditions are met to improve performance.
// * `searchRevertRegions` is true. It depends on the result of the last `solve`.
// * The current solution is not good enough. progressiveRank == -2/0
// * The current solution is not good enough. progressiveRank == 2/0
// * The current best solution is not good enough.
// - The current best solution has progressiveRank < -2 , but contain revert regions.
// - The current best solution has progressiveRank >= -2.
// - The current best solution has progressiveRank > 2 , but contain revert regions.
// - The current best solution has progressiveRank <= 2.
func (r *rankV2) needSearchRevertRegions() bool {
if !r.sche.searchRevertRegions[r.resourceTy] {
return false
}
return (r.cur.progressiveRank == -2 || r.cur.progressiveRank == 0) &&
(r.best == nil || r.best.progressiveRank >= -2 || r.best.revertRegion != nil)
return (r.cur.progressiveRank == 2 || r.cur.progressiveRank == 0) &&
(r.best == nil || r.best.progressiveRank <= 2 || r.best.revertRegion != nil)
}

func (r *rankV2) setSearchRevertRegions() {
// The next solve is allowed to search-revert-regions only when the following conditions are met.
// * No best solution was found this time.
// * The progressiveRank of the best solution == -2. (first is better, second is worsened)
// * The progressiveRank of the best solution == 2. (first is better, second is worsened)
// * The best solution contain revert regions.
searchRevertRegions := r.best == nil || r.best.progressiveRank == -2 || r.best.revertRegion != nil
searchRevertRegions := r.best == nil || r.best.progressiveRank == 2 || r.best.revertRegion != nil
r.sche.searchRevertRegions[r.resourceTy] = searchRevertRegions
if searchRevertRegions {
event := fmt.Sprintf("%s-%s-allow-search-revert-regions", r.rwTy.String(), r.opTy.String())
Expand All @@ -191,15 +191,15 @@ func (r *rankV2) setSearchRevertRegions() {

// calcProgressiveRank calculates `r.cur.progressiveRank`.
// See the comments of `solution.progressiveRank` for more about progressive rank.
// isBetter: score > 0
// isBetter: score < 0
// isNotWorsened: score == 0
// isWorsened: score < 0
// isWorsened: score > 0
// | ↓ firstPriority \ secondPriority → | isBetter | isNotWorsened | isWorsened |
// | isBetter | -4 | -3 | -2 |
// | isNotWorsened | -1 | 1 | 1 |
// | isWorsened | 0 | 1 | 1 |
// | isBetter | 4 | 3 | 2 |
// | isNotWorsened | 1 | -1 | -1 |
// | isWorsened | 0 | -1 | -1 |
func (r *rankV2) calcProgressiveRank() {
r.cur.progressiveRank = 1
r.cur.progressiveRank = -1
r.cur.calcPeersRate(r.firstPriority, r.secondPriority)
if r.cur.getPeersRateFromCache(r.firstPriority) < r.getMinRate(r.firstPriority) &&
r.cur.getPeersRateFromCache(r.secondPriority) < r.getMinRate(r.secondPriority) {
Expand All @@ -208,10 +208,10 @@ func (r *rankV2) calcProgressiveRank() {

if r.resourceTy == writeLeader {
// For write leader, only compare the first priority.
// If the first priority is better, the progressiveRank is -3.
// If the first priority is better, the progressiveRank is 3.
// Because it is not a solution that needs to be optimized.
if r.getScoreByPriorities(r.firstPriority, r.firstPriorityRatios) > 0 {
r.cur.progressiveRank = -3
r.cur.progressiveRank = 3
}
return
}
Expand All @@ -222,16 +222,16 @@ func (r *rankV2) calcProgressiveRank() {
switch {
case firstScore > 0 && secondScore > 0:
// If belonging to the case, all two dim will be more balanced, the best choice.
r.cur.progressiveRank = -4
r.cur.progressiveRank = 4
case firstScore > 0 && secondScore == 0:
// If belonging to the case, the first priority dim will be more balanced, the second priority dim will be not worsened.
r.cur.progressiveRank = -3
r.cur.progressiveRank = 3
case firstScore > 0:
// If belonging to the case, the first priority dim will be more balanced, ignore the second priority dim.
r.cur.progressiveRank = -2
r.cur.progressiveRank = 2
case firstScore == 0 && secondScore > 0:
// If belonging to the case, the first priority dim will be not worsened, the second priority dim will be more balanced.
r.cur.progressiveRank = -1
r.cur.progressiveRank = 1
case secondScore > 0:
// If belonging to the case, the second priority dim will be more balanced, ignore the first priority dim.
// It's a solution that cannot be used directly, but can be optimized.
Expand Down Expand Up @@ -437,12 +437,12 @@ func (r *rankV2) getScoreByPriorities(dim int, rs *rankRatios) int {

// betterThan checks if `r.cur` is a better solution than `old`.
func (r *rankV2) betterThan(old *solution) bool {
if old == nil || r.cur.progressiveRank <= splitProgressiveRank {
if old == nil || r.cur.progressiveRank >= splitProgressiveRank {
return true
}
if r.cur.progressiveRank != old.progressiveRank {
// Smaller rank is better.
return r.cur.progressiveRank < old.progressiveRank
// Bigger rank is better.
return r.cur.progressiveRank > old.progressiveRank
}
if (r.cur.revertRegion == nil) != (old.revertRegion == nil) {
// Fewer revertRegions are better.
Expand Down Expand Up @@ -473,12 +473,12 @@ func (r *rankV2) betterThan(old *solution) bool {
secondCmp := getRkCmpByPriority(r.secondPriority, r.cur.secondScore, old.secondScore,
r.cur.getPeersRateFromCache(r.secondPriority), old.getPeersRateFromCache(r.secondPriority))
switch r.cur.progressiveRank {
case -4, -3, -2: // firstPriority
case 4, 3, 2: // firstPriority
if firstCmp != 0 {
return firstCmp > 0
}
return secondCmp > 0
case -1: // secondPriority
case 1: // secondPriority
if secondCmp != 0 {
return secondCmp > 0
}
Expand Down Expand Up @@ -509,13 +509,13 @@ func getRkCmpByPriority(dim int, curScore, oldScore int, curPeersRate, oldPeersR

func (r *rankV2) rankToDimString() string {
switch r.cur.progressiveRank {
case -4:
case 4:
return "all"
case -3:
case 3:
return utils.DimToString(r.firstPriority)
case -2:
case 2:
return utils.DimToString(r.firstPriority) + "-only"
case -1:
case 1:
return utils.DimToString(r.secondPriority)
default:
return "none"
Expand Down
Loading

0 comments on commit 024656b

Please sign in to comment.