scheduler: add conflict detect for grant hot leader scheduler

Signed-off-by: lhy1024 <[email protected]>
tikv · Sep 13, 2024 · 071b116 · 071b116
1 parent 0ca83cf
commit 071b116
Show file tree

Hide file tree

Showing 3 changed files with 102 additions and 5 deletions.
diff --git a/server/api/scheduler.go b/server/api/scheduler.go
@@ -15,6 +15,9 @@
 package api
 
 import (
+	"encoding/json"
+	"fmt"
+	"io"
 	"net/http"
 	"net/url"
 	"strconv"
@@ -24,7 +27,9 @@ import (
 	"github.com/pingcap/errors"
 	"github.com/pingcap/log"
 	"github.com/tikv/pd/pkg/errs"
+	"github.com/tikv/pd/pkg/mcs/utils/constant"
 	"github.com/tikv/pd/pkg/schedule/types"
+	"github.com/tikv/pd/pkg/slice"
 	"github.com/tikv/pd/pkg/utils/apiutil"
 	"github.com/tikv/pd/server"
 	"github.com/unrolled/render"
@@ -143,6 +148,15 @@ func (h *schedulerHandler) CreateScheduler(w http.ResponseWriter, r *http.Reques
 		}
 		collector(strconv.FormatUint(limit, 10))
 	case types.GrantHotRegionScheduler:
+		schedulers, err := h.getSchedulers()
+		if err != nil {
+			h.r.JSON(w, http.StatusInternalServerError, err.Error())
+			return
+		}
+		if slice.Contains(schedulers, types.BalanceHotRegionScheduler.String()) {
+			h.r.JSON(w, http.StatusBadRequest, "balance-hot-region-scheduler is running, please remove it first")
+			return
+		}
 		leaderID, ok := input["store-leader-id"].(string)
 		if !ok {
 			h.r.JSON(w, http.StatusBadRequest, "missing leader id")
@@ -155,6 +169,16 @@ func (h *schedulerHandler) CreateScheduler(w http.ResponseWriter, r *http.Reques
 		}
 		collector(leaderID)
 		collector(peerIDs)
+	case types.BalanceHotRegionScheduler:
+		schedulers, err := h.getSchedulers()
+		if err != nil {
+			h.r.JSON(w, http.StatusInternalServerError, err.Error())
+			return
+		}
+		if slice.Contains(schedulers, types.GrantHotRegionScheduler.String()) {
+			h.r.JSON(w, http.StatusBadRequest, "grant-hot-region-scheduler is running, please remove it first")
+			return
+		}
 	}
 
 	if err := h.AddScheduler(tp, args...); err != nil {
@@ -246,6 +270,48 @@ func (h *schedulerHandler) PauseOrResumeScheduler(w http.ResponseWriter, r *http
 	h.r.JSON(w, http.StatusOK, "Pause or resume the scheduler successfully.")
 }
 
+func (h *schedulerHandler) getSchedulers() ([]string, error) {
+	rc, err := h.GetRaftCluster()
+	if err != nil {
+		return nil, err
+	}
+	if !rc.IsServiceIndependent(constant.SchedulingServiceName) {
+		schedulers, err := h.GetSchedulerByStatus("", false)
+		if err != nil {
+			return nil, err
+		}
+		return schedulers.([]string), nil
+	}
+
+	addr, ok := h.svr.GetServicePrimaryAddr(h.svr.Context(), constant.SchedulingServiceName)
+	if !ok {
+		return nil, errs.ErrNotFoundSchedulingAddr.FastGenByArgs()
+	}
+	url := fmt.Sprintf("%s/scheduling/api/v1/schedulers", addr)
+	req, err := http.NewRequest(http.MethodGet, url, http.NoBody)
+	if err != nil {
+		return nil, err
+	}
+	resp, err := h.svr.GetHTTPClient().Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		return nil, errs.ErrSchedulingServer.FastGenByArgs(resp.StatusCode)
+	}
+	bs, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+	var schedulers []string
+	err = json.Unmarshal(bs, &schedulers)
+	if err != nil {
+		return nil, err
+	}
+	return schedulers, nil
+}
+
 type schedulerConfigHandler struct {
 	svr *server.Server
 	rd  *render.Render

diff --git a/tools/pd-ctl/pdctl/command/scheduler.go b/tools/pd-ctl/pdctl/command/scheduler.go
@@ -354,9 +354,10 @@ func NewSplitBucketSchedulerCommand() *cobra.Command {
 // NewGrantHotRegionSchedulerCommand returns a command to add a grant-hot-region-scheduler.
 func NewGrantHotRegionSchedulerCommand() *cobra.Command {
 	c := &cobra.Command{
-		Use:   "grant-hot-region-scheduler <store_leader_id> <store_leader_id,store_peer_id_1,store_peer_id_2>",
-		Short: "add a scheduler to grant hot region to fixed stores",
-		Run:   addSchedulerForGrantHotRegionCommandFunc,
+		Use: "grant-hot-region-scheduler <store_leader_id> <store_leader_id,store_peer_id_1,store_peer_id_2>",
+		Short: `add a scheduler to grant hot region to fixed stores. 
+				Note: If there is balance-hot-region-scheduler running, please remove it first, otherwise grant-hot-region-scheduler will not work.`,
+		Run: addSchedulerForGrantHotRegionCommandFunc,
 	}
 	return c
 }

diff --git a/tools/pd-ctl/tests/scheduler/scheduler_test.go b/tools/pd-ctl/tests/scheduler/scheduler_test.go
@@ -488,12 +488,32 @@ func (suite *schedulerTestSuite) checkSchedulerConfig(cluster *pdTests.TestClust
 	})
 
 	// test grant hot region scheduler config
-	checkSchedulerCommand(re, cmd, pdAddr, []string{"-u", pdAddr, "scheduler", "add", "grant-hot-region-scheduler", "1", "1,2,3"}, map[string]bool{
+	// case 1: add grant-hot-region-scheduler when balance-hot-region-scheduler is running
+	echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "add", "grant-hot-region-scheduler", "1", "1,2,3"}, nil)
+	re.Contains(echo, "balance-hot-region-scheduler is running, please remove it first")
+
+	// case 2: add grant-hot-region-scheduler when balance-hot-region-scheduler is paused
+	checkSchedulerCommand(re, cmd, pdAddr, []string{"-u", pdAddr, "scheduler", "pause", "balance-hot-region-scheduler", "60"}, map[string]bool{
 		"balance-region-scheduler":     true,
 		"balance-leader-scheduler":     true,
 		"balance-hot-region-scheduler": true,
-		"grant-hot-region-scheduler":   true,
 	})
+	echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "add", "grant-hot-region-scheduler", "1", "1,2,3"}, nil)
+	re.Contains(echo, "balance-hot-region-scheduler is running, please remove it first")
+
+	// case 3: add grant-hot-region-scheduler when balance-hot-region-scheduler is disabled
+	checkSchedulerCommand(re, cmd, pdAddr, []string{"-u", pdAddr, "scheduler", "remove", "balance-hot-region-scheduler"}, map[string]bool{
+		"balance-region-scheduler": true,
+		"balance-leader-scheduler": true,
+	})
+
+	checkSchedulerCommand(re, cmd, pdAddr, []string{"-u", pdAddr, "scheduler", "add", "grant-hot-region-scheduler", "1", "1,2,3"}, map[string]bool{
+		"balance-region-scheduler":   true,
+		"balance-leader-scheduler":   true,
+		"grant-hot-region-scheduler": true,
+	})
+
+	// case 4: test grant-hot-region-scheduler config
 	var conf3 map[string]any
 	expected3 := map[string]any{
 		"store-id":        []any{float64(1), float64(2), float64(3)},
@@ -509,7 +529,17 @@ func (suite *schedulerTestSuite) checkSchedulerConfig(cluster *pdTests.TestClust
 		mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "grant-hot-region-scheduler"}, &conf3)
 		return reflect.DeepEqual(expected3, conf3)
 	})
+
+	// case 5: remove grant-hot-region-scheduler
+	echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "add", "balance-hot-region-scheduler"}, nil)
+	re.Contains(echo, "grant-hot-region-scheduler is running, please remove it first")
+
 	checkSchedulerCommand(re, cmd, pdAddr, []string{"-u", pdAddr, "scheduler", "remove", "grant-hot-region-scheduler"}, map[string]bool{
+		"balance-region-scheduler": true,
+		"balance-leader-scheduler": true,
+	})
+
+	checkSchedulerCommand(re, cmd, pdAddr, []string{"-u", pdAddr, "scheduler", "add", "balance-hot-region-scheduler"}, map[string]bool{
 		"balance-region-scheduler":     true,
 		"balance-leader-scheduler":     true,
 		"balance-hot-region-scheduler": true,