Merge branch 'master' into test-logger

pkg/keyspace/tso_keyspace_group.go

@@ -167,8 +167,16 @@ func (m *GroupManager) allocNodesToAllKeyspaceGroups(ctx context.Context) {
 	log.Info("start to alloc nodes to all keyspace groups")
 	for {
 		select {
+		case <-m.ctx.Done():
+			// When the group manager is closed, we should stop to alloc nodes to all keyspace groups.
+			// Note: If raftcluster is created failed but the group manager has been bootstrapped,
+			// we need to close this goroutine by m.cancel() rather than ctx.Done() from the raftcluster.
+			// because the ctx.Done() from the raftcluster will be triggered after raftcluster is created successfully.
+			log.Info("server is closed, stop to alloc nodes to all keyspace groups")
+			return
 		case <-ctx.Done():
-			log.Info("stop to alloc nodes to all keyspace groups")
+			// When the API leader is changed, we should stop to alloc nodes to all keyspace groups.
+			log.Info("the raftcluster is closed, stop to alloc nodes to all keyspace groups")
 			return
 		case <-ticker.C:
 		}

server/cluster/cluster.go

@@ -296,13 +296,6 @@ func (c *RaftCluster) Start(s Server) error {
 		return nil
 	}
 
-	if s.IsAPIServiceMode() {
-		err = c.keyspaceGroupManager.Bootstrap(c.ctx)
-		if err != nil {
-			return err
-		}
-	}
-
 	c.ruleManager = placement.NewRuleManager(c.storage, c, c.GetOpts())
 	if c.opt.IsPlacementRulesEnabled() {
 		err = c.ruleManager.Initialize(c.opt.GetMaxReplicas(), c.opt.GetLocationLabels())
@@ -327,6 +320,15 @@ func (c *RaftCluster) Start(s Server) error {
 		log.Error("load external timestamp meets error", zap.Error(err))
 	}
 
+	// bootstrap keyspace group manager after starting other parts successfully.
+	// This order avoids a stuck goroutine in keyspaceGroupManager when it fails to create raftcluster.
+	if s.IsAPIServiceMode() {
+		err = c.keyspaceGroupManager.Bootstrap(c.ctx)
+		if err != nil {
+			return err
+		}
+	}
+
 	c.wg.Add(10)
 	go c.runCoordinator()
 	go c.runMetricsCollectionJob()