From 569f3e059f37c8719946e88aed99b1e432cf32f9 Mon Sep 17 00:00:00 2001 From: glorv Date: Sun, 7 Apr 2024 16:18:20 +0800 Subject: [PATCH] unsafe_recovery: also consider applied index when selects force leader (#7920) close tikv/pd#7919, ref tikv/tikv#16717 Signed-off-by: glorv Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> Co-authored-by: Connor --- go.mod | 2 +- go.sum | 4 +- .../unsafe_recovery_controller.go | 7 ++ .../unsafe_recovery_controller_test.go | 102 ++++++++++++++++++ tests/integrations/go.mod | 2 +- tests/integrations/go.sum | 4 +- tools/go.mod | 2 +- tools/go.sum | 4 +- 8 files changed, 118 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index 7c15f7024a1..2620f5ad0a7 100644 --- a/go.mod +++ b/go.mod @@ -35,7 +35,7 @@ require ( github.com/pingcap/errcode v0.3.0 github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00 - github.com/pingcap/kvproto v0.0.0-20240222024302-881fcbf5bc41 + github.com/pingcap/kvproto v0.0.0-20240403065636-c699538f7aa1 github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21 github.com/pingcap/tidb-dashboard v0.0.0-20240326110213-9768844ff5d7 diff --git a/go.sum b/go.sum index 72d16078de4..d99804c887c 100644 --- a/go.sum +++ b/go.sum @@ -389,8 +389,8 @@ github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c/go.mod h1:X2r9ue github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00 h1:C3N3itkduZXDZFh4N3vQ5HEtld3S+Y+StULhWVvumU0= github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00/go.mod h1:4qGtCB0QK0wBzKtFEGDhxXnSnbQApw1gc9siScUl8ew= github.com/pingcap/kvproto v0.0.0-20191211054548-3c6b38ea5107/go.mod h1:WWLmULLO7l8IOcQG+t+ItJ3fEcrL5FxF0Wu+HrMy26w= -github.com/pingcap/kvproto v0.0.0-20240222024302-881fcbf5bc41 h1:7tDr4J6gGQ3OqBq+lZQkI9wlJIIXFitHjNK8ymU/SEo= -github.com/pingcap/kvproto v0.0.0-20240222024302-881fcbf5bc41/go.mod h1:rXxWk2UnwfUhLXha1jxRWPADw9eMZGWEWCg92Tgmb/8= +github.com/pingcap/kvproto v0.0.0-20240403065636-c699538f7aa1 h1:vDWWJKU6ztczn24XixahtLwcnJ15DOtSRIRM3jVtZNU= +github.com/pingcap/kvproto v0.0.0-20240403065636-c699538f7aa1/go.mod h1:rXxWk2UnwfUhLXha1jxRWPADw9eMZGWEWCg92Tgmb/8= github.com/pingcap/log v0.0.0-20210625125904-98ed8e2eb1c7/go.mod h1:8AanEdAHATuRurdGxZXBz0At+9avep+ub7U1AGYLIMM= github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 h1:HR/ylkkLmGdSSDaD8IDP+SZrdhV1Kibl9KrHxJ9eciw= github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4= diff --git a/pkg/unsaferecovery/unsafe_recovery_controller.go b/pkg/unsaferecovery/unsafe_recovery_controller.go index aa45ba6a2bd..044dbd182e2 100644 --- a/pkg/unsaferecovery/unsafe_recovery_controller.go +++ b/pkg/unsaferecovery/unsafe_recovery_controller.go @@ -34,6 +34,7 @@ import ( sc "github.com/tikv/pd/pkg/schedule/config" "github.com/tikv/pd/pkg/utils/logutil" "github.com/tikv/pd/pkg/utils/syncutil" + "github.com/tikv/pd/pkg/utils/typeutil" "go.uber.org/zap" ) @@ -780,6 +781,12 @@ func (r *regionItem) IsRaftStale(origin *regionItem, u *Controller) bool { func(a, b *regionItem) int { return int(a.report.GetRaftState().GetHardState().GetTerm()) - int(b.report.GetRaftState().GetHardState().GetTerm()) }, + // choose the peer has maximum applied index or last index. + func(a, b *regionItem) int { + maxIdxA := typeutil.MaxUint64(a.report.GetRaftState().GetLastIndex(), a.report.AppliedIndex) + maxIdxB := typeutil.MaxUint64(b.report.GetRaftState().GetLastIndex(), b.report.AppliedIndex) + return int(maxIdxA - maxIdxB) + }, func(a, b *regionItem) int { return int(a.report.GetRaftState().GetLastIndex()) - int(b.report.GetRaftState().GetLastIndex()) }, diff --git a/pkg/unsaferecovery/unsafe_recovery_controller_test.go b/pkg/unsaferecovery/unsafe_recovery_controller_test.go index 6f1fab62164..cce38285212 100644 --- a/pkg/unsaferecovery/unsafe_recovery_controller_test.go +++ b/pkg/unsaferecovery/unsafe_recovery_controller_test.go @@ -1856,3 +1856,105 @@ func newTestStores(n uint64, version string) []*core.StoreInfo { func getTestDeployPath(storeID uint64) string { return fmt.Sprintf("test/store%d", storeID) } + +func TestSelectLeader(t *testing.T) { + re := require.New(t) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + opts := mockconfig.NewTestOptions() + cluster := mockcluster.NewCluster(ctx, opts) + coordinator := schedule.NewCoordinator(ctx, cluster, hbstream.NewTestHeartbeatStreams(ctx, cluster.ID, cluster, true)) + coordinator.Run() + stores := newTestStores(6, "6.0.0") + labels := []*metapb.StoreLabel{ + { + Key: core.EngineKey, + Value: core.EngineTiFlash, + }, + } + stores[5].IsTiFlash() + core.SetStoreLabels(labels)(stores[5]) + for _, store := range stores { + cluster.PutStore(store) + } + recoveryController := NewController(cluster) + + cases := []struct { + peers []*regionItem + leaderID uint64 + }{ + { + peers: []*regionItem{ + newPeer(1, 1, 10, 5, 4), + newPeer(2, 2, 9, 9, 8), + }, + leaderID: 2, + }, + { + peers: []*regionItem{ + newPeer(1, 1, 10, 10, 9), + newPeer(2, 1, 8, 8, 15), + newPeer(3, 1, 12, 11, 11), + }, + leaderID: 2, + }, + { + peers: []*regionItem{ + newPeer(1, 1, 9, 9, 11), + newPeer(2, 1, 10, 8, 7), + newPeer(3, 1, 11, 7, 6), + }, + leaderID: 3, + }, + { + peers: []*regionItem{ + newPeer(1, 1, 11, 11, 8), + newPeer(2, 1, 11, 10, 10), + newPeer(3, 1, 11, 9, 8), + }, + leaderID: 1, + }, + { + peers: []*regionItem{ + newPeer(6, 1, 11, 11, 9), + newPeer(1, 1, 11, 11, 8), + newPeer(2, 1, 11, 10, 10), + newPeer(3, 1, 11, 9, 8), + }, + leaderID: 1, + }, + } + + for i, c := range cases { + peersMap := map[uint64][]*regionItem{ + 1: c.peers, + } + region := &metapb.Region{ + Id: 1, + } + leader := recoveryController.selectLeader(peersMap, region) + re.Equal(leader.Region().Id, c.leaderID, "case: %d", i) + } +} + +func newPeer(storeID, term, lastIndex, committedIndex, appliedIndex uint64) *regionItem { + return ®ionItem{ + storeID: storeID, + report: &pdpb.PeerReport{ + RaftState: &raft_serverpb.RaftLocalState{ + HardState: &eraftpb.HardState{ + Term: term, + Commit: committedIndex, + }, + LastIndex: lastIndex, + }, + RegionState: &raft_serverpb.RegionLocalState{ + Region: &metapb.Region{ + Id: storeID, + }, + }, + AppliedIndex: appliedIndex, + }, + } +} diff --git a/tests/integrations/go.mod b/tests/integrations/go.mod index c786db107e2..cdd244cafc1 100644 --- a/tests/integrations/go.mod +++ b/tests/integrations/go.mod @@ -14,7 +14,7 @@ require ( github.com/go-sql-driver/mysql v1.7.0 github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c - github.com/pingcap/kvproto v0.0.0-20240222024302-881fcbf5bc41 + github.com/pingcap/kvproto v0.0.0-20240403065636-c699538f7aa1 github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 github.com/prometheus/client_golang v1.19.0 github.com/prometheus/client_model v0.6.0 diff --git a/tests/integrations/go.sum b/tests/integrations/go.sum index f795950bc5f..79f7dddd130 100644 --- a/tests/integrations/go.sum +++ b/tests/integrations/go.sum @@ -368,8 +368,8 @@ github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c/go.mod h1:X2r9ue github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c h1:CgbKAHto5CQgWM9fSBIvaxsJHuGP0uM74HXtv3MyyGQ= github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c/go.mod h1:4qGtCB0QK0wBzKtFEGDhxXnSnbQApw1gc9siScUl8ew= github.com/pingcap/kvproto v0.0.0-20191211054548-3c6b38ea5107/go.mod h1:WWLmULLO7l8IOcQG+t+ItJ3fEcrL5FxF0Wu+HrMy26w= -github.com/pingcap/kvproto v0.0.0-20240222024302-881fcbf5bc41 h1:7tDr4J6gGQ3OqBq+lZQkI9wlJIIXFitHjNK8ymU/SEo= -github.com/pingcap/kvproto v0.0.0-20240222024302-881fcbf5bc41/go.mod h1:rXxWk2UnwfUhLXha1jxRWPADw9eMZGWEWCg92Tgmb/8= +github.com/pingcap/kvproto v0.0.0-20240403065636-c699538f7aa1 h1:vDWWJKU6ztczn24XixahtLwcnJ15DOtSRIRM3jVtZNU= +github.com/pingcap/kvproto v0.0.0-20240403065636-c699538f7aa1/go.mod h1:rXxWk2UnwfUhLXha1jxRWPADw9eMZGWEWCg92Tgmb/8= github.com/pingcap/log v0.0.0-20210625125904-98ed8e2eb1c7/go.mod h1:8AanEdAHATuRurdGxZXBz0At+9avep+ub7U1AGYLIMM= github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 h1:HR/ylkkLmGdSSDaD8IDP+SZrdhV1Kibl9KrHxJ9eciw= github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4= diff --git a/tools/go.mod b/tools/go.mod index b0de9a8a9da..8287f834471 100644 --- a/tools/go.mod +++ b/tools/go.mod @@ -21,7 +21,7 @@ require ( github.com/mattn/go-shellwords v1.0.12 github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00 - github.com/pingcap/kvproto v0.0.0-20240222024302-881fcbf5bc41 + github.com/pingcap/kvproto v0.0.0-20240403065636-c699538f7aa1 github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.19.0 diff --git a/tools/go.sum b/tools/go.sum index bea8714ee39..ac6cc75903e 100644 --- a/tools/go.sum +++ b/tools/go.sum @@ -365,8 +365,8 @@ github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c/go.mod h1:X2r9ue github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00 h1:C3N3itkduZXDZFh4N3vQ5HEtld3S+Y+StULhWVvumU0= github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00/go.mod h1:4qGtCB0QK0wBzKtFEGDhxXnSnbQApw1gc9siScUl8ew= github.com/pingcap/kvproto v0.0.0-20191211054548-3c6b38ea5107/go.mod h1:WWLmULLO7l8IOcQG+t+ItJ3fEcrL5FxF0Wu+HrMy26w= -github.com/pingcap/kvproto v0.0.0-20240222024302-881fcbf5bc41 h1:7tDr4J6gGQ3OqBq+lZQkI9wlJIIXFitHjNK8ymU/SEo= -github.com/pingcap/kvproto v0.0.0-20240222024302-881fcbf5bc41/go.mod h1:rXxWk2UnwfUhLXha1jxRWPADw9eMZGWEWCg92Tgmb/8= +github.com/pingcap/kvproto v0.0.0-20240403065636-c699538f7aa1 h1:vDWWJKU6ztczn24XixahtLwcnJ15DOtSRIRM3jVtZNU= +github.com/pingcap/kvproto v0.0.0-20240403065636-c699538f7aa1/go.mod h1:rXxWk2UnwfUhLXha1jxRWPADw9eMZGWEWCg92Tgmb/8= github.com/pingcap/log v0.0.0-20210625125904-98ed8e2eb1c7/go.mod h1:8AanEdAHATuRurdGxZXBz0At+9avep+ub7U1AGYLIMM= github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 h1:HR/ylkkLmGdSSDaD8IDP+SZrdhV1Kibl9KrHxJ9eciw= github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4=