From d48631704def4d49c3cce08e41c4997c48837659 Mon Sep 17 00:00:00 2001 From: Neil Shen Date: Thu, 10 Aug 2023 17:52:56 +0800 Subject: [PATCH 1/4] pkg/core, cluster: save kv and cache if region flashback (#6911) ref tikv/pd#6912, close tikv/tikv#15258 TiKV Region merge checks region in operator and in local strictly, and drops commit merge if there is a mismatch between them. This change is necessary to prevent flashback from blocking Region merge. For more details, see tikv/tikv#15258. Signed-off-by: Neil Shen Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- server/cluster/cluster.go | 2 +- server/cluster/cluster_test.go | 10 ++++++++++ server/core/region.go | 13 +++++++++++++ server/core/region_option.go | 8 ++++++++ 4 files changed, 32 insertions(+), 1 deletion(-) diff --git a/server/cluster/cluster.go b/server/cluster/cluster.go index 9fa4591430e..4159568b139 100644 --- a/server/cluster/cluster.go +++ b/server/cluster/cluster.go @@ -848,7 +848,7 @@ func (c *RaftCluster) processRegionHeartbeat(region *core.RegionInfo) error { c.coordinator.CheckTransferWitnessLeader(region) hasRegionStats := c.regionStats != nil - // Save to storage if meta is updated. + // Save to storage if meta is updated, except for flashback. // Save to cache if meta or leader is updated, or contains any down/pending peer. // Mark isNew if the region in cache does not have leader. isNew, saveKV, saveCache, needSync := regionGuide(region, origin) diff --git a/server/cluster/cluster_test.go b/server/cluster/cluster_test.go index b19fb71bce1..fdae2f24590 100644 --- a/server/cluster/cluster_test.go +++ b/server/cluster/cluster_test.go @@ -834,6 +834,16 @@ func TestRegionHeartbeat(t *testing.T) { regions[i] = region re.NoError(cluster.processRegionHeartbeat(region)) checkRegions(re, cluster.core, regions[:i+1]) + + // Flashback + region = region.Clone(core.WithFlashback(true, 1)) + regions[i] = region + re.NoError(cluster.processRegionHeartbeat(region)) + checkRegions(re, cluster.core, regions[:i+1]) + region = region.Clone(core.WithFlashback(false, 0)) + regions[i] = region + re.NoError(cluster.processRegionHeartbeat(region)) + checkRegions(re, cluster.core, regions[:i+1]) } regionCounts := make(map[uint64]int) diff --git a/server/core/region.go b/server/core/region.go index 6aa686c4b76..bdb9fb269bc 100644 --- a/server/core/region.go +++ b/server/core/region.go @@ -583,6 +583,11 @@ func (r *RegionInfo) GetReplicationStatus() *replication_modepb.RegionReplicatio return r.replicationStatus } +// IsFlashbackChanged returns true if flashback changes. +func (r *RegionInfo) IsFlashbackChanged(l *RegionInfo) bool { + return r.meta.FlashbackStartTs != l.meta.FlashbackStartTs || r.meta.IsInFlashback != l.meta.IsInFlashback +} + // IsFromHeartbeat returns whether the region info is from the region heartbeat. func (r *RegionInfo) IsFromHeartbeat() bool { return r.fromHeartbeat @@ -689,6 +694,14 @@ func GenerateRegionGuideFunc(enableLog bool) RegionGuideFunc { (region.GetReplicationStatus().GetState() != origin.GetReplicationStatus().GetState() || region.GetReplicationStatus().GetStateId() != origin.GetReplicationStatus().GetStateId()) { saveCache = true + return + } + // Do not save to kv, because 1) flashback will be eventually set to + // false, 2) flashback changes almost all regions in a cluster. + // Saving kv may downgrade PD performance when there are many regions. + if region.IsFlashbackChanged(origin) { + saveCache = true + return } if !origin.IsFromHeartbeat() { isNew = true diff --git a/server/core/region_option.go b/server/core/region_option.go index 4fa3de02862..fb2812c1554 100644 --- a/server/core/region_option.go +++ b/server/core/region_option.go @@ -170,6 +170,14 @@ func WithDecConfVer() RegionCreateOption { } } +// WithFlashback set region flashback states. +func WithFlashback(isInFlashback bool, flashbackTS uint64) RegionCreateOption { + return func(region *RegionInfo) { + region.meta.FlashbackStartTs = flashbackTS + region.meta.IsInFlashback = isInFlashback + } +} + // SetCPUUsage sets the CPU usage of the region. func SetCPUUsage(v uint64) RegionCreateOption { return func(region *RegionInfo) { From 0bbe4f68c79d0fd3837e3ade78c103690f71fb79 Mon Sep 17 00:00:00 2001 From: Neil Shen Date: Tue, 15 Aug 2023 15:11:34 +0800 Subject: [PATCH 2/4] update kvproto let kvproto match tikv's Signed-off-by: Neil Shen --- go.mod | 2 +- go.sum | 4 ++-- server/core/region.go | 2 +- server/core/region_option.go | 1 - 4 files changed, 4 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index f79572dcbfc..1eeb1900ef4 100644 --- a/go.mod +++ b/go.mod @@ -25,7 +25,7 @@ require ( github.com/pingcap/errcode v0.3.0 github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c github.com/pingcap/failpoint v0.0.0-20200702092429-9f69995143ce - github.com/pingcap/kvproto v0.0.0-20221104101942-09d82b914df1 + github.com/pingcap/kvproto v0.0.0-20230726063044-73d6d7f3756b github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 github.com/pingcap/sysutil v0.0.0-20211208032423-041a72e5860d github.com/pingcap/tidb-dashboard v0.0.0-20230607035724-661f075fc73f diff --git a/go.sum b/go.sum index a0587f98dcd..19721a5b0de 100644 --- a/go.sum +++ b/go.sum @@ -386,8 +386,8 @@ github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c/go.mod h1:X2r9ue github.com/pingcap/failpoint v0.0.0-20200702092429-9f69995143ce h1:Y1kCxlCtlPTMtVcOkjUcuQKh+YrluSo7+7YMCQSzy30= github.com/pingcap/failpoint v0.0.0-20200702092429-9f69995143ce/go.mod h1:w4PEZ5y16LeofeeGwdgZB4ddv9bLyDuIX+ljstgKZyk= github.com/pingcap/kvproto v0.0.0-20191211054548-3c6b38ea5107/go.mod h1:WWLmULLO7l8IOcQG+t+ItJ3fEcrL5FxF0Wu+HrMy26w= -github.com/pingcap/kvproto v0.0.0-20221104101942-09d82b914df1 h1:iJXUNA0LoOYuuMJ6U0tJGg2gCo/8xGZVhKLvuUWNjzw= -github.com/pingcap/kvproto v0.0.0-20221104101942-09d82b914df1/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI= +github.com/pingcap/kvproto v0.0.0-20230726063044-73d6d7f3756b h1:dLoYgMFgzUaS6fAAPdjA7oGDM0LdCIm+qhgb3PzrDps= +github.com/pingcap/kvproto v0.0.0-20230726063044-73d6d7f3756b/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI= github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8= github.com/pingcap/log v0.0.0-20210625125904-98ed8e2eb1c7/go.mod h1:8AanEdAHATuRurdGxZXBz0At+9avep+ub7U1AGYLIMM= github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 h1:HR/ylkkLmGdSSDaD8IDP+SZrdhV1Kibl9KrHxJ9eciw= diff --git a/server/core/region.go b/server/core/region.go index bdb9fb269bc..8870d30b57a 100644 --- a/server/core/region.go +++ b/server/core/region.go @@ -585,7 +585,7 @@ func (r *RegionInfo) GetReplicationStatus() *replication_modepb.RegionReplicatio // IsFlashbackChanged returns true if flashback changes. func (r *RegionInfo) IsFlashbackChanged(l *RegionInfo) bool { - return r.meta.FlashbackStartTs != l.meta.FlashbackStartTs || r.meta.IsInFlashback != l.meta.IsInFlashback + return r.meta.IsInFlashback != l.meta.IsInFlashback } // IsFromHeartbeat returns whether the region info is from the region heartbeat. diff --git a/server/core/region_option.go b/server/core/region_option.go index fb2812c1554..92fd1aeebec 100644 --- a/server/core/region_option.go +++ b/server/core/region_option.go @@ -173,7 +173,6 @@ func WithDecConfVer() RegionCreateOption { // WithFlashback set region flashback states. func WithFlashback(isInFlashback bool, flashbackTS uint64) RegionCreateOption { return func(region *RegionInfo) { - region.meta.FlashbackStartTs = flashbackTS region.meta.IsInFlashback = isInFlashback } } From 1a1d067066043797462a0e759d1a5dd89bbe0c2f Mon Sep 17 00:00:00 2001 From: Neil Shen Date: Fri, 18 Aug 2023 14:43:04 +0800 Subject: [PATCH 3/4] update kvproto Signed-off-by: Neil Shen --- client/go.mod | 2 +- client/go.sum | 4 ++-- tests/client/go.mod | 2 +- tests/client/go.sum | 5 ++--- 4 files changed, 6 insertions(+), 7 deletions(-) diff --git a/client/go.mod b/client/go.mod index 92706dbc4f1..c8055548f23 100644 --- a/client/go.mod +++ b/client/go.mod @@ -7,7 +7,7 @@ require ( github.com/opentracing/opentracing-go v1.2.0 github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00 - github.com/pingcap/kvproto v0.0.0-20221026112947-f8d61344b172 + github.com/pingcap/kvproto v0.0.0-20230726063044-73d6d7f3756b github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 github.com/prometheus/client_golang v1.11.1 github.com/stretchr/testify v1.7.0 diff --git a/client/go.sum b/client/go.sum index 849d8a8275d..dd018e4c055 100644 --- a/client/go.sum +++ b/client/go.sum @@ -105,8 +105,8 @@ github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c h1:xpW9bvK+HuuTm github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c/go.mod h1:X2r9ueLEUZgtx2cIogM0v4Zj5uvvzhuuiu7Pn8HzMPg= github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00 h1:C3N3itkduZXDZFh4N3vQ5HEtld3S+Y+StULhWVvumU0= github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00/go.mod h1:4qGtCB0QK0wBzKtFEGDhxXnSnbQApw1gc9siScUl8ew= -github.com/pingcap/kvproto v0.0.0-20221026112947-f8d61344b172 h1:FYgKV9znRQmzVrrJDZ0gUfMIvKLAMU1tu1UKJib8bEQ= -github.com/pingcap/kvproto v0.0.0-20221026112947-f8d61344b172/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI= +github.com/pingcap/kvproto v0.0.0-20230726063044-73d6d7f3756b h1:dLoYgMFgzUaS6fAAPdjA7oGDM0LdCIm+qhgb3PzrDps= +github.com/pingcap/kvproto v0.0.0-20230726063044-73d6d7f3756b/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI= github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 h1:HR/ylkkLmGdSSDaD8IDP+SZrdhV1Kibl9KrHxJ9eciw= github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= diff --git a/tests/client/go.mod b/tests/client/go.mod index 57c139c3257..208c8a1b465 100644 --- a/tests/client/go.mod +++ b/tests/client/go.mod @@ -5,7 +5,7 @@ go 1.19 require ( github.com/gogo/protobuf v1.3.2 // indirect github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00 - github.com/pingcap/kvproto v0.0.0-20221104101942-09d82b914df1 + github.com/pingcap/kvproto v0.0.0-20230726063044-73d6d7f3756b github.com/stretchr/testify v1.7.1 github.com/tikv/pd v0.0.0-00010101000000-000000000000 github.com/tikv/pd/client v0.0.0-00010101000000-000000000000 diff --git a/tests/client/go.sum b/tests/client/go.sum index f6d4f13dafa..a37db982334 100644 --- a/tests/client/go.sum +++ b/tests/client/go.sum @@ -329,9 +329,8 @@ github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c/go.mod h1:X2r9ue github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00 h1:C3N3itkduZXDZFh4N3vQ5HEtld3S+Y+StULhWVvumU0= github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00/go.mod h1:4qGtCB0QK0wBzKtFEGDhxXnSnbQApw1gc9siScUl8ew= github.com/pingcap/kvproto v0.0.0-20191211054548-3c6b38ea5107/go.mod h1:WWLmULLO7l8IOcQG+t+ItJ3fEcrL5FxF0Wu+HrMy26w= -github.com/pingcap/kvproto v0.0.0-20221026112947-f8d61344b172/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI= -github.com/pingcap/kvproto v0.0.0-20221104101942-09d82b914df1 h1:iJXUNA0LoOYuuMJ6U0tJGg2gCo/8xGZVhKLvuUWNjzw= -github.com/pingcap/kvproto v0.0.0-20221104101942-09d82b914df1/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI= +github.com/pingcap/kvproto v0.0.0-20230726063044-73d6d7f3756b h1:dLoYgMFgzUaS6fAAPdjA7oGDM0LdCIm+qhgb3PzrDps= +github.com/pingcap/kvproto v0.0.0-20230726063044-73d6d7f3756b/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI= github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8= github.com/pingcap/log v0.0.0-20210625125904-98ed8e2eb1c7/go.mod h1:8AanEdAHATuRurdGxZXBz0At+9avep+ub7U1AGYLIMM= github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 h1:HR/ylkkLmGdSSDaD8IDP+SZrdhV1Kibl9KrHxJ9eciw= From 0f8b77f93d5edb1c4d2a0dd0b381e39cdb32583f Mon Sep 17 00:00:00 2001 From: Neil Shen Date: Fri, 18 Aug 2023 15:12:24 +0800 Subject: [PATCH 4/4] update go mod Signed-off-by: Neil Shen --- tools/pd-tso-bench/go.mod | 1 - tools/pd-tso-bench/go.sum | 5 ++--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/tools/pd-tso-bench/go.mod b/tools/pd-tso-bench/go.mod index bcc8a620468..906a0e6ab1d 100644 --- a/tools/pd-tso-bench/go.mod +++ b/tools/pd-tso-bench/go.mod @@ -3,7 +3,6 @@ module github.com/tools/pd-tso-bench go 1.16 require ( - github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/influxdata/tdigest v0.0.1 github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 diff --git a/tools/pd-tso-bench/go.sum b/tools/pd-tso-bench/go.sum index be96df5ff7f..114f602bb45 100644 --- a/tools/pd-tso-bench/go.sum +++ b/tools/pd-tso-bench/go.sum @@ -16,7 +16,6 @@ github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+Ce github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -106,8 +105,8 @@ github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c h1:xpW9bvK+HuuTm github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c/go.mod h1:X2r9ueLEUZgtx2cIogM0v4Zj5uvvzhuuiu7Pn8HzMPg= github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00 h1:C3N3itkduZXDZFh4N3vQ5HEtld3S+Y+StULhWVvumU0= github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00/go.mod h1:4qGtCB0QK0wBzKtFEGDhxXnSnbQApw1gc9siScUl8ew= -github.com/pingcap/kvproto v0.0.0-20221026112947-f8d61344b172 h1:FYgKV9znRQmzVrrJDZ0gUfMIvKLAMU1tu1UKJib8bEQ= -github.com/pingcap/kvproto v0.0.0-20221026112947-f8d61344b172/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI= +github.com/pingcap/kvproto v0.0.0-20230726063044-73d6d7f3756b h1:dLoYgMFgzUaS6fAAPdjA7oGDM0LdCIm+qhgb3PzrDps= +github.com/pingcap/kvproto v0.0.0-20230726063044-73d6d7f3756b/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI= github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 h1:HR/ylkkLmGdSSDaD8IDP+SZrdhV1Kibl9KrHxJ9eciw= github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=