From 01fadf5ffe7a5f8859ff8b06b211a8db880f0718 Mon Sep 17 00:00:00 2001 From: artem_danilov Date: Tue, 7 Jan 2025 13:52:52 -0800 Subject: [PATCH 1/8] integrate circuitbreaker for region calls Signed-off-by: artem_danilov --- internal/locate/region_cache.go | 23 ++++++++++++++++++++--- tikv/region.go | 6 ++++++ 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/internal/locate/region_cache.go b/internal/locate/region_cache.go index 7fa2e99a9..ce1c1d812 100644 --- a/internal/locate/region_cache.go +++ b/internal/locate/region_cache.go @@ -69,6 +69,7 @@ import ( pd "github.com/tikv/pd/client" "github.com/tikv/pd/client/clients/router" "github.com/tikv/pd/client/opt" + "github.com/tikv/pd/client/pkg/circuitbreaker" "go.uber.org/zap" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -133,6 +134,20 @@ func nextTTL(ts int64) int64 { return ts + regionCacheTTLSec + jitter } +var pdRegionMetaCircuitBreaker = circuitbreaker.NewCircuitBreaker( + "region-meta", + circuitbreaker.Settings{ + ErrorRateThresholdPct: 0, + MinQPSForOpen: 100, + ErrorRateWindow: time.Second * 30, + CoolDownInterval: time.Second * 10, + HalfOpenSuccessCount: 1}) + +// ChangePdRegionMetaCircuitBreakerSettings changes circuit breaker changes for region metadata calls +func ChangePdRegionMetaCircuitBreakerSettings(apply func(config *circuitbreaker.Settings)) { + pdRegionMetaCircuitBreaker.ChangeSettings(apply) +} + // nextTTLWithoutJitter is used for test. func nextTTLWithoutJitter(ts int64) int64 { return ts + regionCacheTTLSec @@ -2070,10 +2085,11 @@ func (c *RegionCache) loadRegion(bo *retry.Backoffer, key []byte, isEndKey bool, start := time.Now() var reg *router.Region var err error + cbCtx := circuitbreaker.WithCircuitBreaker(ctx, pdRegionMetaCircuitBreaker) if searchPrev { - reg, err = c.pdClient.GetPrevRegion(ctx, key, opts...) + reg, err = c.pdClient.GetPrevRegion(cbCtx, key, opts...) } else { - reg, err = c.pdClient.GetRegion(ctx, key, opts...) + reg, err = c.pdClient.GetRegion(cbCtx, key, opts...) } metrics.LoadRegionCacheHistogramWhenCacheMiss.Observe(time.Since(start).Seconds()) if err != nil { @@ -2121,7 +2137,8 @@ func (c *RegionCache) loadRegionByID(bo *retry.Backoffer, regionID uint64) (*Reg } } start := time.Now() - reg, err := c.pdClient.GetRegionByID(ctx, regionID, opt.WithBuckets()) + cbCtx := circuitbreaker.WithCircuitBreaker(ctx, pdRegionMetaCircuitBreaker) + reg, err := c.pdClient.GetRegionByID(cbCtx, regionID, opt.WithBuckets()) metrics.LoadRegionCacheHistogramWithRegionByID.Observe(time.Since(start).Seconds()) if err != nil { metrics.RegionCacheCounterWithGetRegionByIDError.Inc() diff --git a/tikv/region.go b/tikv/region.go index 1bffa6e0f..0bba43abf 100644 --- a/tikv/region.go +++ b/tikv/region.go @@ -45,6 +45,7 @@ import ( "github.com/tikv/client-go/v2/oracle" "github.com/tikv/client-go/v2/tikvrpc" pd "github.com/tikv/pd/client" + "github.com/tikv/pd/client/pkg/circuitbreaker" ) // RPCContext contains data that is needed to send RPC to a region. @@ -197,6 +198,11 @@ func SetRegionCacheTTLSec(t int64) { locate.SetRegionCacheTTLSec(t) } +// ChangePdRegionMetaCircuitBreakerSettings changes circuit breaker settings for region metadata calls +func ChangePdRegionMetaCircuitBreakerSettings(apply func(config *circuitbreaker.Settings)) { + locate.ChangePdRegionMetaCircuitBreakerSettings(apply) +} + // SetRegionCacheTTLWithJitter sets region cache TTL with jitter. The real TTL is in range of [base, base+jitter). func SetRegionCacheTTLWithJitter(base int64, jitter int64) { locate.SetRegionCacheTTLWithJitter(base, jitter) From 7ab69c942f37a5eaac805bed062424cc2f4d5a1a Mon Sep 17 00:00:00 2001 From: artem_danilov Date: Tue, 7 Jan 2025 16:57:58 -0800 Subject: [PATCH 2/8] add tests to verify that circuit breaker is set for GetRegion calls Signed-off-by: artem_danilov --- internal/mockstore/mocktikv/pd.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/internal/mockstore/mocktikv/pd.go b/internal/mockstore/mocktikv/pd.go index c449ea4fc..5791c624f 100644 --- a/internal/mockstore/mocktikv/pd.go +++ b/internal/mockstore/mocktikv/pd.go @@ -55,7 +55,8 @@ import ( "github.com/tikv/pd/client/clients/tso" "github.com/tikv/pd/client/opt" "github.com/tikv/pd/client/pkg/caller" - sd "github.com/tikv/pd/client/servicediscovery" + "github.com/tikv/pd/client/pkg/circuitbreaker" +sd "github.com/tikv/pd/client/servicediscovery" "go.uber.org/atomic" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -226,6 +227,7 @@ func (m *mockTSFuture) Wait() (int64, int64, error) { } func (c *pdClient) GetRegion(ctx context.Context, key []byte, opts ...opt.GetRegionOption) (*router.Region, error) { + enforceCircuitBreakerFor("GetRegion", ctx) region, peer, buckets, downPeers := c.cluster.GetRegionByKey(key) if len(opts) == 0 { buckets = nil @@ -244,6 +246,7 @@ func (c *pdClient) GetRegionFromMember(ctx context.Context, key []byte, memberUR } func (c *pdClient) GetPrevRegion(ctx context.Context, key []byte, opts ...opt.GetRegionOption) (*router.Region, error) { + enforceCircuitBreakerFor("GetPrevRegion", ctx) region, peer, buckets, downPeers := c.cluster.GetPrevRegionByKey(key) if len(opts) == 0 { buckets = nil @@ -252,6 +255,7 @@ func (c *pdClient) GetPrevRegion(ctx context.Context, key []byte, opts ...opt.Ge } func (c *pdClient) GetRegionByID(ctx context.Context, regionID uint64, opts ...opt.GetRegionOption) (*router.Region, error) { + enforceCircuitBreakerFor("GetRegionByID", ctx) region, peer, buckets, downPeers := c.cluster.GetRegionByID(regionID) return &router.Region{Meta: region, Leader: peer, Buckets: buckets, DownPeers: downPeers}, nil } @@ -465,3 +469,9 @@ func (m *pdClient) LoadResourceGroups(ctx context.Context) ([]*rmpb.ResourceGrou func (m *pdClient) GetServiceDiscovery() sd.ServiceDiscovery { return nil } func (m *pdClient) WithCallerComponent(caller.Component) pd.Client { return m } + +func enforceCircuitBreakerFor(name string, ctx context.Context) { + if circuitbreaker.FromContext(ctx) == nil { + panic(fmt.Errorf("CircuitBreaker must be configured for %s", name)) + } +} From c325d48bed7364b04e12f7377e63f3145788f725 Mon Sep 17 00:00:00 2001 From: artem_danilov Date: Tue, 7 Jan 2025 17:05:16 -0800 Subject: [PATCH 3/8] fix formatting Signed-off-by: artem_danilov --- internal/mockstore/mocktikv/pd.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/mockstore/mocktikv/pd.go b/internal/mockstore/mocktikv/pd.go index 5791c624f..f01baaa20 100644 --- a/internal/mockstore/mocktikv/pd.go +++ b/internal/mockstore/mocktikv/pd.go @@ -56,7 +56,7 @@ import ( "github.com/tikv/pd/client/opt" "github.com/tikv/pd/client/pkg/caller" "github.com/tikv/pd/client/pkg/circuitbreaker" -sd "github.com/tikv/pd/client/servicediscovery" + sd "github.com/tikv/pd/client/servicediscovery" "go.uber.org/atomic" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -472,6 +472,6 @@ func (m *pdClient) WithCallerComponent(caller.Component) pd.Client { return m } func enforceCircuitBreakerFor(name string, ctx context.Context) { if circuitbreaker.FromContext(ctx) == nil { - panic(fmt.Errorf("CircuitBreaker must be configured for %s", name)) + panic(fmt.Errorf("CircuitBreaker must be configured for %s", name)) } } From 1d718a1ed4ec2d6ab902c2429988ca64ec99a219 Mon Sep 17 00:00:00 2001 From: artem_danilov Date: Wed, 8 Jan 2025 13:23:44 -0800 Subject: [PATCH 4/8] make all circuit breaker settings configurable Signed-off-by: artem_danilov --- config/client.go | 30 +++++++++++++++++++++++++++++- internal/locate/region_cache.go | 9 +-------- 2 files changed, 30 insertions(+), 9 deletions(-) diff --git a/config/client.go b/config/client.go index c1f5608c0..c21292da8 100644 --- a/config/client.go +++ b/config/client.go @@ -100,6 +100,8 @@ type TiKVClient struct { // If a Region has not been accessed for more than the given duration (in seconds), it // will be reloaded from the PD. RegionCacheTTL uint `toml:"region-cache-ttl" json:"region-cache-ttl"` + // CircuitBreakerSettingsList is the config for default circuit breaker settings + CircuitBreakerSettingsList CircuitBreakerSettingsList `toml:"circuit-breaker" json:"circuit-breaker"` // If a store has been up to the limit, it will return error for successive request to // prevent the store occupying too much token in dispatching level. StoreLimit int64 `toml:"store-limit" json:"store-limit"` @@ -149,6 +151,23 @@ type CoprocessorCache struct { AdmissionMinProcessMs uint64 `toml:"admission-min-process-ms" json:"-"` } +// CircuitBreakerSettings is the config for default circuit breaker settings excluding the error rate +type CircuitBreakerSettings struct { + // ErrorRateEvaluationWindowSeconds defines how long to track errors before evaluating error rate threshold. + ErrorRateEvaluationWindowSeconds uint `toml:"error-rate-evaluation-window-seconds" json:"error-rate-evaluation-window-seconds"` + // MinQPSToOpen defines the average qps over the error-rate-evaluation-window-seconds that must be met before evaluating the error rate threshold. + MinQPSToOpen uint `toml:"min-qps-to-open" json:"min-qps-to-open"` + // CooldownIntervalSeconds defines how long to wait after circuit breaker is open before go to half-open state to send a probe request. + CooldownIntervalSeconds uint `toml:"cooldown-interval-seconds" json:"cooldown-interval-seconds"` + // HalfOpenSuccessCount defines how many subsequent requests to test after cooldown period before fully close the circuit. All request in excess of this count will be errored till the circuit is fully closed pending results of the firsts HalfOpenSuccessCount requests. + HalfOpenSuccessCount uint `toml:"half_open_success_count" json:"half_open_success_count"` +} + +// CircuitBreakerSettingsList is a container to configure all circuit breakers +type CircuitBreakerSettingsList struct { + PDRegionsMetadata CircuitBreakerSettings `toml:"pd-regions-metadata" json:"pd-regions-metadata"` +} + // DefaultTiKVClient returns default config for TiKVClient. func DefaultTiKVClient() TiKVClient { return TiKVClient{ @@ -176,7 +195,16 @@ func DefaultTiKVClient() TiKVClient { EnableChunkRPC: true, - RegionCacheTTL: 600, + RegionCacheTTL: 600, + CircuitBreakerSettingsList: CircuitBreakerSettingsList{ + PDRegionsMetadata: CircuitBreakerSettings{ + ErrorRateEvaluationWindowSeconds: 30, + MinQPSToOpen: 10, + CooldownIntervalSeconds: 10, + HalfOpenSuccessCount: 1, + }, + }, + StoreLimit: 0, StoreLivenessTimeout: DefStoreLivenessTimeout, diff --git a/internal/locate/region_cache.go b/internal/locate/region_cache.go index ce1c1d812..c3874e21a 100644 --- a/internal/locate/region_cache.go +++ b/internal/locate/region_cache.go @@ -134,14 +134,7 @@ func nextTTL(ts int64) int64 { return ts + regionCacheTTLSec + jitter } -var pdRegionMetaCircuitBreaker = circuitbreaker.NewCircuitBreaker( - "region-meta", - circuitbreaker.Settings{ - ErrorRateThresholdPct: 0, - MinQPSForOpen: 100, - ErrorRateWindow: time.Second * 30, - CoolDownInterval: time.Second * 10, - HalfOpenSuccessCount: 1}) +var pdRegionMetaCircuitBreaker = circuitbreaker.NewCircuitBreaker("region-meta", circuitbreaker.AlwaysClosedSettings) // ChangePdRegionMetaCircuitBreakerSettings changes circuit breaker changes for region metadata calls func ChangePdRegionMetaCircuitBreakerSettings(apply func(config *circuitbreaker.Settings)) { From bf5b9450dbbadf2caca53cc6b50524af4c6640a3 Mon Sep 17 00:00:00 2001 From: artem_danilov Date: Wed, 8 Jan 2025 14:20:13 -0800 Subject: [PATCH 5/8] fix naming Signed-off-by: artem_danilov --- config/client.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/client.go b/config/client.go index c21292da8..f2ed2127d 100644 --- a/config/client.go +++ b/config/client.go @@ -160,12 +160,12 @@ type CircuitBreakerSettings struct { // CooldownIntervalSeconds defines how long to wait after circuit breaker is open before go to half-open state to send a probe request. CooldownIntervalSeconds uint `toml:"cooldown-interval-seconds" json:"cooldown-interval-seconds"` // HalfOpenSuccessCount defines how many subsequent requests to test after cooldown period before fully close the circuit. All request in excess of this count will be errored till the circuit is fully closed pending results of the firsts HalfOpenSuccessCount requests. - HalfOpenSuccessCount uint `toml:"half_open_success_count" json:"half_open_success_count"` + HalfOpenSuccessCount uint `toml:"half-open-success-count" json:"half-open-success-count"` } // CircuitBreakerSettingsList is a container to configure all circuit breakers type CircuitBreakerSettingsList struct { - PDRegionsMetadata CircuitBreakerSettings `toml:"pd-regions-metadata" json:"pd-regions-metadata"` + PDRegionsMetadata CircuitBreakerSettings `toml:"pd-region-metadata" json:"pd-region-metadata"` } // DefaultTiKVClient returns default config for TiKVClient. From 01926d5e6bec43347c6ff6bd10445ad3dd71bbba Mon Sep 17 00:00:00 2001 From: artem_danilov Date: Mon, 13 Jan 2025 15:30:05 -0800 Subject: [PATCH 6/8] remove tikvclient config for circuit breaker Signed-off-by: artem_danilov --- config/client.go | 30 +----------------------------- internal/locate/region_cache.go | 24 ++++++++++++++++-------- internal/mockstore/mocktikv/pd.go | 2 ++ 3 files changed, 19 insertions(+), 37 deletions(-) diff --git a/config/client.go b/config/client.go index f2ed2127d..c1f5608c0 100644 --- a/config/client.go +++ b/config/client.go @@ -100,8 +100,6 @@ type TiKVClient struct { // If a Region has not been accessed for more than the given duration (in seconds), it // will be reloaded from the PD. RegionCacheTTL uint `toml:"region-cache-ttl" json:"region-cache-ttl"` - // CircuitBreakerSettingsList is the config for default circuit breaker settings - CircuitBreakerSettingsList CircuitBreakerSettingsList `toml:"circuit-breaker" json:"circuit-breaker"` // If a store has been up to the limit, it will return error for successive request to // prevent the store occupying too much token in dispatching level. StoreLimit int64 `toml:"store-limit" json:"store-limit"` @@ -151,23 +149,6 @@ type CoprocessorCache struct { AdmissionMinProcessMs uint64 `toml:"admission-min-process-ms" json:"-"` } -// CircuitBreakerSettings is the config for default circuit breaker settings excluding the error rate -type CircuitBreakerSettings struct { - // ErrorRateEvaluationWindowSeconds defines how long to track errors before evaluating error rate threshold. - ErrorRateEvaluationWindowSeconds uint `toml:"error-rate-evaluation-window-seconds" json:"error-rate-evaluation-window-seconds"` - // MinQPSToOpen defines the average qps over the error-rate-evaluation-window-seconds that must be met before evaluating the error rate threshold. - MinQPSToOpen uint `toml:"min-qps-to-open" json:"min-qps-to-open"` - // CooldownIntervalSeconds defines how long to wait after circuit breaker is open before go to half-open state to send a probe request. - CooldownIntervalSeconds uint `toml:"cooldown-interval-seconds" json:"cooldown-interval-seconds"` - // HalfOpenSuccessCount defines how many subsequent requests to test after cooldown period before fully close the circuit. All request in excess of this count will be errored till the circuit is fully closed pending results of the firsts HalfOpenSuccessCount requests. - HalfOpenSuccessCount uint `toml:"half-open-success-count" json:"half-open-success-count"` -} - -// CircuitBreakerSettingsList is a container to configure all circuit breakers -type CircuitBreakerSettingsList struct { - PDRegionsMetadata CircuitBreakerSettings `toml:"pd-region-metadata" json:"pd-region-metadata"` -} - // DefaultTiKVClient returns default config for TiKVClient. func DefaultTiKVClient() TiKVClient { return TiKVClient{ @@ -195,16 +176,7 @@ func DefaultTiKVClient() TiKVClient { EnableChunkRPC: true, - RegionCacheTTL: 600, - CircuitBreakerSettingsList: CircuitBreakerSettingsList{ - PDRegionsMetadata: CircuitBreakerSettings{ - ErrorRateEvaluationWindowSeconds: 30, - MinQPSToOpen: 10, - CooldownIntervalSeconds: 10, - HalfOpenSuccessCount: 1, - }, - }, - + RegionCacheTTL: 600, StoreLimit: 0, StoreLivenessTimeout: DefStoreLivenessTimeout, diff --git a/internal/locate/region_cache.go b/internal/locate/region_cache.go index c3874e21a..a7007bda7 100644 --- a/internal/locate/region_cache.go +++ b/internal/locate/region_cache.go @@ -134,7 +134,17 @@ func nextTTL(ts int64) int64 { return ts + regionCacheTTLSec + jitter } -var pdRegionMetaCircuitBreaker = circuitbreaker.NewCircuitBreaker("region-meta", circuitbreaker.AlwaysClosedSettings) +var pdRegionMetaCircuitBreaker = circuitbreaker.NewCircuitBreaker("region-meta", + circuitbreaker.Settings{ + ErrorRateWindow: 30, + MinQPSForOpen: 10, + CoolDownInterval: 10, + HalfOpenSuccessCount: 1, + }) + +func withPDCircuitBreaker(ctx context.Context) context.Context { + return circuitbreaker.WithCircuitBreaker(ctx, pdRegionMetaCircuitBreaker) +} // ChangePdRegionMetaCircuitBreakerSettings changes circuit breaker changes for region metadata calls func ChangePdRegionMetaCircuitBreakerSettings(apply func(config *circuitbreaker.Settings)) { @@ -2078,11 +2088,10 @@ func (c *RegionCache) loadRegion(bo *retry.Backoffer, key []byte, isEndKey bool, start := time.Now() var reg *router.Region var err error - cbCtx := circuitbreaker.WithCircuitBreaker(ctx, pdRegionMetaCircuitBreaker) if searchPrev { - reg, err = c.pdClient.GetPrevRegion(cbCtx, key, opts...) + reg, err = c.pdClient.GetPrevRegion(withPDCircuitBreaker(ctx), key, opts...) } else { - reg, err = c.pdClient.GetRegion(cbCtx, key, opts...) + reg, err = c.pdClient.GetRegion(withPDCircuitBreaker(ctx), key, opts...) } metrics.LoadRegionCacheHistogramWhenCacheMiss.Observe(time.Since(start).Seconds()) if err != nil { @@ -2130,8 +2139,7 @@ func (c *RegionCache) loadRegionByID(bo *retry.Backoffer, regionID uint64) (*Reg } } start := time.Now() - cbCtx := circuitbreaker.WithCircuitBreaker(ctx, pdRegionMetaCircuitBreaker) - reg, err := c.pdClient.GetRegionByID(cbCtx, regionID, opt.WithBuckets()) + reg, err := c.pdClient.GetRegionByID(withPDCircuitBreaker(ctx), regionID, opt.WithBuckets()) metrics.LoadRegionCacheHistogramWithRegionByID.Observe(time.Since(start).Seconds()) if err != nil { metrics.RegionCacheCounterWithGetRegionByIDError.Inc() @@ -2211,7 +2219,7 @@ func (c *RegionCache) scanRegions(bo *retry.Backoffer, startKey, endKey []byte, } start := time.Now() //nolint:staticcheck - regionsInfo, err := c.pdClient.ScanRegions(ctx, startKey, endKey, limit, opt.WithAllowFollowerHandle()) + regionsInfo, err := c.pdClient.ScanRegions(withPDCircuitBreaker(ctx), startKey, endKey, limit, opt.WithAllowFollowerHandle()) metrics.LoadRegionCacheHistogramWithRegions.Observe(time.Since(start).Seconds()) if err != nil { if apicodec.IsDecodeError(err) { @@ -2280,7 +2288,7 @@ func (c *RegionCache) batchScanRegions(bo *retry.Backoffer, keyRanges []router.K if batchOpt.needBuckets { pdOpts = append(pdOpts, opt.WithBuckets()) } - regionsInfo, err := c.pdClient.BatchScanRegions(ctx, keyRanges, limit, pdOpts...) + regionsInfo, err := c.pdClient.BatchScanRegions(withPDCircuitBreaker(ctx), keyRanges, limit, pdOpts...) metrics.LoadRegionCacheHistogramWithBatchScanRegions.Observe(time.Since(start).Seconds()) if err != nil { if st, ok := status.FromError(err); ok && st.Code() == codes.Unimplemented { diff --git a/internal/mockstore/mocktikv/pd.go b/internal/mockstore/mocktikv/pd.go index f01baaa20..f05937ebf 100644 --- a/internal/mockstore/mocktikv/pd.go +++ b/internal/mockstore/mocktikv/pd.go @@ -261,11 +261,13 @@ func (c *pdClient) GetRegionByID(ctx context.Context, regionID uint64, opts ...o } func (c *pdClient) ScanRegions(ctx context.Context, startKey []byte, endKey []byte, limit int, opts ...opt.GetRegionOption) ([]*router.Region, error) { + enforceCircuitBreakerFor("ScanRegions", ctx) regions := c.cluster.ScanRegions(startKey, endKey, limit, opts...) return regions, nil } func (c *pdClient) BatchScanRegions(ctx context.Context, keyRanges []router.KeyRange, limit int, opts ...opt.GetRegionOption) ([]*router.Region, error) { + enforceCircuitBreakerFor("BatchScanRegions", ctx) if _, err := util.EvalFailpoint("mockBatchScanRegionsUnimplemented"); err == nil { return nil, status.Errorf(codes.Unimplemented, "mock BatchScanRegions is not implemented") } From a610a67fdceadc01f937394af8890284fd033441 Mon Sep 17 00:00:00 2001 From: artem_danilov Date: Mon, 13 Jan 2025 19:18:37 -0800 Subject: [PATCH 7/8] rename ChangePdRegionMetaCircuitBreakerSettings Signed-off-by: artem_danilov --- internal/locate/region_cache.go | 4 ++-- tikv/region.go | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/internal/locate/region_cache.go b/internal/locate/region_cache.go index a7007bda7..ee433aeb5 100644 --- a/internal/locate/region_cache.go +++ b/internal/locate/region_cache.go @@ -146,8 +146,8 @@ func withPDCircuitBreaker(ctx context.Context) context.Context { return circuitbreaker.WithCircuitBreaker(ctx, pdRegionMetaCircuitBreaker) } -// ChangePdRegionMetaCircuitBreakerSettings changes circuit breaker changes for region metadata calls -func ChangePdRegionMetaCircuitBreakerSettings(apply func(config *circuitbreaker.Settings)) { +// ChangePDRegionMetaCircuitBreakerSettings changes circuit breaker changes for region metadata calls +func ChangePDRegionMetaCircuitBreakerSettings(apply func(config *circuitbreaker.Settings)) { pdRegionMetaCircuitBreaker.ChangeSettings(apply) } diff --git a/tikv/region.go b/tikv/region.go index 0bba43abf..b9c85ba16 100644 --- a/tikv/region.go +++ b/tikv/region.go @@ -198,9 +198,9 @@ func SetRegionCacheTTLSec(t int64) { locate.SetRegionCacheTTLSec(t) } -// ChangePdRegionMetaCircuitBreakerSettings changes circuit breaker settings for region metadata calls -func ChangePdRegionMetaCircuitBreakerSettings(apply func(config *circuitbreaker.Settings)) { - locate.ChangePdRegionMetaCircuitBreakerSettings(apply) +// ChangePDRegionMetaCircuitBreakerSettings changes circuit breaker settings for region metadata calls +func ChangePDRegionMetaCircuitBreakerSettings(apply func(config *circuitbreaker.Settings)) { + locate.ChangePDRegionMetaCircuitBreakerSettings(apply) } // SetRegionCacheTTLWithJitter sets region cache TTL with jitter. The real TTL is in range of [base, base+jitter). From 53d4f96eb6e0e8e64877c0c22c46b5e7424c3b88 Mon Sep 17 00:00:00 2001 From: artem_danilov Date: Mon, 13 Jan 2025 19:30:24 -0800 Subject: [PATCH 8/8] /retest Signed-off-by: artem_danilov --- internal/locate/region_cache.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/locate/region_cache.go b/internal/locate/region_cache.go index ee433aeb5..1beb71ac7 100644 --- a/internal/locate/region_cache.go +++ b/internal/locate/region_cache.go @@ -142,6 +142,7 @@ var pdRegionMetaCircuitBreaker = circuitbreaker.NewCircuitBreaker("region-meta", HalfOpenSuccessCount: 1, }) +// wrap context with circuit breaker for PD region metadata calls func withPDCircuitBreaker(ctx context.Context) context.Context { return circuitbreaker.WithCircuitBreaker(ctx, pdRegionMetaCircuitBreaker) }