diff --git a/.github/workflows/pd-tests.yaml b/.github/workflows/pd-tests.yaml index e9bd15f379b..1a9e353b21a 100644 --- a/.github/workflows/pd-tests.yaml +++ b/.github/workflows/pd-tests.yaml @@ -6,12 +6,14 @@ on: - release-4.0 - release-5.* - release-6.* + - release-7.* pull_request: branches: - master - release-4.0 - release-5.* - release-6.* + - release-7.* concurrency: group: ${{ github.ref }}-${{ github.workflow }} cancel-in-progress: true @@ -21,9 +23,9 @@ jobs: strategy: fail-fast: true matrix: - worker_id: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] + worker_id: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13] outputs: - job-total: 10 + job-total: 13 steps: - uses: actions/setup-go@v3 with: @@ -42,10 +44,10 @@ jobs: - name: Make Test env: WORKER_ID: ${{ matrix.worker_id }} - WORKER_COUNT: 10 + WORKER_COUNT: 13 + JOB_COUNT: 10 # 11, 12 13 are for other integrations jobs run: | - [ $WORKER_ID -ne $WORKER_COUNT ] && make ci-test-job JOB_COUNT=$(($WORKER_COUNT-1)) JOB_INDEX=$WORKER_ID - [ $WORKER_ID -eq $WORKER_COUNT ] && make ci-test-job-submod + make ci-test-job JOB_COUNT=$(($JOB_COUNT)) JOB_INDEX=$WORKER_ID mv covprofile covprofile_$WORKER_ID sed -i "/failpoint_binding/d" covprofile_$WORKER_ID - name: Upload coverage result ${{ matrix.worker_id }} diff --git a/.github/workflows/tso-function-test.yaml b/.github/workflows/tso-function-test.yaml index fa8008273c8..7c988e465ee 100644 --- a/.github/workflows/tso-function-test.yaml +++ b/.github/workflows/tso-function-test.yaml @@ -5,11 +5,13 @@ on: - master - release-5.* - release-6.* + - release-7.* pull_request: branches: - master - release-5.* - release-6.* + - release-7.* concurrency: group: ${{ github.ref }}-${{ github.workflow }} cancel-in-progress: true diff --git a/.gitignore b/.gitignore index 93e6189a687..748d24872b6 100644 --- a/.gitignore +++ b/.gitignore @@ -24,3 +24,4 @@ report.xml coverage.xml coverage *.txt +go.work* diff --git a/Makefile b/Makefile index 14de2eb5263..94357321eab 100644 --- a/Makefile +++ b/Makefile @@ -149,7 +149,7 @@ static: install-tools @ echo "gofmt ..." @ gofmt -s -l -d $(PACKAGE_DIRECTORIES) 2>&1 | awk '{ print } END { if (NR > 0) { exit 1 } }' @ echo "golangci-lint ..." - @ golangci-lint run --verbose $(PACKAGE_DIRECTORIES) + @ golangci-lint run --verbose $(PACKAGE_DIRECTORIES) --allow-parallel-runners @ echo "revive ..." @ revive -formatter friendly -config revive.toml $(PACKAGES) @@ -215,13 +215,11 @@ basic-test: install-tools ci-test-job: install-tools dashboard-ui @$(FAILPOINT_ENABLE) - CGO_ENABLED=1 go test -timeout=15m -tags deadlock -race -covermode=atomic -coverprofile=covprofile -coverpkg=./... $(shell ./scripts/ci-subtask.sh $(JOB_COUNT) $(JOB_INDEX)) - @$(FAILPOINT_DISABLE) - -ci-test-job-submod: install-tools dashboard-ui - @$(FAILPOINT_ENABLE) - @ for mod in $(SUBMODULES); do cd $$mod && $(MAKE) ci-test-job && cd $(ROOT_PATH) > /dev/null && cat $$mod/covprofile >> covprofile; done - @$(FAILPOINT_DISABLE) + if [[ $(JOB_INDEX) -le 10 ]]; then \ + CGO_ENABLED=1 go test -timeout=15m -tags deadlock -race -covermode=atomic -coverprofile=covprofile -coverpkg=./... $(shell ./scripts/ci-subtask.sh $(JOB_COUNT) $(JOB_INDEX)); \ + else \ + for mod in $(shell ./scripts/ci-subtask.sh $(JOB_COUNT) $(JOB_INDEX)); do cd $$mod && $(MAKE) ci-test-job && cd $(ROOT_PATH) > /dev/null && cat $$mod/covprofile >> covprofile; done; \ + fi TSO_INTEGRATION_TEST_PKGS := $(PD_PKG)/tests/server/tso @@ -244,7 +242,7 @@ test-tso-consistency: install-tools TASK_COUNT=1 TASK_ID=1 -# The command should be used in daily CI，it will split some tasks to run parallel. +# The command should be used in daily CI, it will split some tasks to run parallel. # It should retain report.xml,coverage,coverage.xml and package.list to analyze. test-with-cover-parallel: install-tools dashboard-ui split @$(FAILPOINT_ENABLE) diff --git a/README.md b/README.md index e9a4e0f77ed..cc4a6781b01 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ If you're interested in contributing to PD, see [CONTRIBUTING.md](./CONTRIBUTING ## Build -1. Make sure [*Go*](https://golang.org/) (version 1.19) is installed. +1. Make sure [*Go*](https://golang.org/) (version 1.20) is installed. 2. Use `make` to install PD. PD is installed in the `bin` directory. ## Usage diff --git a/client/Makefile b/client/Makefile index b55b6fc8d9d..dae53222d92 100644 --- a/client/Makefile +++ b/client/Makefile @@ -19,21 +19,24 @@ SHELL := env PATH='$(PATH)' GOBIN='$(GO_TOOLS_BIN_PATH)' $(shell which bash) default: static tidy test test: - CGO_ENABLE=1 go test -race -cover + CGO_ENABLE=1 go test ./... -race -cover basic-test: - CGO_ENABLE=1 go test + CGO_ENABLE=1 go test ./... ci-test-job: - CGO_ENABLED=1 go test -race -covermode=atomic -coverprofile=covprofile -coverpkg=../... github.com/tikv/pd/client + CGO_ENABLED=1 go test ./... -race -covermode=atomic -coverprofile=covprofile -coverpkg=../... github.com/tikv/pd/client install-tools: cd .. && $(MAKE) install-tools static: install-tools + @ echo "gofmt ..." @ gofmt -s -l -d . 2>&1 | awk '{ print } END { if (NR > 0) { exit 1 } }' - @ golangci-lint run -c ../.golangci.yml ./... - @ revive -formatter friendly -config ../revive.toml . + @ echo "golangci-lint ..." + @ golangci-lint run -c ../.golangci.yml --verbose ./... --allow-parallel-runners + @ echo "revive ..." + @ revive -formatter friendly -config ../revive.toml ./... tidy: @ go mod tidy diff --git a/client/client.go b/client/client.go index 0105e6c51eb..b1b6e862e74 100644 --- a/client/client.go +++ b/client/client.go @@ -18,6 +18,7 @@ import ( "context" "fmt" "math/rand" + "runtime/trace" "strings" "sync" "sync/atomic" @@ -33,6 +34,7 @@ import ( "github.com/tikv/pd/client/errs" "github.com/tikv/pd/client/grpcutil" "github.com/tikv/pd/client/tlsutil" + "github.com/tikv/pd/client/tsoutil" "go.uber.org/zap" "google.golang.org/grpc" "google.golang.org/grpc/codes" @@ -40,6 +42,21 @@ import ( "google.golang.org/grpc/status" ) +const ( + // defaultKeyspaceID is the default key space id. + // Valid keyspace id range is [0, 0xFFFFFF](uint24max, or 16777215) + // ​0 is reserved for default keyspace with the name "DEFAULT", It's initialized + // when PD bootstrap and reserved for users who haven't been assigned keyspace. + defaultKeyspaceID = uint32(0) + maxKeyspaceID = uint32(0xFFFFFF) + // nullKeyspaceID is used for api v1 or legacy path where is keyspace agnostic. + nullKeyspaceID = uint32(0xFFFFFFFF) + // defaultKeySpaceGroupID is the default key space group id. + // We also reserved 0 for the keyspace group for the same purpose. + defaultKeySpaceGroupID = uint32(0) + defaultKeyspaceName = "DEFAULT" +) + // Region contains information of a region's meta and its peers. type Region struct { Meta *metapb.Region @@ -67,14 +84,6 @@ type Client interface { // GetLeaderAddr returns current leader's address. It returns "" before // syncing leader from server. GetLeaderAddr() string - // GetTS gets a timestamp from PD. - GetTS(ctx context.Context) (int64, int64, error) - // GetTSAsync gets a timestamp from PD, without block the caller. - GetTSAsync(ctx context.Context) TSFuture - // GetLocalTS gets a local timestamp from PD. - GetLocalTS(ctx context.Context, dcLocation string) (int64, int64, error) - // GetLocalTSAsync gets a local timestamp from PD, without block the caller. - GetLocalTSAsync(ctx context.Context, dcLocation string) TSFuture // GetRegion gets a region and its leader Peer from PD by key. // The region may expire after split. Caller is responsible for caching and // taking care of region change. @@ -137,10 +146,14 @@ type Client interface { // SetExternalTimestamp sets external timestamp SetExternalTimestamp(ctx context.Context, timestamp uint64) error - // MetaStorageClient returns the meta storage client. + // TSOClient is the TSO client. + TSOClient + // MetaStorageClient is the meta storage client. MetaStorageClient // KeyspaceClient manages keyspace metadata. KeyspaceClient + // GCClient manages gcSafePointV2 and serviceSafePointV2 + GCClient // ResourceManagerClient manages resource group metadata and token assignment. ResourceManagerClient // Close closes the client. @@ -162,8 +175,9 @@ func WithExcludeTombstone() GetStoreOption { // RegionsOp represents available options when operate regions type RegionsOp struct { - group string - retryLimit uint64 + group string + retryLimit uint64 + skipStoreLimit bool } // RegionsOption configures RegionsOp @@ -179,6 +193,11 @@ func WithRetry(retry uint64) RegionsOption { return func(op *RegionsOp) { op.retryLimit = retry } } +// WithSkipStoreLimit specify if skip the store limit check during Scatter/Split Regions +func WithSkipStoreLimit() RegionsOption { + return func(op *RegionsOp) { op.skipStoreLimit = true } +} + // GetRegionOp represents available options when getting regions. type GetRegionOp struct { needBuckets bool @@ -204,6 +223,8 @@ var ( errClosing = errors.New("[pd] closing") // errTSOLength is returned when the number of response timestamps is inconsistent with request. errTSOLength = errors.New("[pd] tso length in rpc response is incorrect") + // errInvalidRespHeader is returned when the response doesn't contain service mode info unexpectedly. + errNoServiceModeReturned = errors.New("[pd] no service mode returned") ) // ClientOption configures client. @@ -237,15 +258,64 @@ func WithMaxErrorRetry(count int) ClientOption { } } +// WithMetricsLabels configures the client with metrics labels. +func WithMetricsLabels(labels prometheus.Labels) ClientOption { + return func(c *client) { + c.option.metricsLabels = labels + } +} + +// WithInitMetricsOption configures the client with metrics labels. +func WithInitMetricsOption(initMetrics bool) ClientOption { + return func(c *client) { + c.option.initMetrics = initMetrics + } +} + +// WithAllowTSOFallback configures the client with `allowTSOFallback` option. +// NOTICE: This should only be used for testing. +func WithAllowTSOFallback() ClientOption { + return func(c *client) { + c.option.allowTSOFallback = true + } +} + var _ Client = (*client)(nil) -type client struct { - keyspaceID uint32 - // svcDiscovery is for pd service discovery - svcDiscovery ServiceDiscovery +// serviceModeKeeper is for service mode switching. +type serviceModeKeeper struct { + // RMutex here is for the future usage that there might be multiple goroutines + // triggering service mode switching concurrently. + sync.RWMutex + serviceMode pdpb.ServiceMode tsoClient *tsoClient + tsoSvcDiscovery ServiceDiscovery +} + +func (k *serviceModeKeeper) close() { + k.Lock() + defer k.Unlock() + switch k.serviceMode { + case pdpb.ServiceMode_API_SVC_MODE: + k.tsoSvcDiscovery.Close() + fallthrough + case pdpb.ServiceMode_PD_SVC_MODE: + if k.tsoClient != nil { + k.tsoClient.Close() + } + case pdpb.ServiceMode_UNKNOWN_SVC_MODE: + } +} + +type client struct { + keyspaceID uint32 + svrUrls []string + pdSvcDiscovery ServiceDiscovery tokenDispatcher *tokenDispatcher + // For service mode switching. + serviceModeKeeper + // For internal usage. updateTokenConnectionCh chan struct{} leaderNetworkFailure int32 @@ -253,6 +323,7 @@ type client struct { ctx context.Context cancel context.CancelFunc wg sync.WaitGroup + tlsCfg *tlsutil.TLSConfig option *option } @@ -268,60 +339,153 @@ type SecurityOption struct { } // NewClient creates a PD client. -func NewClient(svrAddrs []string, security SecurityOption, opts ...ClientOption) (Client, error) { +func NewClient( + svrAddrs []string, security SecurityOption, opts ...ClientOption, +) (Client, error) { return NewClientWithContext(context.Background(), svrAddrs, security, opts...) } -// NewClientWithContext creates a PD client with context. -func NewClientWithContext(ctx context.Context, svrAddrs []string, security SecurityOption, opts ...ClientOption) (Client, error) { - log.Info("[pd] create pd client with endpoints", zap.Strings("pd-address", svrAddrs)) - c, clientCtx, clientCancel, tlsCfg := createClient(ctx, 0, &security) - // Inject the client options. - for _, opt := range opts { - opt(c) +// NewClientWithContext creates a PD client with context. This API uses the default keyspace id 0. +func NewClientWithContext( + ctx context.Context, svrAddrs []string, + security SecurityOption, opts ...ClientOption, +) (Client, error) { + return createClientWithKeyspace(ctx, nullKeyspaceID, svrAddrs, security, opts...) +} + +// NewClientWithKeyspace creates a client with context and the specified keyspace id. +// And now, it's only for test purpose. +func NewClientWithKeyspace( + ctx context.Context, keyspaceID uint32, svrAddrs []string, + security SecurityOption, opts ...ClientOption, +) (Client, error) { + if keyspaceID < defaultKeyspaceID || keyspaceID > maxKeyspaceID { + return nil, errors.Errorf("invalid keyspace id %d. It must be in the range of [%d, %d]", + keyspaceID, defaultKeyspaceID, maxKeyspaceID) } + return createClientWithKeyspace(ctx, keyspaceID, svrAddrs, security, opts...) +} - c.svcDiscovery = newPDServiceDiscovery(clientCtx, clientCancel, &c.wg, addrsToUrls(svrAddrs), tlsCfg, c.option) - c.tsoClient = newTSOClient(clientCtx, clientCancel, &c.wg, c.option, c.keyspaceID, c.svcDiscovery, c.svcDiscovery.(tsoAllocatorEventSource), &pdTSOStreamBuilderFactory{}) - if err := c.setup(); err != nil { - c.cancel() - return nil, err +// createClientWithKeyspace creates a client with context and the specified keyspace id. +func createClientWithKeyspace( + ctx context.Context, keyspaceID uint32, svrAddrs []string, + security SecurityOption, opts ...ClientOption, +) (Client, error) { + tlsCfg := &tlsutil.TLSConfig{ + CAPath: security.CAPath, + CertPath: security.CertPath, + KeyPath: security.KeyPath, + + SSLCABytes: security.SSLCABytes, + SSLCertBytes: security.SSLCertBytes, + SSLKEYBytes: security.SSLKEYBytes, } - if err := c.tsoClient.setup(); err != nil { - c.cancel() - return nil, err + + clientCtx, clientCancel := context.WithCancel(ctx) + c := &client{ + updateTokenConnectionCh: make(chan struct{}, 1), + ctx: clientCtx, + cancel: clientCancel, + keyspaceID: keyspaceID, + svrUrls: addrsToUrls(svrAddrs), + tlsCfg: tlsCfg, + option: newOption(), } - return c, nil -} -// NewTSOClientWithContext creates a TSO client with context. -// TODO: -// Merge NewClientWithContext with this API after we let client detect service mode provided on the server side. -// Before that, internal tools call this function to use mcs service. -func NewTSOClientWithContext(ctx context.Context, keyspaceID uint32, svrAddrs []string, security SecurityOption, opts ...ClientOption) (Client, error) { - log.Info("[tso] create tso client with endpoints", zap.Strings("pd(api)-address", svrAddrs)) - c, clientCtx, clientCancel, tlsCfg := createClient(ctx, keyspaceID, &security) // Inject the client options. for _, opt := range opts { opt(c) } - c.svcDiscovery = newPDServiceDiscovery(clientCtx, clientCancel, &c.wg, addrsToUrls(svrAddrs), tlsCfg, c.option) + c.pdSvcDiscovery = newPDServiceDiscovery( + clientCtx, clientCancel, &c.wg, c.setServiceMode, + nil, keyspaceID, c.svrUrls, c.tlsCfg, c.option) if err := c.setup(); err != nil { c.cancel() return nil, err } - tsoSvcDiscovery := newTSOServiceDiscovery(clientCtx, clientCancel, &c.wg, MetaStorageClient(c), c.GetClusterID(c.ctx), keyspaceID, addrsToUrls(svrAddrs), tlsCfg, c.option) - c.tsoClient = newTSOClient(clientCtx, clientCancel, &c.wg, c.option, c.keyspaceID, tsoSvcDiscovery, tsoSvcDiscovery.(tsoAllocatorEventSource), &tsoTSOStreamBuilderFactory{}) - if err := c.tsoClient.setup(); err != nil { - c.cancel() - return nil, err - } return c, nil } -func createClient(ctx context.Context, keyspaceID uint32, security *SecurityOption) (*client, context.Context, context.CancelFunc, *tlsutil.TLSConfig) { +// APIVersion is the API version the server and the client is using. +// See more details in https://github.com/tikv/rfcs/blob/master/text/0069-api-v2.md#kvproto +type APIVersion int + +// The API versions the client supports. +// As for V1TTL, client won't use it and we just remove it. +const ( + V1 APIVersion = iota + _ + V2 +) + +// APIContext is the context for API version. +type APIContext interface { + GetAPIVersion() (apiVersion APIVersion) + GetKeyspaceName() (keyspaceName string) +} + +type apiContextV1 struct{} + +// NewAPIContextV1 creates a API context for V1. +func NewAPIContextV1() APIContext { + return &apiContextV1{} +} + +// GetAPIVersion returns the API version. +func (apiCtx *apiContextV1) GetAPIVersion() (version APIVersion) { + return V1 +} + +// GetKeyspaceName returns the keyspace name. +func (apiCtx *apiContextV1) GetKeyspaceName() (keyspaceName string) { + return "" +} + +type apiContextV2 struct { + keyspaceName string +} + +// NewAPIContextV2 creates a API context with the specified keyspace name for V2. +func NewAPIContextV2(keyspaceName string) APIContext { + if len(keyspaceName) == 0 { + keyspaceName = defaultKeyspaceName + } + return &apiContextV2{keyspaceName: keyspaceName} +} + +// GetAPIVersion returns the API version. +func (apiCtx *apiContextV2) GetAPIVersion() (version APIVersion) { + return V2 +} + +// GetKeyspaceName returns the keyspace name. +func (apiCtx *apiContextV2) GetKeyspaceName() (keyspaceName string) { + return apiCtx.keyspaceName +} + +// NewClientWithAPIContext creates a client according to the API context. +func NewClientWithAPIContext( + ctx context.Context, apiCtx APIContext, svrAddrs []string, + security SecurityOption, opts ...ClientOption, +) (Client, error) { + apiVersion, keyspaceName := apiCtx.GetAPIVersion(), apiCtx.GetKeyspaceName() + switch apiVersion { + case V1: + return NewClientWithContext(ctx, svrAddrs, security, opts...) + case V2: + return newClientWithKeyspaceName(ctx, keyspaceName, svrAddrs, security, opts...) + default: + return nil, errors.Errorf("[pd] invalid API version %d", apiVersion) + } +} + +// newClientWithKeyspaceName creates a client with context and the specified keyspace name. +func newClientWithKeyspaceName( + ctx context.Context, keyspaceName string, svrAddrs []string, + security SecurityOption, opts ...ClientOption, +) (Client, error) { tlsCfg := &tlsutil.TLSConfig{ CAPath: security.CAPath, CertPath: security.CertPath, @@ -337,21 +501,79 @@ func createClient(ctx context.Context, keyspaceID uint32, security *SecurityOpti updateTokenConnectionCh: make(chan struct{}, 1), ctx: clientCtx, cancel: clientCancel, - keyspaceID: keyspaceID, + svrUrls: addrsToUrls(svrAddrs), + tlsCfg: tlsCfg, option: newOption(), } - return c, clientCtx, clientCancel, tlsCfg + // Inject the client options. + for _, opt := range opts { + opt(c) + } + + updateKeyspaceIDCb := func() error { + if err := c.initRetry(c.loadKeyspaceMeta, keyspaceName); err != nil { + return err + } + // c.keyspaceID is the source of truth for keyspace id. + c.pdSvcDiscovery.(*pdServiceDiscovery).SetKeyspaceID(c.keyspaceID) + return nil + } + + // Create a PD service discovery with null keyspace id, then query the real id wth the keyspace name, + // finally update the keyspace id to the PD service discovery for the following interactions. + c.pdSvcDiscovery = newPDServiceDiscovery( + clientCtx, clientCancel, &c.wg, c.setServiceMode, updateKeyspaceIDCb, nullKeyspaceID, c.svrUrls, c.tlsCfg, c.option) + if err := c.setup(); err != nil { + c.cancel() + return nil, err + } + log.Info("[pd] create pd client with endpoints and keyspace", + zap.Strings("pd-address", svrAddrs), + zap.String("keyspace-name", keyspaceName), + zap.Uint32("keyspace-id", c.keyspaceID)) + return c, nil +} + +func (c *client) initRetry(f func(s string) error, str string) error { + var err error + ticker := time.NewTicker(time.Second) + defer ticker.Stop() + for i := 0; i < c.option.maxRetryTimes; i++ { + if err = f(str); err == nil { + return nil + } + select { + case <-c.ctx.Done(): + return err + case <-ticker.C: + } + } + return errors.WithStack(err) +} + +func (c *client) loadKeyspaceMeta(keyspace string) error { + keyspaceMeta, err := c.LoadKeyspace(context.TODO(), keyspace) + if err != nil { + return err + } + c.keyspaceID = keyspaceMeta.GetId() + return nil } func (c *client) setup() error { + // Init the metrics. + if c.option.initMetrics { + initAndRegisterMetrics(c.option.metricsLabels) + } + // Init the client base. - if err := c.svcDiscovery.Init(); err != nil { + if err := c.pdSvcDiscovery.Init(); err != nil { return err } // Register callbacks - c.svcDiscovery.AddServingAddrSwitchedCallback(c.scheduleUpdateTokenConnection) + c.pdSvcDiscovery.AddServingAddrSwitchedCallback(c.scheduleUpdateTokenConnection) // Create dispatchers c.createTokenDispatcher() @@ -366,8 +588,8 @@ func (c *client) Close() { c.cancel() c.wg.Wait() - c.tsoClient.Close() - c.svcDiscovery.Close() + c.serviceModeKeeper.close() + c.pdSvcDiscovery.Close() if c.tokenDispatcher != nil { tokenErr := errors.WithStack(errClosing) @@ -376,6 +598,78 @@ func (c *client) Close() { } } +func (c *client) setServiceMode(newMode pdpb.ServiceMode) { + c.Lock() + defer c.Unlock() + + if newMode == c.serviceMode { + return + } + log.Info("[pd] changing service mode", + zap.String("old-mode", c.serviceMode.String()), + zap.String("new-mode", newMode.String())) + // Re-create a new TSO client. + var ( + newTSOCli *tsoClient + newTSOSvcDiscovery ServiceDiscovery + ) + switch newMode { + case pdpb.ServiceMode_PD_SVC_MODE: + newTSOCli = newTSOClient(c.ctx, c.option, + c.pdSvcDiscovery, &pdTSOStreamBuilderFactory{}) + case pdpb.ServiceMode_API_SVC_MODE: + newTSOSvcDiscovery = newTSOServiceDiscovery( + c.ctx, MetaStorageClient(c), c.pdSvcDiscovery, + c.GetClusterID(c.ctx), c.keyspaceID, c.tlsCfg, c.option) + // At this point, the keyspace group isn't known yet. Starts from the default keyspace group, + // and will be updated later. + newTSOCli = newTSOClient(c.ctx, c.option, + newTSOSvcDiscovery, &tsoTSOStreamBuilderFactory{}) + if err := newTSOSvcDiscovery.Init(); err != nil { + log.Error("[pd] failed to initialize tso service discovery. keep the current service mode", + zap.Strings("svr-urls", c.svrUrls), + zap.String("current-mode", c.serviceMode.String()), + zap.Error(err)) + return + } + case pdpb.ServiceMode_UNKNOWN_SVC_MODE: + log.Warn("[pd] intend to switch to unknown service mode, just return") + return + } + newTSOCli.Setup() + // Replace the old TSO client. + oldTSOClient := c.tsoClient + c.tsoClient = newTSOCli + oldTSOClient.Close() + // Replace the old TSO service discovery if needed. + oldTSOSvcDiscovery := c.tsoSvcDiscovery + // If newTSOSvcDiscovery is nil, that's expected, as it means we are switching to PD service mode and + // no tso microservice discovery is needed. + c.tsoSvcDiscovery = newTSOSvcDiscovery + // Close the old TSO service discovery safely after both the old client and service discovery are replaced. + if oldTSOSvcDiscovery != nil { + // We are switching from API service mode to PD service mode, so delete the old tso microservice discovery. + oldTSOSvcDiscovery.Close() + } + oldMode := c.serviceMode + c.serviceMode = newMode + log.Info("[pd] service mode changed", + zap.String("old-mode", oldMode.String()), + zap.String("new-mode", newMode.String())) +} + +func (c *client) getTSOClient() *tsoClient { + c.RLock() + defer c.RUnlock() + return c.tsoClient +} + +func (c *client) getServiceMode() pdpb.ServiceMode { + c.RLock() + defer c.RUnlock() + return c.serviceMode +} + func (c *client) scheduleUpdateTokenConnection() { select { case c.updateTokenConnectionCh <- struct{}{}: @@ -385,17 +679,17 @@ func (c *client) scheduleUpdateTokenConnection() { // GetClusterID returns the ClusterID. func (c *client) GetClusterID(context.Context) uint64 { - return c.svcDiscovery.GetClusterID() + return c.pdSvcDiscovery.GetClusterID() } // GetLeaderAddr returns the leader address. func (c *client) GetLeaderAddr() string { - return c.svcDiscovery.GetServingAddr() + return c.pdSvcDiscovery.GetServingAddr() } // GetServiceDiscovery returns the client-side service discovery object func (c *client) GetServiceDiscovery() ServiceDiscovery { - return c.svcDiscovery + return c.pdSvcDiscovery } // UpdateOption updates the client option. @@ -443,7 +737,7 @@ func (c *client) leaderCheckLoop() { func (c *client) checkLeaderHealth(ctx context.Context) { ctx, cancel := context.WithTimeout(ctx, c.option.timeout) defer cancel() - if client := c.svcDiscovery.GetServingEndpointClientConn(); client != nil { + if client := c.pdSvcDiscovery.GetServingEndpointClientConn(); client != nil { healthCli := healthpb.NewHealthClient(client) resp, err := healthCli.Check(ctx, &healthpb.HealthCheckRequest{Service: ""}) rpcErr, ok := status.FromError(err) @@ -466,7 +760,12 @@ func (c *client) GetAllMembers(ctx context.Context) ([]*pdpb.Member, error) { ctx, cancel := context.WithTimeout(ctx, c.option.timeout) req := &pdpb.GetMembersRequest{Header: c.requestHeader()} ctx = grpcutil.BuildForwardContext(ctx, c.GetLeaderAddr()) - resp, err := c.getClient().GetMembers(ctx, req) + protoClient := c.getClient() + if protoClient == nil { + cancel() + return nil, errs.ErrClientGetProtoClient + } + resp, err := protoClient.GetMembers(ctx, req) cancel() if err = c.respForErr(cmdFailDurationGetAllMembers, start, err, resp.GetHeader()); err != nil { return nil, err @@ -476,7 +775,7 @@ func (c *client) GetAllMembers(ctx context.Context) ([]*pdpb.Member, error) { // leaderClient gets the client of current PD leader. func (c *client) leaderClient() pdpb.PDClient { - if client := c.svcDiscovery.GetServingEndpointClientConn(); client != nil { + if client := c.pdSvcDiscovery.GetServingEndpointClientConn(); client != nil { return pdpb.NewPDClient(client) } return nil @@ -486,7 +785,7 @@ func (c *client) leaderClient() pdpb.PDClient { // backup service endpoints randomly. Backup service endpoints are followers in a // quorum-based cluster or secondaries in a primary/secondary configured cluster. func (c *client) backupClientConn() (*grpc.ClientConn, string) { - addrs := c.svcDiscovery.GetBackupAddrs() + addrs := c.pdSvcDiscovery.GetBackupAddrs() if len(addrs) < 1 { return nil, "" } @@ -496,7 +795,7 @@ func (c *client) backupClientConn() (*grpc.ClientConn, string) { ) for i := 0; i < len(addrs); i++ { addr := addrs[rand.Intn(len(addrs))] - if cc, err = c.svcDiscovery.GetOrCreateGRPCConn(addr); err != nil { + if cc, err = c.pdSvcDiscovery.GetOrCreateGRPCConn(addr); err != nil { continue } healthCtx, healthCancel := context.WithTimeout(c.ctx, c.option.timeout) @@ -525,6 +824,7 @@ func (c *client) GetTSAsync(ctx context.Context) TSFuture { } func (c *client) GetLocalTSAsync(ctx context.Context, dcLocation string) TSFuture { + defer trace.StartRegion(ctx, "GetLocalTSAsync").End() if span := opentracing.SpanFromContext(ctx); span != nil { span = opentracing.StartSpan("GetLocalTSAsync", opentracing.ChildOf(span.Context())) ctx = opentracing.ContextWithSpan(ctx, span) @@ -533,14 +833,19 @@ func (c *client) GetLocalTSAsync(ctx context.Context, dcLocation string) TSFutur req := tsoReqPool.Get().(*tsoRequest) req.requestCtx = ctx req.clientCtx = c.ctx + tsoClient := c.getTSOClient() req.start = time.Now() - req.keyspaceID = c.keyspaceID req.dcLocation = dcLocation - if err := c.tsoClient.dispatchRequest(dcLocation, req); err != nil { + if tsoClient == nil { + req.done <- errs.ErrClientGetTSO.FastGenByArgs("tso client is nil") + return req + } + + if err := tsoClient.dispatchRequest(dcLocation, req); err != nil { // Wait for a while and try again time.Sleep(50 * time.Millisecond) - if err = c.tsoClient.dispatchRequest(dcLocation, req); err != nil { + if err = tsoClient.dispatchRequest(dcLocation, req); err != nil { req.done <- err } } @@ -557,6 +862,50 @@ func (c *client) GetLocalTS(ctx context.Context, dcLocation string) (physical in return resp.Wait() } +func (c *client) GetMinTS(ctx context.Context) (physical int64, logical int64, err error) { + // Handle compatibility issue in case of PD/API server doesn't support GetMinTS API. + serviceMode := c.getServiceMode() + switch serviceMode { + case pdpb.ServiceMode_UNKNOWN_SVC_MODE: + return 0, 0, errs.ErrClientGetMinTSO.FastGenByArgs("unknown service mode") + case pdpb.ServiceMode_PD_SVC_MODE: + // If the service mode is switched to API during GetTS() call, which happens during migration, + // returning the default timeline should be fine. + return c.GetTS(ctx) + case pdpb.ServiceMode_API_SVC_MODE: + default: + return 0, 0, errs.ErrClientGetMinTSO.FastGenByArgs("undefined service mode") + } + + // Call GetMinTS API to get the minimal TS from the API leader. + protoClient := c.getClient() + if protoClient == nil { + return 0, 0, errs.ErrClientGetProtoClient + } + + resp, err := protoClient.GetMinTS(ctx, &pdpb.GetMinTSRequest{ + Header: c.requestHeader(), + }) + if err != nil { + if strings.Contains(err.Error(), "Unimplemented") { + // If the method is not supported, we fallback to GetTS. + return c.GetTS(ctx) + } + return 0, 0, errs.ErrClientGetMinTSO.Wrap(err).GenWithStackByCause() + } + if resp == nil { + attachErr := errors.Errorf("error:%s", "no min ts info collected") + return 0, 0, errs.ErrClientGetMinTSO.Wrap(attachErr).GenWithStackByCause() + } + if resp.GetHeader().GetError() != nil { + attachErr := errors.Errorf("error:%s s", resp.GetHeader().GetError().String()) + return 0, 0, errs.ErrClientGetMinTSO.Wrap(attachErr).GenWithStackByCause() + } + + minTS := resp.GetTimestamp() + return minTS.Physical, tsoutil.AddLogical(minTS.Logical, 0, minTS.SuffixBits), nil +} + func handleRegionResponse(res *pdpb.GetRegionResponse) *Region { if res.Region == nil { return nil @@ -593,7 +942,12 @@ func (c *client) GetRegion(ctx context.Context, key []byte, opts ...GetRegionOpt NeedBuckets: options.needBuckets, } ctx = grpcutil.BuildForwardContext(ctx, c.GetLeaderAddr()) - resp, err := c.getClient().GetRegion(ctx, req) + protoClient := c.getClient() + if protoClient == nil { + cancel() + return nil, errs.ErrClientGetProtoClient + } + resp, err := protoClient.GetRegion(ctx, req) cancel() if err = c.respForErr(cmdFailDurationGetRegion, start, err, resp.GetHeader()); err != nil { @@ -616,7 +970,7 @@ func (c *client) GetRegionFromMember(ctx context.Context, key []byte, memberURLs var resp *pdpb.GetRegionResponse for _, url := range memberURLs { - conn, err := c.svcDiscovery.GetOrCreateGRPCConn(url) + conn, err := c.pdSvcDiscovery.GetOrCreateGRPCConn(url) if err != nil { log.Error("[pd] can't get grpc connection", zap.String("member-URL", url), errs.ZapError(err)) continue @@ -637,7 +991,7 @@ func (c *client) GetRegionFromMember(ctx context.Context, key []byte, memberURLs if resp == nil { cmdFailDurationGetRegion.Observe(time.Since(start).Seconds()) - c.svcDiscovery.ScheduleCheckMemberChanged() + c.pdSvcDiscovery.ScheduleCheckMemberChanged() errorMsg := fmt.Sprintf("[pd] can't get region info from member URLs: %+v", memberURLs) return nil, errors.WithStack(errors.New(errorMsg)) } @@ -663,7 +1017,12 @@ func (c *client) GetPrevRegion(ctx context.Context, key []byte, opts ...GetRegio NeedBuckets: options.needBuckets, } ctx = grpcutil.BuildForwardContext(ctx, c.GetLeaderAddr()) - resp, err := c.getClient().GetPrevRegion(ctx, req) + protoClient := c.getClient() + if protoClient == nil { + cancel() + return nil, errs.ErrClientGetProtoClient + } + resp, err := protoClient.GetPrevRegion(ctx, req) cancel() if err = c.respForErr(cmdFailDurationGetPrevRegion, start, err, resp.GetHeader()); err != nil { @@ -691,7 +1050,12 @@ func (c *client) GetRegionByID(ctx context.Context, regionID uint64, opts ...Get NeedBuckets: options.needBuckets, } ctx = grpcutil.BuildForwardContext(ctx, c.GetLeaderAddr()) - resp, err := c.getClient().GetRegionByID(ctx, req) + protoClient := c.getClient() + if protoClient == nil { + cancel() + return nil, errs.ErrClientGetProtoClient + } + resp, err := protoClient.GetRegionByID(ctx, req) cancel() if err = c.respForErr(cmdFailedDurationGetRegionByID, start, err, resp.GetHeader()); err != nil { @@ -721,7 +1085,12 @@ func (c *client) ScanRegions(ctx context.Context, key, endKey []byte, limit int) Limit: int32(limit), } scanCtx = grpcutil.BuildForwardContext(scanCtx, c.GetLeaderAddr()) - resp, err := c.getClient().ScanRegions(scanCtx, req) + protoClient := c.getClient() + if protoClient == nil { + cancel() + return nil, errs.ErrClientGetProtoClient + } + resp, err := protoClient.ScanRegions(scanCtx, req) if err = c.respForErr(cmdFailedDurationScanRegions, start, err, resp.GetHeader()); err != nil { return nil, err @@ -772,7 +1141,12 @@ func (c *client) GetStore(ctx context.Context, storeID uint64) (*metapb.Store, e StoreId: storeID, } ctx = grpcutil.BuildForwardContext(ctx, c.GetLeaderAddr()) - resp, err := c.getClient().GetStore(ctx, req) + protoClient := c.getClient() + if protoClient == nil { + cancel() + return nil, errs.ErrClientGetProtoClient + } + resp, err := protoClient.GetStore(ctx, req) cancel() if err = c.respForErr(cmdFailedDurationGetStore, start, err, resp.GetHeader()); err != nil { @@ -812,7 +1186,12 @@ func (c *client) GetAllStores(ctx context.Context, opts ...GetStoreOption) ([]*m ExcludeTombstoneStores: options.excludeTombstone, } ctx = grpcutil.BuildForwardContext(ctx, c.GetLeaderAddr()) - resp, err := c.getClient().GetAllStores(ctx, req) + protoClient := c.getClient() + if protoClient == nil { + cancel() + return nil, errs.ErrClientGetProtoClient + } + resp, err := protoClient.GetAllStores(ctx, req) cancel() if err = c.respForErr(cmdFailedDurationGetAllStores, start, err, resp.GetHeader()); err != nil { @@ -835,7 +1214,12 @@ func (c *client) UpdateGCSafePoint(ctx context.Context, safePoint uint64) (uint6 SafePoint: safePoint, } ctx = grpcutil.BuildForwardContext(ctx, c.GetLeaderAddr()) - resp, err := c.getClient().UpdateGCSafePoint(ctx, req) + protoClient := c.getClient() + if protoClient == nil { + cancel() + return 0, errs.ErrClientGetProtoClient + } + resp, err := protoClient.UpdateGCSafePoint(ctx, req) cancel() if err = c.respForErr(cmdFailedDurationUpdateGCSafePoint, start, err, resp.GetHeader()); err != nil { @@ -865,7 +1249,12 @@ func (c *client) UpdateServiceGCSafePoint(ctx context.Context, serviceID string, SafePoint: safePoint, } ctx = grpcutil.BuildForwardContext(ctx, c.GetLeaderAddr()) - resp, err := c.getClient().UpdateServiceGCSafePoint(ctx, req) + protoClient := c.getClient() + if protoClient == nil { + cancel() + return 0, errs.ErrClientGetProtoClient + } + resp, err := protoClient.UpdateServiceGCSafePoint(ctx, req) cancel() if err = c.respForErr(cmdFailedDurationUpdateServiceGCSafePoint, start, err, resp.GetHeader()); err != nil { @@ -893,7 +1282,12 @@ func (c *client) scatterRegionsWithGroup(ctx context.Context, regionID uint64, g Group: group, } ctx = grpcutil.BuildForwardContext(ctx, c.GetLeaderAddr()) - resp, err := c.getClient().ScatterRegion(ctx, req) + protoClient := c.getClient() + if protoClient == nil { + cancel() + return errs.ErrClientGetProtoClient + } + resp, err := protoClient.ScatterRegion(ctx, req) cancel() if err != nil { return err @@ -933,7 +1327,12 @@ func (c *client) SplitAndScatterRegions(ctx context.Context, splitKeys [][]byte, } ctx = grpcutil.BuildForwardContext(ctx, c.GetLeaderAddr()) - return c.getClient().SplitAndScatterRegions(ctx, req) + protoClient := c.getClient() + if protoClient == nil { + cancel() + return nil, errs.ErrClientGetProtoClient + } + return protoClient.SplitAndScatterRegions(ctx, req) } func (c *client) GetOperator(ctx context.Context, regionID uint64) (*pdpb.GetOperatorResponse, error) { @@ -951,7 +1350,12 @@ func (c *client) GetOperator(ctx context.Context, regionID uint64) (*pdpb.GetOpe RegionId: regionID, } ctx = grpcutil.BuildForwardContext(ctx, c.GetLeaderAddr()) - return c.getClient().GetOperator(ctx, req) + protoClient := c.getClient() + if protoClient == nil { + cancel() + return nil, errs.ErrClientGetProtoClient + } + return protoClient.GetOperator(ctx, req) } // SplitRegions split regions by given split keys @@ -974,12 +1378,17 @@ func (c *client) SplitRegions(ctx context.Context, splitKeys [][]byte, opts ...R RetryLimit: options.retryLimit, } ctx = grpcutil.BuildForwardContext(ctx, c.GetLeaderAddr()) - return c.getClient().SplitRegions(ctx, req) + protoClient := c.getClient() + if protoClient == nil { + cancel() + return nil, errs.ErrClientGetProtoClient + } + return protoClient.SplitRegions(ctx, req) } func (c *client) requestHeader() *pdpb.RequestHeader { return &pdpb.RequestHeader{ - ClusterId: c.svcDiscovery.GetClusterID(), + ClusterId: c.pdSvcDiscovery.GetClusterID(), } } @@ -992,14 +1401,20 @@ func (c *client) scatterRegionsWithOptions(ctx context.Context, regionsID []uint } ctx, cancel := context.WithTimeout(ctx, c.option.timeout) req := &pdpb.ScatterRegionRequest{ - Header: c.requestHeader(), - Group: options.group, - RegionsId: regionsID, - RetryLimit: options.retryLimit, + Header: c.requestHeader(), + Group: options.group, + RegionsId: regionsID, + RetryLimit: options.retryLimit, + SkipStoreLimit: options.skipStoreLimit, } ctx = grpcutil.BuildForwardContext(ctx, c.GetLeaderAddr()) - resp, err := c.getClient().ScatterRegion(ctx, req) + protoClient := c.getClient() + if protoClient == nil { + cancel() + return nil, errs.ErrClientGetProtoClient + } + resp, err := protoClient.ScatterRegion(ctx, req) cancel() if err != nil { @@ -1026,8 +1441,13 @@ func addrsToUrls(addrs []string) []string { // IsLeaderChange will determine whether there is a leader change. func IsLeaderChange(err error) bool { + if err == errs.ErrClientTSOStreamClosed { + return true + } errMsg := err.Error() - return strings.Contains(errMsg, errs.NotLeaderErr) || strings.Contains(errMsg, errs.MismatchLeaderErr) + return strings.Contains(errMsg, errs.NotLeaderErr) || + strings.Contains(errMsg, errs.MismatchLeaderErr) || + strings.Contains(errMsg, errs.NotServedErr) } func trimHTTPPrefix(str string) string { @@ -1037,7 +1457,11 @@ func trimHTTPPrefix(str string) string { } func (c *client) LoadGlobalConfig(ctx context.Context, names []string, configPath string) ([]GlobalConfigItem, int64, error) { - resp, err := c.getClient().LoadGlobalConfig(ctx, &pdpb.LoadGlobalConfigRequest{Names: names, ConfigPath: configPath}) + protoClient := c.getClient() + if protoClient == nil { + return nil, 0, errs.ErrClientGetProtoClient + } + resp, err := protoClient.LoadGlobalConfig(ctx, &pdpb.LoadGlobalConfigRequest{Names: names, ConfigPath: configPath}) if err != nil { return nil, 0, err } @@ -1062,7 +1486,11 @@ func (c *client) StoreGlobalConfig(ctx context.Context, configPath string, items for i, it := range items { resArr[i] = &pdpb.GlobalConfigItem{Name: it.Name, Value: it.Value, Kind: it.EventType, Payload: it.PayLoad} } - _, err := c.getClient().StoreGlobalConfig(ctx, &pdpb.StoreGlobalConfigRequest{Changes: resArr, ConfigPath: configPath}) + protoClient := c.getClient() + if protoClient == nil { + return errs.ErrClientGetProtoClient + } + _, err := protoClient.StoreGlobalConfig(ctx, &pdpb.StoreGlobalConfigRequest{Changes: resArr, ConfigPath: configPath}) if err != nil { return err } @@ -1073,7 +1501,11 @@ func (c *client) WatchGlobalConfig(ctx context.Context, configPath string, revis // TODO: Add retry mechanism // register watch components there globalConfigWatcherCh := make(chan []GlobalConfigItem, 16) - res, err := c.getClient().WatchGlobalConfig(ctx, &pdpb.WatchGlobalConfigRequest{ + protoClient := c.getClient() + if protoClient == nil { + return nil, errs.ErrClientGetProtoClient + } + res, err := protoClient.WatchGlobalConfig(ctx, &pdpb.WatchGlobalConfigRequest{ ConfigPath: configPath, Revision: revision, }) @@ -1115,7 +1547,11 @@ func (c *client) WatchGlobalConfig(ctx context.Context, configPath string, revis } func (c *client) GetExternalTimestamp(ctx context.Context) (uint64, error) { - resp, err := c.getClient().GetExternalTimestamp(ctx, &pdpb.GetExternalTimestampRequest{ + protoClient := c.getClient() + if protoClient == nil { + return 0, errs.ErrClientGetProtoClient + } + resp, err := protoClient.GetExternalTimestamp(ctx, &pdpb.GetExternalTimestampRequest{ Header: c.requestHeader(), }) if err != nil { @@ -1129,7 +1565,11 @@ func (c *client) GetExternalTimestamp(ctx context.Context) (uint64, error) { } func (c *client) SetExternalTimestamp(ctx context.Context, timestamp uint64) error { - resp, err := c.getClient().SetExternalTimestamp(ctx, &pdpb.SetExternalTimestampRequest{ + protoClient := c.getClient() + if protoClient == nil { + return errs.ErrClientGetProtoClient + } + resp, err := protoClient.SetExternalTimestamp(ctx, &pdpb.SetExternalTimestampRequest{ Header: c.requestHeader(), Timestamp: timestamp, }) @@ -1147,7 +1587,7 @@ func (c *client) respForErr(observer prometheus.Observer, start time.Time, err e if err != nil || header.GetError() != nil { observer.Observe(time.Since(start).Seconds()) if err != nil { - c.svcDiscovery.ScheduleCheckMemberChanged() + c.pdSvcDiscovery.ScheduleCheckMemberChanged() return errors.WithStack(err) } return errors.WithStack(errors.New(header.GetError().String())) @@ -1158,5 +1598,9 @@ func (c *client) respForErr(observer prometheus.Observer, start time.Time, err e // GetTSOAllocators returns {dc-location -> TSO allocator leader URL} connection map // For test only. func (c *client) GetTSOAllocators() *sync.Map { - return c.tsoClient.GetTSOAllocators() + tsoClient := c.getTSOClient() + if tsoClient == nil { + return nil + } + return tsoClient.GetTSOAllocators() } diff --git a/client/client_test.go b/client/client_test.go index 43c0cc5c308..e82fe861a0e 100644 --- a/client/client_test.go +++ b/client/client_test.go @@ -24,6 +24,7 @@ import ( "github.com/stretchr/testify/require" "github.com/tikv/pd/client/testutil" "github.com/tikv/pd/client/tlsutil" + "github.com/tikv/pd/client/tsoutil" "go.uber.org/goleak" "google.golang.org/grpc" ) @@ -32,13 +33,13 @@ func TestMain(m *testing.M) { goleak.VerifyTestMain(m, testutil.LeakOptions...) } -func TestTsLessEqual(t *testing.T) { +func TestTSLessEqual(t *testing.T) { re := require.New(t) - re.True(tsLessEqual(9, 9, 9, 9)) - re.True(tsLessEqual(8, 9, 9, 8)) - re.False(tsLessEqual(9, 8, 8, 9)) - re.False(tsLessEqual(9, 8, 9, 6)) - re.True(tsLessEqual(9, 6, 9, 8)) + re.True(tsoutil.TSLessEqual(9, 9, 9, 9)) + re.True(tsoutil.TSLessEqual(8, 9, 9, 8)) + re.False(tsoutil.TSLessEqual(9, 8, 8, 9)) + re.False(tsoutil.TSLessEqual(9, 8, 9, 6)) + re.True(tsoutil.TSLessEqual(9, 6, 9, 8)) } func TestUpdateURLs(t *testing.T) { @@ -58,11 +59,11 @@ func TestUpdateURLs(t *testing.T) { cli := &pdServiceDiscovery{option: newOption()} cli.urls.Store([]string{}) cli.updateURLs(members[1:]) - re.Equal(getURLs([]*pdpb.Member{members[1], members[3], members[2]}), cli.GetURLs()) + re.Equal(getURLs([]*pdpb.Member{members[1], members[3], members[2]}), cli.GetServiceURLs()) cli.updateURLs(members[1:]) - re.Equal(getURLs([]*pdpb.Member{members[1], members[3], members[2]}), cli.GetURLs()) + re.Equal(getURLs([]*pdpb.Member{members[1], members[3], members[2]}), cli.GetServiceURLs()) cli.updateURLs(members) - re.Equal(getURLs([]*pdpb.Member{members[1], members[3], members[2], members[0]}), cli.GetURLs()) + re.Equal(getURLs([]*pdpb.Member{members[1], members[3], members[2], members[0]}), cli.GetServiceURLs()) } const testClientURL = "tmp://test.url:5255" diff --git a/client/errs/errno.go b/client/errs/errno.go index 21e387536f8..646af81929d 100644 --- a/client/errs/errno.go +++ b/client/errs/errno.go @@ -21,23 +21,40 @@ import ( ) const ( - // NotLeaderErr indicates the the non-leader member received the requests which should be received by leader. + // NotLeaderErr indicates the non-leader member received the requests which should be received by leader. + // Note: keep the same as the ones defined on the server side, because the client side checks if an error message + // contains this string to judge whether the leader is changed. NotLeaderErr = "is not leader" - // MismatchLeaderErr indicates the the non-leader member received the requests which should be received by leader. + // MismatchLeaderErr indicates the non-leader member received the requests which should be received by leader. + // Note: keep the same as the ones defined on the server side, because the client side checks if an error message + // contains this string to judge whether the leader is changed. MismatchLeaderErr = "mismatch leader id" - RetryTimeoutErr = "retry timeout" + // NotServedErr indicates an tso node/pod received the requests for the keyspace groups which are not served by it. + // Note: keep the same as the ones defined on the server side, because the client side checks if an error message + // contains this string to judge whether the leader is changed. + NotServedErr = "is not served" + // RetryTimeoutErr indicates the server is busy. + RetryTimeoutErr = "retry timeout" ) // client errors var ( - ErrClientCreateTSOStream = errors.Normalize("create TSO stream failed, %s", errors.RFCCodeText("PD:client:ErrClientCreateTSOStream")) - ErrClientGetTSOTimeout = errors.Normalize("get TSO timeout", errors.RFCCodeText("PD:client:ErrClientGetTSOTimeout")) - ErrClientGetTSO = errors.Normalize("get TSO failed, %v", errors.RFCCodeText("PD:client:ErrClientGetTSO")) - ErrClientGetLeader = errors.Normalize("get leader from %v error", errors.RFCCodeText("PD:client:ErrClientGetLeader")) - ErrClientGetMember = errors.Normalize("get member failed", errors.RFCCodeText("PD:client:ErrClientGetMember")) - ErrClientUpdateMember = errors.Normalize("update member failed, %v", errors.RFCCodeText("PD:client:ErrUpdateMember")) - ErrClientProtoUnmarshal = errors.Normalize("failed to unmarshal proto", errors.RFCCodeText("PD:proto:ErrClientProtoUnmarshal")) - ErrClientGetMultiResponse = errors.Normalize("get invalid value response %v, must only one", errors.RFCCodeText("PD:client:ErrClientGetMultiResponse")) + ErrClientGetProtoClient = errors.Normalize("failed to get proto client", errors.RFCCodeText("PD:client:ErrClientGetProtoClient")) + ErrClientGetMetaStorageClient = errors.Normalize("failed to get meta storage client", errors.RFCCodeText("PD:client:ErrClientGetMetaStorageClient")) + ErrClientCreateTSOStream = errors.Normalize("create TSO stream failed, %s", errors.RFCCodeText("PD:client:ErrClientCreateTSOStream")) + ErrClientTSOStreamClosed = errors.Normalize("encountered TSO stream being closed unexpectedly", errors.RFCCodeText("PD:client:ErrClientTSOStreamClosed")) + ErrClientGetTSOTimeout = errors.Normalize("get TSO timeout", errors.RFCCodeText("PD:client:ErrClientGetTSOTimeout")) + ErrClientGetTSO = errors.Normalize("get TSO failed, %v", errors.RFCCodeText("PD:client:ErrClientGetTSO")) + ErrClientGetMinTSO = errors.Normalize("get min TSO failed, %v", errors.RFCCodeText("PD:client:ErrClientGetMinTSO")) + ErrClientGetLeader = errors.Normalize("get leader failed, %v", errors.RFCCodeText("PD:client:ErrClientGetLeader")) + ErrClientGetMember = errors.Normalize("get member failed", errors.RFCCodeText("PD:client:ErrClientGetMember")) + ErrClientGetClusterInfo = errors.Normalize("get cluster info failed", errors.RFCCodeText("PD:client:ErrClientGetClusterInfo")) + ErrClientUpdateMember = errors.Normalize("update member failed, %v", errors.RFCCodeText("PD:client:ErrUpdateMember")) + ErrClientProtoUnmarshal = errors.Normalize("failed to unmarshal proto", errors.RFCCodeText("PD:proto:ErrClientProtoUnmarshal")) + ErrClientGetMultiResponse = errors.Normalize("get invalid value response %v, must only one", errors.RFCCodeText("PD:client:ErrClientGetMultiResponse")) + ErrClientGetServingEndpoint = errors.Normalize("get serving endpoint failed", errors.RFCCodeText("PD:client:ErrClientGetServingEndpoint")) + ErrClientFindGroupByKeyspaceID = errors.Normalize("can't find keyspace group by keyspace id", errors.RFCCodeText("PD:client:ErrClientFindGroupByKeyspaceID")) + ErrClientWatchGCSafePointV2Stream = errors.Normalize("watch gc safe point v2 stream failed, %s", errors.RFCCodeText("PD:client:ErrClientWatchGCSafePointV2Stream")) ) // grpcutil errors @@ -75,6 +92,7 @@ var ( ErrClientResourceGroupThrottled = errors.Normalize("exceeded resource group quota limitation", errors.RFCCodeText("PD:client:ErrClientResourceGroupThrottled")) ) +// ErrClientGetResourceGroup is the error type for getting resource group. type ErrClientGetResourceGroup struct { ResourceGroupName string Cause string diff --git a/client/gc_client.go b/client/gc_client.go new file mode 100644 index 00000000000..c573836d2ba --- /dev/null +++ b/client/gc_client.go @@ -0,0 +1,137 @@ +// Copyright 2023 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pd + +import ( + "context" + "time" + + "github.com/opentracing/opentracing-go" + "github.com/pingcap/kvproto/pkg/pdpb" + "github.com/pingcap/log" + "github.com/tikv/pd/client/errs" + "github.com/tikv/pd/client/grpcutil" + "go.uber.org/zap" +) + +// GCClient is a client for doing GC +type GCClient interface { + UpdateGCSafePointV2(ctx context.Context, keyspaceID uint32, safePoint uint64) (uint64, error) + UpdateServiceSafePointV2(ctx context.Context, keyspaceID uint32, serviceID string, ttl int64, safePoint uint64) (uint64, error) + WatchGCSafePointV2(ctx context.Context, revision int64) (chan []*pdpb.SafePointEvent, error) +} + +// UpdateGCSafePointV2 update gc safe point for the given keyspace. +func (c *client) UpdateGCSafePointV2(ctx context.Context, keyspaceID uint32, safePoint uint64) (uint64, error) { + if span := opentracing.SpanFromContext(ctx); span != nil { + span = opentracing.StartSpan("pdclient.UpdateGCSafePointV2", opentracing.ChildOf(span.Context())) + defer span.Finish() + } + start := time.Now() + defer func() { cmdDurationUpdateGCSafePointV2.Observe(time.Since(start).Seconds()) }() + + ctx, cancel := context.WithTimeout(ctx, c.option.timeout) + req := &pdpb.UpdateGCSafePointV2Request{ + Header: c.requestHeader(), + KeyspaceId: keyspaceID, + SafePoint: safePoint, + } + ctx = grpcutil.BuildForwardContext(ctx, c.GetLeaderAddr()) + protoClient := c.getClient() + if protoClient == nil { + cancel() + return 0, errs.ErrClientGetProtoClient + } + resp, err := protoClient.UpdateGCSafePointV2(ctx, req) + cancel() + + if err = c.respForErr(cmdFailedDurationUpdateGCSafePointV2, start, err, resp.GetHeader()); err != nil { + return 0, err + } + return resp.GetNewSafePoint(), nil +} + +// UpdateServiceSafePointV2 update service safe point for the given keyspace. +func (c *client) UpdateServiceSafePointV2(ctx context.Context, keyspaceID uint32, serviceID string, ttl int64, safePoint uint64) (uint64, error) { + if span := opentracing.SpanFromContext(ctx); span != nil { + span = opentracing.StartSpan("pdclient.UpdateServiceSafePointV2", opentracing.ChildOf(span.Context())) + defer span.Finish() + } + start := time.Now() + defer func() { cmdDurationUpdateServiceSafePointV2.Observe(time.Since(start).Seconds()) }() + + ctx, cancel := context.WithTimeout(ctx, c.option.timeout) + req := &pdpb.UpdateServiceSafePointV2Request{ + Header: c.requestHeader(), + KeyspaceId: keyspaceID, + ServiceId: []byte(serviceID), + SafePoint: safePoint, + Ttl: ttl, + } + ctx = grpcutil.BuildForwardContext(ctx, c.GetLeaderAddr()) + protoClient := c.getClient() + if protoClient == nil { + cancel() + return 0, errs.ErrClientGetProtoClient + } + resp, err := protoClient.UpdateServiceSafePointV2(ctx, req) + cancel() + if err = c.respForErr(cmdFailedDurationUpdateServiceSafePointV2, start, err, resp.GetHeader()); err != nil { + return 0, err + } + return resp.GetMinSafePoint(), nil +} + +// WatchGCSafePointV2 watch gc safe point change. +func (c *client) WatchGCSafePointV2(ctx context.Context, revision int64) (chan []*pdpb.SafePointEvent, error) { + SafePointEventsChan := make(chan []*pdpb.SafePointEvent) + req := &pdpb.WatchGCSafePointV2Request{ + Header: c.requestHeader(), + Revision: revision, + } + + protoClient := c.getClient() + if protoClient == nil { + return nil, errs.ErrClientGetProtoClient + } + stream, err := protoClient.WatchGCSafePointV2(ctx, req) + if err != nil { + close(SafePointEventsChan) + return nil, err + } + go func() { + defer func() { + close(SafePointEventsChan) + if r := recover(); r != nil { + log.Error("[pd] panic in gc client `WatchGCSafePointV2`", zap.Any("error", r)) + return + } + }() + for { + select { + case <-ctx.Done(): + return + default: + resp, err := stream.Recv() + if err != nil { + log.Error("watch gc safe point v2 error", errs.ZapError(errs.ErrClientWatchGCSafePointV2Stream, err)) + return + } + SafePointEventsChan <- resp.GetEvents() + } + } + }() + return SafePointEventsChan, err +} diff --git a/client/go.mod b/client/go.mod index 05e04a5ecee..cf1349ca20a 100644 --- a/client/go.mod +++ b/client/go.mod @@ -3,39 +3,40 @@ module github.com/tikv/pd/client go 1.20 require ( - github.com/elastic/gosigar v0.14.2 + github.com/cloudfoundry/gosigar v1.3.6 github.com/gogo/protobuf v1.3.2 github.com/opentracing/opentracing-go v1.2.0 github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00 - github.com/pingcap/kvproto v0.0.0-20230228041042-1e9aca94bab6 + github.com/pingcap/kvproto v0.0.0-20230727073445-53e1f8730c30 github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 github.com/prometheus/client_golang v1.11.1 - github.com/stretchr/testify v1.8.1 + github.com/stretchr/testify v1.8.2 go.uber.org/goleak v1.1.11 - go.uber.org/zap v1.20.0 - google.golang.org/grpc v1.51.0 + go.uber.org/zap v1.24.0 + golang.org/x/exp v0.0.0-20230711005742-c3f37128e5a4 + google.golang.org/grpc v1.54.0 ) require ( - github.com/benbjohnson/clock v1.1.0 // indirect + github.com/benbjohnson/clock v1.3.0 // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/cespare/xxhash/v2 v2.1.1 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/golang/protobuf v1.5.2 // indirect + github.com/golang/protobuf v1.5.3 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_model v0.2.0 // indirect github.com/prometheus/common v0.26.0 // indirect github.com/prometheus/procfs v0.6.0 // indirect - go.uber.org/atomic v1.9.0 // indirect - go.uber.org/multierr v1.7.0 // indirect - golang.org/x/net v0.2.0 // indirect - golang.org/x/sys v0.2.0 // indirect - golang.org/x/text v0.4.0 // indirect - google.golang.org/genproto v0.0.0-20221202195650-67e5cbc046fd // indirect - google.golang.org/protobuf v1.28.1 // indirect - gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect + go.uber.org/atomic v1.10.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + golang.org/x/net v0.10.0 // indirect + golang.org/x/sys v0.8.0 // indirect + golang.org/x/text v0.9.0 // indirect + google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect + google.golang.org/protobuf v1.30.0 // indirect + gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/client/go.sum b/client/go.sum index 4e998a9029a..2884e84a1fa 100644 --- a/client/go.sum +++ b/client/go.sum @@ -1,24 +1,26 @@ cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= -github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= +github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= +github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cloudfoundry/gosigar v1.3.6 h1:gIc08FbB3QPb+nAQhINIK/qhf5REKkY0FTGgRGXkcVc= +github.com/cloudfoundry/gosigar v1.3.6/go.mod h1:lNWstu5g5gw59O09Y+wsMNFzBSnU8a0u+Sfx4dq360E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/elastic/gosigar v0.14.2 h1:Dg80n8cr90OZ7x+bAax/QjoW/XqTI11RmA79ZwIm9/4= -github.com/elastic/gosigar v0.14.2/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs= +github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= @@ -40,14 +42,14 @@ github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvq github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= -github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= @@ -73,6 +75,9 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= +github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= +github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE= github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= github.com/pingcap/errors v0.11.0/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= @@ -81,8 +86,8 @@ github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c h1:xpW9bvK+HuuTm github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c/go.mod h1:X2r9ueLEUZgtx2cIogM0v4Zj5uvvzhuuiu7Pn8HzMPg= github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00 h1:C3N3itkduZXDZFh4N3vQ5HEtld3S+Y+StULhWVvumU0= github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00/go.mod h1:4qGtCB0QK0wBzKtFEGDhxXnSnbQApw1gc9siScUl8ew= -github.com/pingcap/kvproto v0.0.0-20230228041042-1e9aca94bab6 h1:bgLRG7gPJCq6aduA65ZV7xWQBThTcuarBB9VdfAzV4g= -github.com/pingcap/kvproto v0.0.0-20230228041042-1e9aca94bab6/go.mod h1:KUrW1FGoznGMMTssYBu0czfAhn6vQcIrHyZoSC6T990= +github.com/pingcap/kvproto v0.0.0-20230727073445-53e1f8730c30 h1:EvqKcDT7ceGLW0mXqM8Cp5Z8DfgQRnwj2YTnlCLj2QI= +github.com/pingcap/kvproto v0.0.0-20230727073445-53e1f8730c30/go.mod h1:r0q/CFcwvyeRhKtoqzmWMBebrtpIziQQ9vR+JKh1knc= github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 h1:HR/ylkkLmGdSSDaD8IDP+SZrdhV1Kibl9KrHxJ9eciw= github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -123,28 +128,32 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= -github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= +github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= +go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI= go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= -go.uber.org/multierr v1.7.0 h1:zaiO/rmgFjbmCXdSYJWQcdvOCsthmdaHfr3Gm2Kx4Ec= go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= -go.uber.org/zap v1.20.0 h1:N4oPlghZwYG55MlU6LXk/Zp00FVNE9X9wrYO8CEs4lc= -go.uber.org/zap v1.20.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw= +go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= +go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/exp v0.0.0-20230711005742-c3f37128e5a4 h1:QLureRX3moex6NVu/Lr4MGakp9FdA7sBHGBmvRW7NaM= +golang.org/x/exp v0.0.0-20230711005742-c3f37128e5a4/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -161,8 +170,8 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.2.0 h1:sZfSu1wtKLGlWI4ZZayP0ck9Y73K1ynO6gqzTdBVdPU= -golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= +golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -171,7 +180,6 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20180810173357-98c5dad5d1a0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -187,14 +195,14 @@ golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.2.0 h1:ljd4t30dBnAvMZaQCevtY0xLLD0A+bRZXbgLMLU1F/A= -golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.4.0 h1:BrVqGRd7+k1DiOgtnFvAkoQEWQvBc25ouMJM6429SFg= -golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -203,17 +211,17 @@ golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.12 h1:VveCTK38A2rkS8ZqFY25HIDFscX5X9OoEhJd3quQmXU= +golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/genproto v0.0.0-20221202195650-67e5cbc046fd h1:OjndDrsik+Gt+e6fs45z9AxiewiKyLKYpA45W5Kpkks= -google.golang.org/genproto v0.0.0-20221202195650-67e5cbc046fd/go.mod h1:cTsE614GARnxrLsqKREzmNYJACSWWpAWdNMwnD7c2BE= -google.golang.org/grpc v1.51.0 h1:E1eGv1FTqoLIdnBCZufiSHgKjlqG6fKFf6pPWtMTh8U= -google.golang.org/grpc v1.51.0/go.mod h1:wgNDFcnuBGmxLKI/qn4T+m5BtEBYXJPvibbUPsAIPww= +google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 h1:KpwkzHKEF7B9Zxg18WzOa7djJ+Ha5DzthMyZYQfEn2A= +google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU= +google.golang.org/grpc v1.54.0 h1:EhTqbhiYeixwWQtAEZAxmV9MGqcjEU2mFx52xCzNyag= +google.golang.org/grpc v1.54.0/go.mod h1:PUSEXI6iWghWaB6lXM4knEgpJNu2qUcKfDtNci3EC2g= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -222,15 +230,17 @@ google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzi google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= -google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= +google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= +gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= +gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/client/grpcutil/grpcutil.go b/client/grpcutil/grpcutil.go index 89daaac257e..125f1125721 100644 --- a/client/grpcutil/grpcutil.go +++ b/client/grpcutil/grpcutil.go @@ -78,6 +78,7 @@ func BuildForwardContext(ctx context.Context, addr string) context.Context { func GetOrCreateGRPCConn(ctx context.Context, clientConns *sync.Map, addr string, tlsCfg *tlsutil.TLSConfig, opt ...grpc.DialOption) (*grpc.ClientConn, error) { conn, ok := clientConns.Load(addr) if ok { + // TODO: check the connection state. return conn.(*grpc.ClientConn), nil } tlsConfig, err := tlsCfg.ToTLSConfig() @@ -90,11 +91,13 @@ func GetOrCreateGRPCConn(ctx context.Context, clientConns *sync.Map, addr string if err != nil { return nil, err } - if old, ok := clientConns.Load(addr); ok { - cc.Close() - log.Debug("use old connection", zap.String("target", cc.Target()), zap.String("state", cc.GetState().String())) - return old.(*grpc.ClientConn), nil + conn, loaded := clientConns.LoadOrStore(addr, cc) + if !loaded { + // Successfully stored the connection. + return cc, nil } - clientConns.Store(addr, cc) + cc.Close() + cc = conn.(*grpc.ClientConn) + log.Debug("use existing connection", zap.String("target", cc.Target()), zap.String("state", cc.GetState().String())) return cc, nil } diff --git a/client/keyspace_client.go b/client/keyspace_client.go index 9864ad8a2ed..d9b9172dd69 100644 --- a/client/keyspace_client.go +++ b/client/keyspace_client.go @@ -21,24 +21,24 @@ import ( "github.com/opentracing/opentracing-go" "github.com/pingcap/errors" "github.com/pingcap/kvproto/pkg/keyspacepb" - "github.com/pingcap/log" "github.com/tikv/pd/client/grpcutil" - "go.uber.org/zap" ) // KeyspaceClient manages keyspace metadata. type KeyspaceClient interface { // LoadKeyspace load and return target keyspace's metadata. LoadKeyspace(ctx context.Context, name string) (*keyspacepb.KeyspaceMeta, error) - // WatchKeyspaces watches keyspace meta changes. - WatchKeyspaces(ctx context.Context) (chan []*keyspacepb.KeyspaceMeta, error) // UpdateKeyspaceState updates target keyspace's state. UpdateKeyspaceState(ctx context.Context, id uint32, state keyspacepb.KeyspaceState) (*keyspacepb.KeyspaceMeta, error) + // WatchKeyspaces watches keyspace meta changes. + WatchKeyspaces(ctx context.Context) (chan []*keyspacepb.KeyspaceMeta, error) + // GetAllKeyspaces get all keyspace's metadata. + GetAllKeyspaces(ctx context.Context, startID uint32, limit uint32) ([]*keyspacepb.KeyspaceMeta, error) } // keyspaceClient returns the KeyspaceClient from current PD leader. func (c *client) keyspaceClient() keyspacepb.KeyspaceClient { - if client := c.svcDiscovery.GetServingEndpointClientConn(); client != nil { + if client := c.pdSvcDiscovery.GetServingEndpointClientConn(); client != nil { return keyspacepb.NewKeyspaceClient(client) } return nil @@ -63,7 +63,7 @@ func (c *client) LoadKeyspace(ctx context.Context, name string) (*keyspacepb.Key if err != nil { cmdFailedDurationLoadKeyspace.Observe(time.Since(start).Seconds()) - c.svcDiscovery.ScheduleCheckMemberChanged() + c.pdSvcDiscovery.ScheduleCheckMemberChanged() return nil, err } @@ -75,44 +75,6 @@ func (c *client) LoadKeyspace(ctx context.Context, name string) (*keyspacepb.Key return resp.Keyspace, nil } -// WatchKeyspaces watches keyspace meta changes. -// It returns a stream of slices of keyspace metadata. -// The first message in stream contains all current keyspaceMeta, -// all subsequent messages contains new put events for all keyspaces. -func (c *client) WatchKeyspaces(ctx context.Context) (chan []*keyspacepb.KeyspaceMeta, error) { - keyspaceWatcherChan := make(chan []*keyspacepb.KeyspaceMeta) - req := &keyspacepb.WatchKeyspacesRequest{ - Header: c.requestHeader(), - } - stream, err := c.keyspaceClient().WatchKeyspaces(ctx, req) - if err != nil { - close(keyspaceWatcherChan) - return nil, err - } - go func() { - defer func() { - if r := recover(); r != nil { - log.Error("[pd] panic in keyspace client `WatchKeyspaces`", zap.Any("error", r)) - return - } - }() - for { - select { - case <-ctx.Done(): - close(keyspaceWatcherChan) - return - default: - resp, err := stream.Recv() - if err != nil { - return - } - keyspaceWatcherChan <- resp.Keyspaces - } - } - }() - return keyspaceWatcherChan, err -} - // UpdateKeyspaceState attempts to update the keyspace specified by ID to the target state, // it will also record StateChangedAt for the given keyspace if a state change took place. // Currently, legal operations includes: @@ -142,7 +104,7 @@ func (c *client) UpdateKeyspaceState(ctx context.Context, id uint32, state keysp if err != nil { cmdFailedDurationUpdateKeyspaceState.Observe(time.Since(start).Seconds()) - c.svcDiscovery.ScheduleCheckMemberChanged() + c.pdSvcDiscovery.ScheduleCheckMemberChanged() return nil, err } @@ -153,3 +115,43 @@ func (c *client) UpdateKeyspaceState(ctx context.Context, id uint32, state keysp return resp.Keyspace, nil } + +// WatchKeyspaces watches keyspace meta changes. +// It returns a stream of slices of keyspace metadata. +// The first message in stream contains all current keyspaceMeta, +// all subsequent messages contains new put events for all keyspaces. +func (c *client) WatchKeyspaces(ctx context.Context) (chan []*keyspacepb.KeyspaceMeta, error) { + return nil, errors.Errorf("WatchKeyspaces unimplemented") +} + +// GetAllKeyspaces get all keyspaces metadata. +func (c *client) GetAllKeyspaces(ctx context.Context, startID uint32, limit uint32) ([]*keyspacepb.KeyspaceMeta, error) { + if span := opentracing.SpanFromContext(ctx); span != nil { + span = opentracing.StartSpan("keyspaceClient.GetAllKeyspaces", opentracing.ChildOf(span.Context())) + defer span.Finish() + } + start := time.Now() + defer func() { cmdDurationGetAllKeyspaces.Observe(time.Since(start).Seconds()) }() + ctx, cancel := context.WithTimeout(ctx, c.option.timeout) + req := &keyspacepb.GetAllKeyspacesRequest{ + Header: c.requestHeader(), + StartId: startID, + Limit: limit, + } + ctx = grpcutil.BuildForwardContext(ctx, c.GetLeaderAddr()) + resp, err := c.keyspaceClient().GetAllKeyspaces(ctx, req) + cancel() + + if err != nil { + cmdDurationGetAllKeyspaces.Observe(time.Since(start).Seconds()) + c.pdSvcDiscovery.ScheduleCheckMemberChanged() + return nil, err + } + + if resp.Header.GetError() != nil { + cmdDurationGetAllKeyspaces.Observe(time.Since(start).Seconds()) + return nil, errors.Errorf("Get all keyspaces metadata failed: %s", resp.Header.GetError().String()) + } + + return resp.Keyspaces, nil +} diff --git a/client/meta_storage_client.go b/client/meta_storage_client.go index 0b0bd023c47..b203fb914d3 100644 --- a/client/meta_storage_client.go +++ b/client/meta_storage_client.go @@ -20,11 +20,11 @@ import ( "github.com/opentracing/opentracing-go" "github.com/pingcap/errors" + "github.com/pingcap/failpoint" "github.com/pingcap/kvproto/pkg/meta_storagepb" - "github.com/pingcap/log" "github.com/prometheus/client_golang/prometheus" + "github.com/tikv/pd/client/errs" "github.com/tikv/pd/client/grpcutil" - "go.uber.org/zap" ) // MetaStorageClient is the interface for meta storage client. @@ -39,7 +39,7 @@ type MetaStorageClient interface { // metaStorageClient gets the meta storage client from current PD leader. func (c *client) metaStorageClient() meta_storagepb.MetaStorageClient { - if client := c.svcDiscovery.GetServingEndpointClientConn(); client != nil { + if client := c.pdSvcDiscovery.GetServingEndpointClientConn(); client != nil { return meta_storagepb.NewMetaStorageClient(client) } return nil @@ -125,7 +125,12 @@ func (c *client) Put(ctx context.Context, key, value []byte, opts ...OpOption) ( PrevKv: options.prevKv, } ctx = grpcutil.BuildForwardContext(ctx, c.GetLeaderAddr()) - resp, err := c.metaStorageClient().Put(ctx, req) + cli := c.metaStorageClient() + if cli == nil { + cancel() + return nil, errs.ErrClientGetMetaStorageClient + } + resp, err := cli.Put(ctx, req) cancel() if err = c.respForMetaStorageErr(cmdFailedDurationPut, start, err, resp.GetHeader()); err != nil { @@ -158,7 +163,12 @@ func (c *client) Get(ctx context.Context, key []byte, opts ...OpOption) (*meta_s Revision: options.revision, } ctx = grpcutil.BuildForwardContext(ctx, c.GetLeaderAddr()) - resp, err := c.metaStorageClient().Get(ctx, req) + cli := c.metaStorageClient() + if cli == nil { + cancel() + return nil, errs.ErrClientGetMetaStorageClient + } + resp, err := cli.Get(ctx, req) cancel() if err = c.respForMetaStorageErr(cmdFailedDurationGet, start, err, resp.GetHeader()); err != nil { @@ -177,7 +187,11 @@ func (c *client) Watch(ctx context.Context, key []byte, opts ...OpOption) (chan options.rangeEnd = getPrefix(key) } - res, err := c.metaStorageClient().Watch(ctx, &meta_storagepb.WatchRequest{ + cli := c.metaStorageClient() + if cli == nil { + return nil, errs.ErrClientGetMetaStorageClient + } + res, err := cli.Watch(ctx, &meta_storagepb.WatchRequest{ Key: key, RangeEnd: options.rangeEnd, StartRevision: options.revision, @@ -190,13 +204,12 @@ func (c *client) Watch(ctx context.Context, key []byte, opts ...OpOption) (chan go func() { defer func() { close(eventCh) - if r := recover(); r != nil { - log.Error("[pd] panic in client `Watch`", zap.Any("error", r)) - return - } }() for { resp, err := res.Recv() + failpoint.Inject("watchStreamError", func() { + err = errors.Errorf("fake error") + }) if err != nil { return } @@ -214,7 +227,7 @@ func (c *client) respForMetaStorageErr(observer prometheus.Observer, start time. if err != nil || header.GetError() != nil { observer.Observe(time.Since(start).Seconds()) if err != nil { - c.svcDiscovery.ScheduleCheckMemberChanged() + c.pdSvcDiscovery.ScheduleCheckMemberChanged() return errors.WithStack(err) } return errors.WithStack(errors.New(header.GetError().String())) diff --git a/client/metrics.go b/client/metrics.go index 06c97a34785..1895306eca2 100644 --- a/client/metrics.go +++ b/client/metrics.go @@ -14,114 +14,203 @@ package pd -import "github.com/prometheus/client_golang/prometheus" +import ( + "sync/atomic" + + "github.com/prometheus/client_golang/prometheus" +) + +// make sure register metrics only once +var initialized int32 + +func init() { + initMetrics(prometheus.Labels{}) + initCmdDurations() +} + +func initAndRegisterMetrics(constLabels prometheus.Labels) { + if atomic.CompareAndSwapInt32(&initialized, 0, 1) { + // init metrics with constLabels + initMetrics(constLabels) + initCmdDurations() + // register metrics + registerMetrics() + } +} var ( + cmdDuration *prometheus.HistogramVec + cmdFailedDuration *prometheus.HistogramVec + requestDuration *prometheus.HistogramVec + tsoBestBatchSize prometheus.Histogram + tsoBatchSize prometheus.Histogram + tsoBatchSendLatency prometheus.Histogram + requestForwarded *prometheus.GaugeVec +) + +func initMetrics(constLabels prometheus.Labels) { cmdDuration = prometheus.NewHistogramVec( prometheus.HistogramOpts{ - Namespace: "pd_client", - Subsystem: "cmd", - Name: "handle_cmds_duration_seconds", - Help: "Bucketed histogram of processing time (s) of handled success cmds.", - Buckets: prometheus.ExponentialBuckets(0.0005, 2, 13), + Namespace: "pd_client", + Subsystem: "cmd", + Name: "handle_cmds_duration_seconds", + Help: "Bucketed histogram of processing time (s) of handled success cmds.", + ConstLabels: constLabels, + Buckets: prometheus.ExponentialBuckets(0.0005, 2, 13), }, []string{"type"}) cmdFailedDuration = prometheus.NewHistogramVec( prometheus.HistogramOpts{ - Namespace: "pd_client", - Subsystem: "cmd", - Name: "handle_failed_cmds_duration_seconds", - Help: "Bucketed histogram of processing time (s) of failed handled cmds.", - Buckets: prometheus.ExponentialBuckets(0.0005, 2, 13), + Namespace: "pd_client", + Subsystem: "cmd", + Name: "handle_failed_cmds_duration_seconds", + Help: "Bucketed histogram of processing time (s) of failed handled cmds.", + ConstLabels: constLabels, + Buckets: prometheus.ExponentialBuckets(0.0005, 2, 13), }, []string{"type"}) requestDuration = prometheus.NewHistogramVec( prometheus.HistogramOpts{ - Namespace: "pd_client", - Subsystem: "request", - Name: "handle_requests_duration_seconds", - Help: "Bucketed histogram of processing time (s) of handled requests.", - Buckets: prometheus.ExponentialBuckets(0.0005, 2, 13), + Namespace: "pd_client", + Subsystem: "request", + Name: "handle_requests_duration_seconds", + Help: "Bucketed histogram of processing time (s) of handled requests.", + ConstLabels: constLabels, + Buckets: prometheus.ExponentialBuckets(0.0005, 2, 13), }, []string{"type"}) tsoBestBatchSize = prometheus.NewHistogram( prometheus.HistogramOpts{ - Namespace: "pd_client", - Subsystem: "request", - Name: "handle_tso_best_batch_size", - Help: "Bucketed histogram of the best batch size of handled requests.", - Buckets: prometheus.ExponentialBuckets(1, 2, 13), + Namespace: "pd_client", + Subsystem: "request", + Name: "handle_tso_best_batch_size", + Help: "Bucketed histogram of the best batch size of handled requests.", + ConstLabels: constLabels, + Buckets: prometheus.ExponentialBuckets(1, 2, 13), }) tsoBatchSize = prometheus.NewHistogram( prometheus.HistogramOpts{ - Namespace: "pd_client", - Subsystem: "request", - Name: "handle_tso_batch_size", - Help: "Bucketed histogram of the batch size of handled requests.", - Buckets: []float64{1, 2, 4, 8, 10, 14, 18, 22, 26, 30, 35, 40, 45, 50, 60, 70, 80, 90, 100, 110, 120, 140, 160, 180, 200, 500, 1000}, + Namespace: "pd_client", + Subsystem: "request", + Name: "handle_tso_batch_size", + Help: "Bucketed histogram of the batch size of handled requests.", + ConstLabels: constLabels, + Buckets: []float64{1, 2, 4, 8, 10, 14, 18, 22, 26, 30, 35, 40, 45, 50, 60, 70, 80, 90, 100, 110, 120, 140, 160, 180, 200, 500, 1000}, }) tsoBatchSendLatency = prometheus.NewHistogram( prometheus.HistogramOpts{ - Namespace: "pd_client", - Subsystem: "request", - Name: "tso_batch_send_latency", - Buckets: prometheus.ExponentialBuckets(1, 2, 34), // 1ns ~ 8s - Help: "tso batch send latency", + Namespace: "pd_client", + Subsystem: "request", + Name: "tso_batch_send_latency", + ConstLabels: constLabels, + Buckets: prometheus.ExponentialBuckets(1, 2, 34), // 1ns ~ 8s + Help: "tso batch send latency", }) requestForwarded = prometheus.NewGaugeVec( prometheus.GaugeOpts{ - Namespace: "pd_client", - Subsystem: "request", - Name: "forwarded_status", - Help: "The status to indicate if the request is forwarded", + Namespace: "pd_client", + Subsystem: "request", + Name: "forwarded_status", + Help: "The status to indicate if the request is forwarded", + ConstLabels: constLabels, }, []string{"host", "delegate"}) -) +} var ( + cmdDurationWait prometheus.Observer + cmdDurationTSO prometheus.Observer + cmdDurationTSOAsyncWait prometheus.Observer + cmdDurationGetRegion prometheus.Observer + cmdDurationGetAllMembers prometheus.Observer + cmdDurationGetPrevRegion prometheus.Observer + cmdDurationGetRegionByID prometheus.Observer + cmdDurationScanRegions prometheus.Observer + cmdDurationGetStore prometheus.Observer + cmdDurationGetAllStores prometheus.Observer + cmdDurationUpdateGCSafePoint prometheus.Observer + cmdDurationUpdateServiceGCSafePoint prometheus.Observer + cmdDurationScatterRegion prometheus.Observer + cmdDurationScatterRegions prometheus.Observer + cmdDurationGetOperator prometheus.Observer + cmdDurationSplitRegions prometheus.Observer + cmdDurationSplitAndScatterRegions prometheus.Observer + cmdDurationLoadKeyspace prometheus.Observer + cmdDurationUpdateKeyspaceState prometheus.Observer + cmdDurationGetAllKeyspaces prometheus.Observer + cmdDurationGet prometheus.Observer + cmdDurationPut prometheus.Observer + cmdDurationUpdateGCSafePointV2 prometheus.Observer + cmdDurationUpdateServiceSafePointV2 prometheus.Observer + + cmdFailDurationGetRegion prometheus.Observer + cmdFailDurationTSO prometheus.Observer + cmdFailDurationGetAllMembers prometheus.Observer + cmdFailDurationGetPrevRegion prometheus.Observer + cmdFailedDurationGetRegionByID prometheus.Observer + cmdFailedDurationScanRegions prometheus.Observer + cmdFailedDurationGetStore prometheus.Observer + cmdFailedDurationGetAllStores prometheus.Observer + cmdFailedDurationUpdateGCSafePoint prometheus.Observer + cmdFailedDurationUpdateServiceGCSafePoint prometheus.Observer + cmdFailedDurationLoadKeyspace prometheus.Observer + cmdFailedDurationUpdateKeyspaceState prometheus.Observer + requestDurationTSO prometheus.Observer + cmdFailedDurationGet prometheus.Observer + cmdFailedDurationPut prometheus.Observer + cmdFailedDurationUpdateGCSafePointV2 prometheus.Observer + cmdFailedDurationUpdateServiceSafePointV2 prometheus.Observer +) + +func initCmdDurations() { // WithLabelValues is a heavy operation, define variable to avoid call it every time. - cmdDurationWait = cmdDuration.WithLabelValues("wait") - cmdDurationTSO = cmdDuration.WithLabelValues("tso") - cmdDurationTSOAsyncWait = cmdDuration.WithLabelValues("tso_async_wait") - cmdDurationGetRegion = cmdDuration.WithLabelValues("get_region") - cmdDurationGetAllMembers = cmdDuration.WithLabelValues("get_member_info") - cmdDurationGetPrevRegion = cmdDuration.WithLabelValues("get_prev_region") - cmdDurationGetRegionByID = cmdDuration.WithLabelValues("get_region_byid") - cmdDurationScanRegions = cmdDuration.WithLabelValues("scan_regions") - cmdDurationGetStore = cmdDuration.WithLabelValues("get_store") - cmdDurationGetAllStores = cmdDuration.WithLabelValues("get_all_stores") - cmdDurationUpdateGCSafePoint = cmdDuration.WithLabelValues("update_gc_safe_point") + cmdDurationWait = cmdDuration.WithLabelValues("wait") + cmdDurationTSO = cmdDuration.WithLabelValues("tso") + cmdDurationTSOAsyncWait = cmdDuration.WithLabelValues("tso_async_wait") + cmdDurationGetRegion = cmdDuration.WithLabelValues("get_region") + cmdDurationGetAllMembers = cmdDuration.WithLabelValues("get_member_info") + cmdDurationGetPrevRegion = cmdDuration.WithLabelValues("get_prev_region") + cmdDurationGetRegionByID = cmdDuration.WithLabelValues("get_region_byid") + cmdDurationScanRegions = cmdDuration.WithLabelValues("scan_regions") + cmdDurationGetStore = cmdDuration.WithLabelValues("get_store") + cmdDurationGetAllStores = cmdDuration.WithLabelValues("get_all_stores") + cmdDurationUpdateGCSafePoint = cmdDuration.WithLabelValues("update_gc_safe_point") cmdDurationUpdateServiceGCSafePoint = cmdDuration.WithLabelValues("update_service_gc_safe_point") - cmdDurationScatterRegion = cmdDuration.WithLabelValues("scatter_region") - cmdDurationScatterRegions = cmdDuration.WithLabelValues("scatter_regions") - cmdDurationGetOperator = cmdDuration.WithLabelValues("get_operator") - cmdDurationSplitRegions = cmdDuration.WithLabelValues("split_regions") - cmdDurationSplitAndScatterRegions = cmdDuration.WithLabelValues("split_and_scatter_regions") - cmdDurationLoadKeyspace = cmdDuration.WithLabelValues("load_keyspace") - cmdDurationUpdateKeyspaceState = cmdDuration.WithLabelValues("update_keyspace_state") - cmdDurationGet = cmdDuration.WithLabelValues("get") - cmdDurationPut = cmdDuration.WithLabelValues("put") - - cmdFailDurationGetRegion = cmdFailedDuration.WithLabelValues("get_region") - cmdFailDurationTSO = cmdFailedDuration.WithLabelValues("tso") - cmdFailDurationGetAllMembers = cmdFailedDuration.WithLabelValues("get_member_info") - cmdFailDurationGetPrevRegion = cmdFailedDuration.WithLabelValues("get_prev_region") - cmdFailedDurationGetRegionByID = cmdFailedDuration.WithLabelValues("get_region_byid") - cmdFailedDurationScanRegions = cmdFailedDuration.WithLabelValues("scan_regions") - cmdFailedDurationGetStore = cmdFailedDuration.WithLabelValues("get_store") - cmdFailedDurationGetAllStores = cmdFailedDuration.WithLabelValues("get_all_stores") - cmdFailedDurationUpdateGCSafePoint = cmdFailedDuration.WithLabelValues("update_gc_safe_point") + cmdDurationScatterRegion = cmdDuration.WithLabelValues("scatter_region") + cmdDurationScatterRegions = cmdDuration.WithLabelValues("scatter_regions") + cmdDurationGetOperator = cmdDuration.WithLabelValues("get_operator") + cmdDurationSplitRegions = cmdDuration.WithLabelValues("split_regions") + cmdDurationSplitAndScatterRegions = cmdDuration.WithLabelValues("split_and_scatter_regions") + cmdDurationLoadKeyspace = cmdDuration.WithLabelValues("load_keyspace") + cmdDurationUpdateKeyspaceState = cmdDuration.WithLabelValues("update_keyspace_state") + cmdDurationGetAllKeyspaces = cmdDuration.WithLabelValues("get_all_keyspaces") + cmdDurationGet = cmdDuration.WithLabelValues("get") + cmdDurationPut = cmdDuration.WithLabelValues("put") + cmdDurationUpdateGCSafePointV2 = cmdDuration.WithLabelValues("update_gc_safe_point_v2") + cmdDurationUpdateServiceSafePointV2 = cmdDuration.WithLabelValues("update_service_safe_point_v2") + + cmdFailDurationGetRegion = cmdFailedDuration.WithLabelValues("get_region") + cmdFailDurationTSO = cmdFailedDuration.WithLabelValues("tso") + cmdFailDurationGetAllMembers = cmdFailedDuration.WithLabelValues("get_member_info") + cmdFailDurationGetPrevRegion = cmdFailedDuration.WithLabelValues("get_prev_region") + cmdFailedDurationGetRegionByID = cmdFailedDuration.WithLabelValues("get_region_byid") + cmdFailedDurationScanRegions = cmdFailedDuration.WithLabelValues("scan_regions") + cmdFailedDurationGetStore = cmdFailedDuration.WithLabelValues("get_store") + cmdFailedDurationGetAllStores = cmdFailedDuration.WithLabelValues("get_all_stores") + cmdFailedDurationUpdateGCSafePoint = cmdFailedDuration.WithLabelValues("update_gc_safe_point") cmdFailedDurationUpdateServiceGCSafePoint = cmdFailedDuration.WithLabelValues("update_service_gc_safe_point") - cmdFailedDurationLoadKeyspace = cmdDuration.WithLabelValues("load_keyspace") - cmdFailedDurationUpdateKeyspaceState = cmdDuration.WithLabelValues("update_keyspace_state") - requestDurationTSO = requestDuration.WithLabelValues("tso") - cmdFailedDurationGet = cmdFailedDuration.WithLabelValues("get") - cmdFailedDurationPut = cmdFailedDuration.WithLabelValues("put") -) + cmdFailedDurationLoadKeyspace = cmdFailedDuration.WithLabelValues("load_keyspace") + cmdFailedDurationUpdateKeyspaceState = cmdFailedDuration.WithLabelValues("update_keyspace_state") + requestDurationTSO = requestDuration.WithLabelValues("tso") + cmdFailedDurationGet = cmdFailedDuration.WithLabelValues("get") + cmdFailedDurationPut = cmdFailedDuration.WithLabelValues("put") + cmdFailedDurationUpdateGCSafePointV2 = cmdFailedDuration.WithLabelValues("update_gc_safe_point_v2") + cmdFailedDurationUpdateServiceSafePointV2 = cmdFailedDuration.WithLabelValues("update_service_safe_point_v2") +} -func init() { +func registerMetrics() { prometheus.MustRegister(cmdDuration) prometheus.MustRegister(cmdFailedDuration) prometheus.MustRegister(requestDuration) diff --git a/client/option.go b/client/option.go index e2fcfcbbef4..9d46c7b1a70 100644 --- a/client/option.go +++ b/client/option.go @@ -19,6 +19,7 @@ import ( "time" "github.com/pingcap/errors" + "github.com/prometheus/client_golang/prometheus" "google.golang.org/grpc" ) @@ -51,6 +52,9 @@ type option struct { timeout time.Duration maxRetryTimes int enableForwarding bool + metricsLabels prometheus.Labels + initMetrics bool + allowTSOFallback bool // Dynamic options. dynamicOptions [dynamicOptionCount]atomic.Value @@ -64,6 +68,7 @@ func newOption() *option { timeout: defaultPDTimeout, maxRetryTimes: maxInitClusterRetries, enableTSOFollowerProxyCh: make(chan struct{}, 1), + initMetrics: true, } co.dynamicOptions[MaxTSOBatchWaitInterval].Store(defaultMaxTSOBatchWaitInterval) diff --git a/client/pd_service_discovery.go b/client/pd_service_discovery.go index b4152b8c5e1..4499c9e17c0 100644 --- a/client/pd_service_discovery.go +++ b/client/pd_service_discovery.go @@ -18,6 +18,7 @@ import ( "context" "reflect" "sort" + "strings" "sync" "sync/atomic" "time" @@ -34,8 +35,17 @@ import ( ) const ( - globalDCLocation = "global" - memberUpdateInterval = time.Minute + globalDCLocation = "global" + memberUpdateInterval = time.Minute + serviceModeUpdateInterval = 3 * time.Second + updateMemberTimeout = time.Second // Use a shorter timeout to recover faster from network isolation. +) + +type serviceType int + +const ( + apiService serviceType = iota + tsoService ) // ServiceDiscovery defines the general interface for service discovery on a quorum-based cluster @@ -47,8 +57,14 @@ type ServiceDiscovery interface { Close() // GetClusterID returns the ID of the cluster GetClusterID() uint64 - // GetURLs returns the URLs of the servers. - GetURLs() []string + // GetKeyspaceID returns the ID of the keyspace + GetKeyspaceID() uint32 + // GetKeyspaceGroupID returns the ID of the keyspace group + GetKeyspaceGroupID() uint32 + // DiscoverServiceURLs discovers the microservice with the specified type and returns the server urls. + DiscoverMicroservice(svcType serviceType) ([]string, error) + // GetServiceURLs returns the URLs of the servers providing the service + GetServiceURLs() []string // GetServingEndpointClientConn returns the grpc client connection of the serving endpoint // which is the leader in a quorum-based cluster or the primary in a primary/secondary // configured cluster. @@ -81,6 +97,7 @@ type ServiceDiscovery interface { AddServiceAddrsSwitchedCallback(callbacks ...func()) } +type updateKeyspaceIDFunc func() error type tsoLocalServAddrsUpdatedFunc func(map[string]string) error type tsoGlobalServAddrUpdatedFunc func(string) error @@ -110,13 +127,15 @@ type pdServiceDiscovery struct { // addr -> a gRPC connection clientConns sync.Map // Store as map[string]*grpc.ClientConn - // leaderSwitchedCbs will be called after the leader swichted + // serviceModeUpdateCb will be called when the service mode gets updated + serviceModeUpdateCb func(pdpb.ServiceMode) + // leaderSwitchedCbs will be called after the leader switched leaderSwitchedCbs []func() // membersChangedCbs will be called after there is any membership change in the // leader and followers membersChangedCbs []func() // tsoLocalAllocLeadersUpdatedCb will be called when the local tso allocator - // leader list is updated. The input is a map {DC Localtion -> Leader Addr} + // leader list is updated. The input is a map {DC Location -> Leader Addr} tsoLocalAllocLeadersUpdatedCb tsoLocalServAddrsUpdatedFunc // tsoGlobalAllocLeaderUpdatedCb will be called when the global tso allocator // leader is updated. @@ -129,49 +148,76 @@ type pdServiceDiscovery struct { cancel context.CancelFunc closeOnce sync.Once - tlsCfg *tlsutil.TLSConfig + updateKeyspaceIDCb updateKeyspaceIDFunc + keyspaceID uint32 + tlsCfg *tlsutil.TLSConfig // Client option. option *option } -// newPDServiceDiscovery returns a new baseClient. -func newPDServiceDiscovery(ctx context.Context, cancel context.CancelFunc, - wg *sync.WaitGroup, urls []string, tlsCfg *tlsutil.TLSConfig, option *option) *pdServiceDiscovery { +// newPDServiceDiscovery returns a new PD service discovery-based client. +func newPDServiceDiscovery( + ctx context.Context, cancel context.CancelFunc, + wg *sync.WaitGroup, + serviceModeUpdateCb func(pdpb.ServiceMode), + updateKeyspaceIDCb updateKeyspaceIDFunc, + keyspaceID uint32, + urls []string, tlsCfg *tlsutil.TLSConfig, option *option, +) *pdServiceDiscovery { pdsd := &pdServiceDiscovery{ - checkMembershipCh: make(chan struct{}, 1), - ctx: ctx, - cancel: cancel, - wg: wg, - tlsCfg: tlsCfg, - option: option, + checkMembershipCh: make(chan struct{}, 1), + ctx: ctx, + cancel: cancel, + wg: wg, + serviceModeUpdateCb: serviceModeUpdateCb, + updateKeyspaceIDCb: updateKeyspaceIDCb, + keyspaceID: keyspaceID, + tlsCfg: tlsCfg, + option: option, } pdsd.urls.Store(urls) return pdsd } func (c *pdServiceDiscovery) Init() error { - if !c.isInitialized { - if err := c.initRetry(c.initClusterID); err != nil { - c.cancel() - return err - } - if err := c.initRetry(c.updateMember); err != nil { - c.cancel() + if c.isInitialized { + return nil + } + + if err := c.initRetry(c.initClusterID); err != nil { + c.cancel() + return err + } + if err := c.initRetry(c.updateMember); err != nil { + c.cancel() + return err + } + log.Info("[pd] init cluster id", zap.Uint64("cluster-id", c.clusterID)) + + // We need to update the keyspace ID before we discover and update the service mode + // so that TSO in API mode can be initialized with the correct keyspace ID. + if c.updateKeyspaceIDCb != nil { + if err := c.updateKeyspaceIDCb(); err != nil { return err } - log.Info("[pd] init cluster id", zap.Uint64("cluster-id", c.clusterID)) - - c.wg.Add(1) - go c.memberLoop() + } - c.isInitialized = true + if err := c.checkServiceModeChanged(); err != nil { + log.Warn("[pd] failed to check service mode and will check later", zap.Error(err)) } + c.wg.Add(2) + go c.updateMemberLoop() + go c.updateServiceModeLoop() + + c.isInitialized = true return nil } func (c *pdServiceDiscovery) initRetry(f func() error) error { var err error + ticker := time.NewTicker(time.Second) + defer ticker.Stop() for i := 0; i < c.option.maxRetryTimes; i++ { if err = f(); err == nil { return nil @@ -179,41 +225,72 @@ func (c *pdServiceDiscovery) initRetry(f func() error) error { select { case <-c.ctx.Done(): return err - case <-time.After(time.Second): + case <-ticker.C: } } return errors.WithStack(err) } -func (c *pdServiceDiscovery) memberLoop() { +func (c *pdServiceDiscovery) updateMemberLoop() { defer c.wg.Done() ctx, cancel := context.WithCancel(c.ctx) defer cancel() + ticker := time.NewTicker(memberUpdateInterval) + defer ticker.Stop() for { select { - case <-c.checkMembershipCh: - case <-time.After(memberUpdateInterval): case <-ctx.Done(): return + case <-ticker.C: + case <-c.checkMembershipCh: } failpoint.Inject("skipUpdateMember", func() { failpoint.Continue() }) if err := c.updateMember(); err != nil { - log.Error("[pd] failed to update member", errs.ZapError(err)) + log.Error("[pd] failed to update member", zap.Strings("urls", c.GetServiceURLs()), errs.ZapError(err)) } } } -// Close releases all resources +func (c *pdServiceDiscovery) updateServiceModeLoop() { + defer c.wg.Done() + failpoint.Inject("skipUpdateServiceMode", func() { + failpoint.Return() + }) + failpoint.Inject("usePDServiceMode", func() { + c.serviceModeUpdateCb(pdpb.ServiceMode_PD_SVC_MODE) + failpoint.Return() + }) + + ctx, cancel := context.WithCancel(c.ctx) + defer cancel() + ticker := time.NewTicker(serviceModeUpdateInterval) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + } + if err := c.checkServiceModeChanged(); err != nil { + log.Error("[pd] failed to update service mode", + zap.Strings("urls", c.GetServiceURLs()), errs.ZapError(err)) + c.ScheduleCheckMemberChanged() // check if the leader changed + } + } +} + +// Close releases all resources. func (c *pdServiceDiscovery) Close() { c.closeOnce.Do(func() { - log.Info("close pd service discovery") + log.Info("[pd] close pd service discovery client") c.clientConns.Range(func(key, cc interface{}) bool { if err := cc.(*grpc.ClientConn).Close(); err != nil { - log.Error("[pd] failed to close gRPC clientConn", errs.ZapError(errs.ErrCloseGRPCConn, err)) + log.Error("[pd] failed to close grpc clientConn", errs.ZapError(errs.ErrCloseGRPCConn, err)) } c.clientConns.Delete(key) return true @@ -226,13 +303,55 @@ func (c *pdServiceDiscovery) GetClusterID() uint64 { return c.clusterID } -// GetURLs returns the URLs of the servers. +// GetKeyspaceID returns the ID of the keyspace +func (c *pdServiceDiscovery) GetKeyspaceID() uint32 { + return c.keyspaceID +} + +// SetKeyspaceID sets the ID of the keyspace +func (c *pdServiceDiscovery) SetKeyspaceID(keyspaceID uint32) { + c.keyspaceID = keyspaceID +} + +// GetKeyspaceGroupID returns the ID of the keyspace group +func (c *pdServiceDiscovery) GetKeyspaceGroupID() uint32 { + // PD/API service only supports the default keyspace group + return defaultKeySpaceGroupID +} + +// DiscoverServiceURLs discovers the microservice with the specified type and returns the server urls. +func (c *pdServiceDiscovery) DiscoverMicroservice(svcType serviceType) (urls []string, err error) { + switch svcType { + case apiService: + urls = c.GetServiceURLs() + case tsoService: + leaderAddr := c.getLeaderAddr() + if len(leaderAddr) > 0 { + clusterInfo, err := c.getClusterInfo(c.ctx, leaderAddr, c.option.timeout) + if err != nil { + log.Error("[pd] failed to get cluster info", + zap.String("leader-addr", leaderAddr), errs.ZapError(err)) + return nil, err + } + urls = clusterInfo.TsoUrls + } else { + err = errors.New("failed to get leader addr") + return nil, err + } + default: + panic("invalid service type") + } + + return urls, nil +} + +// GetServiceURLs returns the URLs of the servers. // For testing use. It should only be called when the client is closed. -func (c *pdServiceDiscovery) GetURLs() []string { +func (c *pdServiceDiscovery) GetServiceURLs() []string { return c.urls.Load().([]string) } -// GetServingAddr returns the grpc client connection of the serving endpoint +// GetServingEndpointClientConn returns the grpc client connection of the serving endpoint // which is the leader in a quorum-based cluster or the primary in a primary/secondary // configured cluster. func (c *pdServiceDiscovery) GetServingEndpointClientConn() *grpc.ClientConn { @@ -294,6 +413,10 @@ func (c *pdServiceDiscovery) SetTSOLocalServAddrsUpdatedCallback(callback tsoLoc // SetTSOGlobalServAddrUpdatedCallback adds a callback which will be called when the global tso // allocator leader is updated. func (c *pdServiceDiscovery) SetTSOGlobalServAddrUpdatedCallback(callback tsoGlobalServAddrUpdatedFunc) { + addr := c.getLeaderAddr() + if len(addr) > 0 { + callback(addr) + } c.tsoGlobalAllocLeaderUpdatedCb = callback } @@ -318,11 +441,11 @@ func (c *pdServiceDiscovery) getFollowerAddrs() []string { func (c *pdServiceDiscovery) initClusterID() error { ctx, cancel := context.WithCancel(c.ctx) defer cancel() - var clusterID uint64 - for _, u := range c.GetURLs() { - members, err := c.getMembers(ctx, u, c.option.timeout) + clusterID := uint64(0) + for _, url := range c.GetServiceURLs() { + members, err := c.getMembers(ctx, url, c.option.timeout) if err != nil || members.GetHeader() == nil { - log.Warn("[pd] failed to get cluster id", zap.String("url", u), errs.ZapError(err)) + log.Warn("[pd] failed to get cluster id", zap.String("url", url), errs.ZapError(err)) continue } if clusterID == 0 { @@ -345,14 +468,39 @@ func (c *pdServiceDiscovery) initClusterID() error { return nil } +func (c *pdServiceDiscovery) checkServiceModeChanged() error { + leaderAddr := c.getLeaderAddr() + if len(leaderAddr) == 0 { + return errors.New("no leader found") + } + + clusterInfo, err := c.getClusterInfo(c.ctx, leaderAddr, c.option.timeout) + if err != nil { + if strings.Contains(err.Error(), "Unimplemented") { + // If the method is not supported, we set it to pd mode. + // TODO: it's a hack way to solve the compatibility issue. + // we need to remove this after all maintained version supports the method. + c.serviceModeUpdateCb(pdpb.ServiceMode_PD_SVC_MODE) + return nil + } + return err + } + if clusterInfo == nil || len(clusterInfo.ServiceModes) == 0 { + return errors.WithStack(errNoServiceModeReturned) + } + c.serviceModeUpdateCb(clusterInfo.ServiceModes[0]) + return nil +} + func (c *pdServiceDiscovery) updateMember() error { - for i, u := range c.GetURLs() { + for i, url := range c.GetServiceURLs() { failpoint.Inject("skipFirstUpdateMember", func() { if i == 0 { failpoint.Continue() } }) - members, err := c.getMembers(c.ctx, u, updateMemberTimeout) + + members, err := c.getMembers(c.ctx, url, updateMemberTimeout) // Check the cluster ID. if err == nil && members.GetHeader().GetClusterId() != c.clusterID { err = errs.ErrClientUpdateMember.FastGenByArgs("cluster id does not match") @@ -361,7 +509,7 @@ func (c *pdServiceDiscovery) updateMember() error { var errTSO error if err == nil { if members.GetLeader() == nil || len(members.GetLeader().GetClientUrls()) == 0 { - err = errs.ErrClientGetLeader.FastGenByArgs("leader address don't exist") + err = errs.ErrClientGetLeader.FastGenByArgs("leader address doesn't exist") } // Still need to update TsoAllocatorLeaders, even if there is no PD leader errTSO = c.switchTSOAllocatorLeaders(members.GetTsoAllocatorLeaders()) @@ -370,7 +518,7 @@ func (c *pdServiceDiscovery) updateMember() error { // Failed to get members if err != nil { log.Info("[pd] cannot update member from this address", - zap.String("address", u), + zap.String("address", url), errs.ZapError(err)) select { case <-c.ctx.Done(): @@ -390,7 +538,26 @@ func (c *pdServiceDiscovery) updateMember() error { // the error of `switchTSOAllocatorLeader` will be returned. return errTSO } - return errs.ErrClientGetMember.FastGenByArgs(c.GetURLs()) + return errs.ErrClientGetMember.FastGenByArgs() +} + +func (c *pdServiceDiscovery) getClusterInfo(ctx context.Context, url string, timeout time.Duration) (*pdpb.GetClusterInfoResponse, error) { + ctx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + cc, err := c.GetOrCreateGRPCConn(url) + if err != nil { + return nil, err + } + clusterInfo, err := pdpb.NewPDClient(cc).GetClusterInfo(ctx, &pdpb.GetClusterInfoRequest{}) + if err != nil { + attachErr := errors.Errorf("error:%s target:%s status:%s", err, cc.Target(), cc.GetState().String()) + return nil, errs.ErrClientGetClusterInfo.Wrap(attachErr).GenWithStackByCause() + } + if clusterInfo.GetHeader().GetError() != nil { + attachErr := errors.Errorf("error:%s target:%s status:%s", clusterInfo.GetHeader().GetError().String(), cc.Target(), cc.GetState().String()) + return nil, errs.ErrClientGetClusterInfo.Wrap(attachErr).GenWithStackByCause() + } + return clusterInfo, nil } func (c *pdServiceDiscovery) getMembers(ctx context.Context, url string, timeout time.Duration) (*pdpb.GetMembersResponse, error) { @@ -419,7 +586,7 @@ func (c *pdServiceDiscovery) updateURLs(members []*pdpb.Member) { } sort.Strings(urls) - oldURLs := c.GetURLs() + oldURLs := c.GetServiceURLs() // the url list is same. if reflect.DeepEqual(oldURLs, urls) { return diff --git a/client/resource_group/controller/config.go b/client/resource_group/controller/config.go index 1f575d15bf3..2095bc60601 100644 --- a/client/resource_group/controller/config.go +++ b/client/resource_group/controller/config.go @@ -32,8 +32,6 @@ var ( ) const ( - initialRequestUnits = 10000 - bufferRUs = 2000 // movingAvgFactor is the weight applied to a new "sample" of RU usage (with one // sample per mainLoopUpdateInterval). // @@ -42,38 +40,76 @@ const ( // 0.5^(1 second / mainLoopUpdateInterval) movingAvgFactor = 0.5 notifyFraction = 0.1 + tokenReserveFraction = 0.8 consumptionsReportingThreshold = 100 extendedReportingPeriodFactor = 4 // defaultGroupCleanupInterval is the interval to clean up the deleted resource groups in memory. - defaultGroupCleanupInterval = 10 * time.Minute + defaultGroupCleanupInterval = 5 * time.Minute // defaultGroupStateUpdateInterval is the interval to update the state of the resource groups. defaultGroupStateUpdateInterval = 1 * time.Second // targetPeriod indicate how long it is expected to cost token when acquiring token. - defaultTargetPeriod = 10 * time.Second + // According to the resource control Grafana panel and Prometheus sampling period, the period should be the factor of 15. + defaultTargetPeriod = 5 * time.Second + // defaultMaxWaitDuration is the max duration to wait for the token before throwing error. + defaultMaxWaitDuration = time.Second ) const ( - defaultReadBaseCost = 0.25 + + // 1 RU = 8 storage read requests + defaultReadBaseCost = 1. / 8 // 0.125 + // 1 RU = 2 storage read batch requests + defaultReadPerBatchBaseCost = 1. / 2 // 0.5 + // 1 RU = 1 storage write request defaultWriteBaseCost = 1 + // 1 RU = 1 storage write batch request + defaultWritePerBatchBaseCost = 1 // 1 RU = 64 KiB read bytes defaultReadCostPerByte = 1. / (64 * 1024) // 1 RU = 1 KiB written bytes defaultWriteCostPerByte = 1. / 1024 // 1 RU = 3 millisecond CPU time defaultCPUMsCost = 1. / 3 + + // Because the resource manager has not been deployed in microservice mode, + // do not enable this function. + defaultDegradedModeWaitDuration = "0s" + defaultAvgBatchProportion = 0.7 ) +// Config is the configuration of the resource manager controller which includes some option for client needed. +type Config struct { + // EnableDegradedMode is to control whether resource control client enable degraded mode when server is disconnect. + DegradedModeWaitDuration string `toml:"degraded-mode-wait-duration" json:"degraded-mode-wait-duration"` + + // RequestUnit is the configuration determines the coefficients of the RRU and WRU cost. + // This configuration should be modified carefully. + RequestUnit RequestUnitConfig `toml:"request-unit" json:"request-unit"` +} + +// DefaultConfig returns the default resource manager controller configuration. +func DefaultConfig() *Config { + return &Config{ + DegradedModeWaitDuration: defaultDegradedModeWaitDuration, + RequestUnit: DefaultRequestUnitConfig(), + } +} + // RequestUnitConfig is the configuration of the request units, which determines the coefficients of // the RRU and WRU cost. This configuration should be modified carefully. type RequestUnitConfig struct { // ReadBaseCost is the base cost for a read request. No matter how many bytes read/written or // the CPU times taken for a request, this cost is inevitable. ReadBaseCost float64 `toml:"read-base-cost" json:"read-base-cost"` + // ReadPerBatchBaseCost is the base cost for a read request with batch. + ReadPerBatchBaseCost float64 `toml:"read-per-batch-base-cost" json:"read-per-batch-base-cost"` // ReadCostPerByte is the cost for each byte read. It's 1 RU = 64 KiB by default. ReadCostPerByte float64 `toml:"read-cost-per-byte" json:"read-cost-per-byte"` // WriteBaseCost is the base cost for a write request. No matter how many bytes read/written or // the CPU times taken for a request, this cost is inevitable. WriteBaseCost float64 `toml:"write-base-cost" json:"write-base-cost"` + // WritePerBatchBaseCost is the base cost for a write request with batch. + WritePerBatchBaseCost float64 `toml:"write-per-batch-base-cost" json:"write-per-batch-base-cost"` // WriteCostPerByte is the cost for each byte written. It's 1 RU = 1 KiB by default. WriteCostPerByte float64 `toml:"write-cost-per-byte" json:"write-cost-per-byte"` // CPUMsCost is the cost for each millisecond of CPU time taken. @@ -82,45 +118,60 @@ type RequestUnitConfig struct { } // DefaultRequestUnitConfig returns the default request unit configuration. -func DefaultRequestUnitConfig() *RequestUnitConfig { - return &RequestUnitConfig{ - ReadBaseCost: defaultReadBaseCost, - ReadCostPerByte: defaultReadCostPerByte, - WriteBaseCost: defaultWriteBaseCost, - WriteCostPerByte: defaultWriteCostPerByte, - CPUMsCost: defaultCPUMsCost, +func DefaultRequestUnitConfig() RequestUnitConfig { + return RequestUnitConfig{ + ReadBaseCost: defaultReadBaseCost, + ReadPerBatchBaseCost: defaultReadPerBatchBaseCost, + ReadCostPerByte: defaultReadCostPerByte, + WriteBaseCost: defaultWriteBaseCost, + WritePerBatchBaseCost: defaultWritePerBatchBaseCost, + WriteCostPerByte: defaultWriteCostPerByte, + CPUMsCost: defaultCPUMsCost, } } -// Config is the configuration of the resource units, which gives the read/write request +// RUConfig is the configuration of the resource units, which gives the read/write request // units or request resource cost standards. It should be calculated by a given `RequestUnitConfig` // or `RequestResourceConfig`. -type Config struct { +type RUConfig struct { // RU model config - ReadBaseCost RequestUnit - ReadBytesCost RequestUnit - WriteBaseCost RequestUnit - WriteBytesCost RequestUnit - CPUMsCost RequestUnit + ReadBaseCost RequestUnit + ReadPerBatchBaseCost RequestUnit + ReadBytesCost RequestUnit + WriteBaseCost RequestUnit + WritePerBatchBaseCost RequestUnit + WriteBytesCost RequestUnit + CPUMsCost RequestUnit // The CPU statistics need to distinguish between different environments. - isSingleGroupByKeyspace bool + isSingleGroupByKeyspace bool + maxWaitDuration time.Duration + DegradedModeWaitDuration time.Duration } -// DefaultConfig returns the default configuration. -func DefaultConfig() *Config { - return GenerateConfig( - DefaultRequestUnitConfig(), +// DefaultRUConfig returns the default configuration. +func DefaultRUConfig() *RUConfig { + return GenerateRUConfig( + DefaultConfig(), ) } -// GenerateConfig generates the configuration by the given request unit configuration. -func GenerateConfig(ruConfig *RequestUnitConfig) *Config { - cfg := &Config{ - ReadBaseCost: RequestUnit(ruConfig.ReadBaseCost), - ReadBytesCost: RequestUnit(ruConfig.ReadCostPerByte), - WriteBaseCost: RequestUnit(ruConfig.WriteBaseCost), - WriteBytesCost: RequestUnit(ruConfig.WriteCostPerByte), - CPUMsCost: RequestUnit(ruConfig.CPUMsCost), +// GenerateRUConfig generates the configuration by the given request unit configuration. +func GenerateRUConfig(config *Config) *RUConfig { + cfg := &RUConfig{ + ReadBaseCost: RequestUnit(config.RequestUnit.ReadBaseCost), + ReadPerBatchBaseCost: RequestUnit(config.RequestUnit.ReadPerBatchBaseCost), + ReadBytesCost: RequestUnit(config.RequestUnit.ReadCostPerByte), + WriteBaseCost: RequestUnit(config.RequestUnit.WriteBaseCost), + WritePerBatchBaseCost: RequestUnit(config.RequestUnit.WritePerBatchBaseCost), + WriteBytesCost: RequestUnit(config.RequestUnit.WriteCostPerByte), + CPUMsCost: RequestUnit(config.RequestUnit.CPUMsCost), + maxWaitDuration: defaultMaxWaitDuration, + } + duration, err := time.ParseDuration(config.DegradedModeWaitDuration) + if err != nil { + cfg.DegradedModeWaitDuration, _ = time.ParseDuration(defaultDegradedModeWaitDuration) + } else { + cfg.DegradedModeWaitDuration = duration } return cfg } diff --git a/client/resource_group/controller/controller.go b/client/resource_group/controller/controller.go old mode 100644 new mode 100755 index f1fc729ad4b..245a9acfe2d --- a/client/resource_group/controller/controller.go +++ b/client/resource_group/controller/controller.go @@ -18,42 +18,62 @@ import ( "context" "encoding/json" "math" + "strings" "sync" "sync/atomic" "time" + "github.com/gogo/protobuf/proto" "github.com/pingcap/errors" + "github.com/pingcap/failpoint" + "github.com/pingcap/kvproto/pkg/meta_storagepb" rmpb "github.com/pingcap/kvproto/pkg/resource_manager" "github.com/pingcap/log" + "github.com/prometheus/client_golang/prometheus" pd "github.com/tikv/pd/client" "github.com/tikv/pd/client/errs" "go.uber.org/zap" + "golang.org/x/exp/slices" ) const ( - requestUnitConfigPath = "resource_group/ru_config" - defaultMaxWaitDuration = time.Second - maxRetry = 3 - maxNotificationChanLen = 200 + controllerConfigPath = "resource_group/controller" + maxRetry = 10 + retryInterval = 50 * time.Millisecond + maxNotificationChanLen = 200 + needTokensAmplification = 1.1 + trickleReserveDuration = 1250 * time.Millisecond + + watchRetryInterval = 30 * time.Second +) + +type selectType int + +const ( + periodicReport selectType = 0 + lowToken selectType = 1 ) // ResourceGroupKVInterceptor is used as quota limit controller for resource group using kv store. type ResourceGroupKVInterceptor interface { - // OnRequestWait is used to check whether resource group has enough tokens. It maybe needs wait some time. - OnRequestWait(ctx context.Context, resourceGroupName string, info RequestInfo) error - // OnResponse is used to consume tokens after receiving response - OnResponse(ctx context.Context, resourceGroupName string, req RequestInfo, resp ResponseInfo) error + // OnRequestWait is used to check whether resource group has enough tokens. It maybe needs to wait some time. + OnRequestWait(ctx context.Context, resourceGroupName string, info RequestInfo) (*rmpb.Consumption, *rmpb.Consumption, error) + // OnResponse is used to consume tokens after receiving response. + OnResponse(resourceGroupName string, req RequestInfo, resp ResponseInfo) (*rmpb.Consumption, error) + // IsBackgroundRequest If the resource group has background jobs, we should not record consumption and wait for it. + IsBackgroundRequest(ctx context.Context, resourceGroupName, requestResource string) bool } -// ResourceGroupProvider provides some api to interact with resource manager server。 +// ResourceGroupProvider provides some api to interact with resource manager server. type ResourceGroupProvider interface { - ListResourceGroups(ctx context.Context) ([]*rmpb.ResourceGroup, error) GetResourceGroup(ctx context.Context, resourceGroupName string) (*rmpb.ResourceGroup, error) AddResourceGroup(ctx context.Context, metaGroup *rmpb.ResourceGroup) (string, error) ModifyResourceGroup(ctx context.Context, metaGroup *rmpb.ResourceGroup) (string, error) DeleteResourceGroup(ctx context.Context, resourceGroupName string) (string, error) AcquireTokenBuckets(ctx context.Context, request *rmpb.TokenBucketsRequest) ([]*rmpb.TokenBucketResponse, error) LoadGlobalConfig(ctx context.Context, names []string, configPath string) ([]pd.GlobalConfigItem, int64, error) + LoadResourceGroups(ctx context.Context) ([]*rmpb.ResourceGroup, int64, error) + Watch(ctx context.Context, key []byte, opts ...pd.OpOption) (chan []*meta_storagepb.Event, error) } // ResourceControlCreateOption create a ResourceGroupsController with the optional settings. @@ -62,18 +82,25 @@ type ResourceControlCreateOption func(controller *ResourceGroupsController) // EnableSingleGroupByKeyspace is the option to enable single group by keyspace feature. func EnableSingleGroupByKeyspace() ResourceControlCreateOption { return func(controller *ResourceGroupsController) { - controller.config.isSingleGroupByKeyspace = true + controller.ruConfig.isSingleGroupByKeyspace = true + } +} + +// WithMaxWaitDuration is the option to set the max wait duration for acquiring token buckets. +func WithMaxWaitDuration(d time.Duration) ResourceControlCreateOption { + return func(controller *ResourceGroupsController) { + controller.ruConfig.maxWaitDuration = d } } var _ ResourceGroupKVInterceptor = (*ResourceGroupsController)(nil) -// ResourceGroupsController impls ResourceGroupKVInterceptor. +// ResourceGroupsController implements ResourceGroupKVInterceptor. type ResourceGroupsController struct { clientUniqueID uint64 provider ResourceGroupProvider groupsController sync.Map - config *Config + ruConfig *RUConfig loopCtx context.Context loopCancel func() @@ -86,22 +113,14 @@ type ResourceGroupsController struct { tokenResponseChan chan []*rmpb.TokenBucketResponse // When the token bucket of a resource group is updated, it will be sent to the channel. tokenBucketUpdateChan chan *groupCostController + responseDeadlineCh <-chan time.Time run struct { - now time.Time - lastRequestTimeLowRU time.Time - lastRequestTime time.Time - - // requestInProgress is true if we are in the process of sending a request. - // It gets set to false when we receive the response in the main loop, - // even in error cases. - requestInProgress bool - - // requestNeedsRetry is set if the last token bucket request encountered an - // error. This triggers a retry attempt on the next tick. - // - // Note: requestNeedsRetry and requestInProgress are never true at the same time. - requestNeedsRetry bool + responseDeadline *time.Timer + inDegradedMode bool + // currentRequests is used to record the request and resource group. + // Currently, we don't do multiple `AcquireTokenBuckets`` at the same time, so there are no concurrency problems with `currentRequests`. + currentRequests []*rmpb.TokenBucketRequest } } @@ -113,18 +132,19 @@ func NewResourceGroupController( requestUnitConfig *RequestUnitConfig, opts ...ResourceControlCreateOption, ) (*ResourceGroupsController, error) { - if requestUnitConfig == nil { - var err error - requestUnitConfig, err = loadRequestUnitConfig(ctx, provider) - if err != nil { - return nil, err - } + config, err := loadServerConfig(ctx, provider) + if err != nil { + return nil, err } - config := GenerateConfig(requestUnitConfig) + if requestUnitConfig != nil { + config.RequestUnit = *requestUnitConfig + } + log.Info("load resource controller config", zap.Reflect("config", config)) + ruConfig := GenerateRUConfig(config) controller := &ResourceGroupsController{ clientUniqueID: clientUniqueID, provider: provider, - config: config, + ruConfig: ruConfig, lowTokenNotifyChan: make(chan struct{}, 1), tokenResponseChan: make(chan []*rmpb.TokenBucketResponse, 1), tokenBucketUpdateChan: make(chan *groupCostController, maxNotificationChanLen), @@ -132,29 +152,30 @@ func NewResourceGroupController( for _, opt := range opts { opt(controller) } - controller.calculators = []ResourceCalculator{newKVCalculator(controller.config), newSQLCalculator(controller.config)} + controller.calculators = []ResourceCalculator{newKVCalculator(controller.ruConfig), newSQLCalculator(controller.ruConfig)} return controller, nil } -func loadRequestUnitConfig(ctx context.Context, provider ResourceGroupProvider) (*RequestUnitConfig, error) { - items, _, err := provider.LoadGlobalConfig(ctx, nil, requestUnitConfigPath) +func loadServerConfig(ctx context.Context, provider ResourceGroupProvider) (*Config, error) { + items, _, err := provider.LoadGlobalConfig(ctx, nil, controllerConfigPath) if err != nil { return nil, err } if len(items) == 0 { - return nil, errors.Errorf("failed to load the ru config from remote server") + log.Warn("[resource group controller] server does not save config, load config failed") + return DefaultConfig(), nil } - ruConfig := &RequestUnitConfig{} - err = json.Unmarshal(items[0].PayLoad, ruConfig) + config := &Config{} + err = json.Unmarshal(items[0].PayLoad, config) if err != nil { return nil, err } - return ruConfig, nil + return config, nil } // GetConfig returns the config of controller. It's only used for test. -func (c *ResourceGroupsController) GetConfig() *Config { - return c.config +func (c *ResourceGroupsController) GetConfig() *RUConfig { + return c.ruConfig } // Source List @@ -165,43 +186,120 @@ const ( // Start starts ResourceGroupController service. func (c *ResourceGroupsController) Start(ctx context.Context) { - c.initRunState() c.loopCtx, c.loopCancel = context.WithCancel(ctx) go func() { + if c.ruConfig.DegradedModeWaitDuration > 0 { + c.run.responseDeadline = time.NewTimer(c.ruConfig.DegradedModeWaitDuration) + c.run.responseDeadline.Stop() + defer c.run.responseDeadline.Stop() + } cleanupTicker := time.NewTicker(defaultGroupCleanupInterval) defer cleanupTicker.Stop() stateUpdateTicker := time.NewTicker(defaultGroupStateUpdateInterval) defer stateUpdateTicker.Stop() + emergencyTokenAcquisitionTicker := time.NewTicker(defaultTargetPeriod) + defer emergencyTokenAcquisitionTicker.Stop() + + failpoint.Inject("fastCleanup", func() { + cleanupTicker.Stop() + cleanupTicker = time.NewTicker(100 * time.Millisecond) + // because of checking `gc.run.consumption` in cleanupTicker, + // so should also change the stateUpdateTicker. + stateUpdateTicker.Stop() + stateUpdateTicker = time.NewTicker(200 * time.Millisecond) + }) + failpoint.Inject("acceleratedReportingPeriod", func() { + stateUpdateTicker.Stop() + stateUpdateTicker = time.NewTicker(time.Millisecond * 100) + }) + + _, revision, err := c.provider.LoadResourceGroups(ctx) + if err != nil { + log.Warn("load resource group revision failed", zap.Error(err)) + } + var watchChannel chan []*meta_storagepb.Event + if !c.ruConfig.isSingleGroupByKeyspace { + watchChannel, err = c.provider.Watch(ctx, pd.GroupSettingsPathPrefixBytes, pd.WithRev(revision), pd.WithPrefix()) + } + watchRetryTimer := time.NewTimer(watchRetryInterval) + if err == nil || c.ruConfig.isSingleGroupByKeyspace { + watchRetryTimer.Stop() + } + defer watchRetryTimer.Stop() for { select { case <-c.loopCtx.Done(): + resourceGroupStatusGauge.Reset() return + case <-c.responseDeadlineCh: + c.run.inDegradedMode = true + c.executeOnAllGroups((*groupCostController).applyDegradedMode) + log.Warn("[resource group controller] enter degraded mode") case resp := <-c.tokenResponseChan: - c.run.requestInProgress = false if resp != nil { - c.updateRunState() + c.executeOnAllGroups((*groupCostController).updateRunState) c.handleTokenBucketResponse(resp) - } else { - // A nil response indicates a failure (which would have been logged). - c.run.requestNeedsRetry = true } + c.run.currentRequests = nil case <-cleanupTicker.C: - if err := c.cleanUpResourceGroup(c.loopCtx); err != nil { - log.Error("[resource group controller] clean up resource groups failed", zap.Error(err)) - } + c.cleanUpResourceGroup() case <-stateUpdateTicker.C: - c.updateRunState() - c.updateAvgRequestResourcePerSec() - if c.run.requestNeedsRetry || c.shouldReportConsumption() { - c.run.requestNeedsRetry = false - c.collectTokenBucketRequests(c.loopCtx, FromPeriodReport, false /* select all */) + c.executeOnAllGroups((*groupCostController).updateRunState) + c.executeOnAllGroups((*groupCostController).updateAvgRequestResourcePerSec) + if len(c.run.currentRequests) == 0 { + c.collectTokenBucketRequests(c.loopCtx, FromPeriodReport, periodicReport /* select resource groups which should be reported periodically */) } case <-c.lowTokenNotifyChan: - c.updateRunState() - c.updateAvgRequestResourcePerSec() - if !c.run.requestInProgress { - c.collectTokenBucketRequests(c.loopCtx, FromLowRU, true /* only select low tokens resource group */) + c.executeOnAllGroups((*groupCostController).updateRunState) + c.executeOnAllGroups((*groupCostController).updateAvgRequestResourcePerSec) + if len(c.run.currentRequests) == 0 { + c.collectTokenBucketRequests(c.loopCtx, FromLowRU, lowToken /* select low tokens resource group */) + } + if c.run.inDegradedMode { + c.executeOnAllGroups((*groupCostController).applyDegradedMode) + } + case <-emergencyTokenAcquisitionTicker.C: + c.executeOnAllGroups((*groupCostController).resetEmergencyTokenAcquisition) + case resp, ok := <-watchChannel: + failpoint.Inject("disableWatch", func() { + if c.ruConfig.isSingleGroupByKeyspace { + panic("disableWatch") + } + }) + if !ok { + watchChannel = nil + watchRetryTimer.Reset(watchRetryInterval) + failpoint.Inject("watchStreamError", func() { + watchRetryTimer.Reset(20 * time.Millisecond) + }) + continue + } + for _, item := range resp { + revision = item.Kv.ModRevision + group := &rmpb.ResourceGroup{} + if err := proto.Unmarshal(item.Kv.Value, group); err != nil { + continue + } + switch item.Type { + case meta_storagepb.Event_PUT: + if item, ok := c.groupsController.Load(group.Name); ok { + gc := item.(*groupCostController) + gc.modifyMeta(group) + } + case meta_storagepb.Event_DELETE: + if _, ok := c.groupsController.LoadAndDelete(group.Name); ok { + resourceGroupStatusGauge.DeleteLabelValues(group.Name) + } + } + } + case <-watchRetryTimer.C: + watchChannel, err = c.provider.Watch(ctx, pd.GroupSettingsPathPrefixBytes, pd.WithRev(revision), pd.WithPrefix()) + if err != nil { + watchRetryTimer.Reset(watchRetryInterval) + failpoint.Inject("watchStreamError", func() { + watchRetryTimer.Reset(20 * time.Millisecond) + }) } case gc := <-c.tokenBucketUpdateChan: now := gc.run.now @@ -232,13 +330,16 @@ func (c *ResourceGroupsController) tryGetResourceGroup(ctx context.Context, name if err != nil { return nil, err } + if group == nil { + return nil, errors.Errorf("%s does not exists", name) + } // Check again to prevent initializing the same resource group concurrently. if tmp, ok := c.groupsController.Load(name); ok { gc := tmp.(*groupCostController) return gc, nil } // Initialize the resource group controller. - gc, err := newGroupCostController(group, c.config, c.lowTokenNotifyChan, c.tokenBucketUpdateChan) + gc, err := newGroupCostController(group, c.ruConfig, c.lowTokenNotifyChan, c.tokenBucketUpdateChan) if err != nil { return nil, err } @@ -247,82 +348,52 @@ func (c *ResourceGroupsController) tryGetResourceGroup(ctx context.Context, name // Check again to prevent initializing the same resource group concurrently. tmp, loaded := c.groupsController.LoadOrStore(group.GetName(), gc) if !loaded { + resourceGroupStatusGauge.WithLabelValues(name).Set(1) log.Info("[resource group controller] create resource group cost controller", zap.String("name", group.GetName())) } return tmp.(*groupCostController), nil } -func (c *ResourceGroupsController) cleanUpResourceGroup(ctx context.Context) error { - groups, err := c.provider.ListResourceGroups(ctx) - if err != nil { - return errs.ErrClientListResourceGroup.FastGenByArgs(err.Error()) - } - latestGroups := make(map[string]struct{}) - for _, group := range groups { - latestGroups[group.GetName()] = struct{}{} - } - // TODO: maybe we should also clean up those resource groups that have not been used for a long time. +func (c *ResourceGroupsController) cleanUpResourceGroup() { c.groupsController.Range(func(key, value any) bool { resourceGroupName := key.(string) - if _, ok := latestGroups[resourceGroupName]; !ok { - c.groupsController.Delete(key) + gc := value.(*groupCostController) + // Check for stale resource groups, which will be deleted when consumption is continuously unchanged. + gc.mu.Lock() + latestConsumption := *gc.mu.consumption + gc.mu.Unlock() + if equalRU(latestConsumption, *gc.run.consumption) { + if gc.tombstone { + c.groupsController.Delete(resourceGroupName) + resourceGroupStatusGauge.DeleteLabelValues(resourceGroupName) + return true + } + gc.tombstone = true + } else { + gc.tombstone = false } return true }) - return nil -} - -func (c *ResourceGroupsController) initRunState() { - now := time.Now() - c.run.now = now - c.run.lastRequestTime = now - c.run.lastRequestTimeLowRU = now } -func (c *ResourceGroupsController) updateRunState() { - c.run.now = time.Now() +func (c *ResourceGroupsController) executeOnAllGroups(f func(controller *groupCostController)) { c.groupsController.Range(func(name, value any) bool { - gc := value.(*groupCostController) - gc.updateRunState() + f(value.(*groupCostController)) return true }) } -func (c *ResourceGroupsController) shouldReportConsumption() bool { - if c.run.requestInProgress { - return false - } - maxDuration := func(a, b time.Duration) time.Duration { - if a > b { - return a - } - return b - } - timeSinceLastRequest := maxDuration(c.run.now.Sub(c.run.lastRequestTime), c.run.now.Sub(c.run.lastRequestTimeLowRU)) - if timeSinceLastRequest >= defaultTargetPeriod { - if timeSinceLastRequest >= extendedReportingPeriodFactor*defaultTargetPeriod { - return true +func (c *ResourceGroupsController) handleTokenBucketResponse(resp []*rmpb.TokenBucketResponse) { + if c.responseDeadlineCh != nil { + if c.run.responseDeadline.Stop() { + select { + case <-c.run.responseDeadline.C: + default: + } } - ret := false - c.groupsController.Range(func(name, value any) bool { - gc := value.(*groupCostController) - ret = ret || gc.shouldReportConsumption() - return !ret - }) - return ret + c.responseDeadlineCh = nil } - return false -} - -func (c *ResourceGroupsController) updateAvgRequestResourcePerSec() { - c.groupsController.Range(func(name, value any) bool { - gc := value.(*groupCostController) - gc.updateAvgRequestResourcePerSec() - return true - }) -} - -func (c *ResourceGroupsController) handleTokenBucketResponse(resp []*rmpb.TokenBucketResponse) { + c.run.inDegradedMode = false for _, res := range resp { name := res.GetResourceGroupName() v, ok := c.groupsController.Load(name) @@ -335,82 +406,142 @@ func (c *ResourceGroupsController) handleTokenBucketResponse(resp []*rmpb.TokenB } } -func (c *ResourceGroupsController) collectTokenBucketRequests(ctx context.Context, source string, onlySelectLow bool) { - requests := make([]*rmpb.TokenBucketRequest, 0) +func (c *ResourceGroupsController) collectTokenBucketRequests(ctx context.Context, source string, typ selectType) { + c.run.currentRequests = make([]*rmpb.TokenBucketRequest, 0) c.groupsController.Range(func(name, value any) bool { gc := value.(*groupCostController) - request := gc.collectRequestAndConsumption(onlySelectLow) + request := gc.collectRequestAndConsumption(typ) if request != nil { - requests = append(requests, request) + c.run.currentRequests = append(c.run.currentRequests, request) + gc.tokenRequestCounter.Inc() } return true }) - if len(requests) > 0 { - c.sendTokenBucketRequests(ctx, requests, source) + if len(c.run.currentRequests) > 0 { + c.sendTokenBucketRequests(ctx, c.run.currentRequests, source) } } func (c *ResourceGroupsController) sendTokenBucketRequests(ctx context.Context, requests []*rmpb.TokenBucketRequest, source string) { now := time.Now() - switch source { - case FromLowRU: - c.run.lastRequestTimeLowRU = now - case FromPeriodReport: - c.run.lastRequestTime = now - } - c.run.requestInProgress = true req := &rmpb.TokenBucketsRequest{ Requests: requests, TargetRequestPeriodMs: uint64(defaultTargetPeriod / time.Millisecond), + ClientUniqueId: c.clientUniqueID, + } + if c.ruConfig.DegradedModeWaitDuration > 0 && c.responseDeadlineCh == nil { + c.run.responseDeadline.Reset(c.ruConfig.DegradedModeWaitDuration) + c.responseDeadlineCh = c.run.responseDeadline.C } go func() { log.Debug("[resource group controller] send token bucket request", zap.Time("now", now), zap.Any("req", req.Requests), zap.String("source", source)) resp, err := c.provider.AcquireTokenBuckets(ctx, req) + latency := time.Since(now) if err != nil { // Don't log any errors caused by the stopper canceling the context. if !errors.ErrorEqual(err, context.Canceled) { log.L().Sugar().Infof("[resource group controller] token bucket rpc error: %v", err) } resp = nil + failedTokenRequestDuration.Observe(latency.Seconds()) + } else { + successfulTokenRequestDuration.Observe(latency.Seconds()) } - log.Debug("[resource group controller] token bucket response", zap.Time("now", time.Now()), zap.Any("resp", resp), zap.String("source", source), zap.Duration("latency", time.Since(now))) + log.Debug("[resource group controller] token bucket response", zap.Time("now", time.Now()), zap.Any("resp", resp), zap.String("source", source), zap.Duration("latency", latency)) c.tokenResponseChan <- resp }() } -// OnRequestWait is used to check whether resource group has enough tokens. It maybe needs wait some time. +// OnRequestWait is used to check whether resource group has enough tokens. It maybe needs to wait some time. func (c *ResourceGroupsController) OnRequestWait( ctx context.Context, resourceGroupName string, info RequestInfo, -) (err error) { +) (*rmpb.Consumption, *rmpb.Consumption, error) { gc, err := c.tryGetResourceGroup(ctx, resourceGroupName) if err != nil { - return err + return nil, nil, err } return gc.onRequestWait(ctx, info) } // OnResponse is used to consume tokens after receiving response -func (c *ResourceGroupsController) OnResponse(_ context.Context, resourceGroupName string, req RequestInfo, resp ResponseInfo) error { +func (c *ResourceGroupsController) OnResponse( + resourceGroupName string, req RequestInfo, resp ResponseInfo, +) (*rmpb.Consumption, error) { tmp, ok := c.groupsController.Load(resourceGroupName) if !ok { - log.Warn("[resource group controller] resource group name does not exist", zap.String("resourceGroupName", resourceGroupName)) + log.Warn("[resource group controller] resource group name does not exist", zap.String("name", resourceGroupName)) + return &rmpb.Consumption{}, nil } - gc := tmp.(*groupCostController) - gc.onResponse(req, resp) - return nil + return tmp.(*groupCostController).onResponse(req, resp) } -type groupCostController struct { - *rmpb.ResourceGroup - mainCfg *Config - calculators []ResourceCalculator - mode rmpb.GroupMode +// IsBackgroundRequest If the resource group has background jobs, we should not record consumption and wait for it. +func (c *ResourceGroupsController) IsBackgroundRequest(ctx context.Context, + resourceGroupName, requestResource string) bool { + gc, err := c.tryGetResourceGroup(ctx, resourceGroupName) + if err != nil { + failedRequestCounter.WithLabelValues(resourceGroupName).Inc() + return false + } + + return c.checkBackgroundSettings(ctx, gc.getMeta().BackgroundSettings, requestResource) +} + +func (c *ResourceGroupsController) checkBackgroundSettings(ctx context.Context, bg *rmpb.BackgroundSettings, requestResource string) bool { + // fallback to default resource group. + if bg == nil { + resourceGroupName := "default" + gc, err := c.tryGetResourceGroup(ctx, resourceGroupName) + if err != nil { + failedRequestCounter.WithLabelValues(resourceGroupName).Inc() + return false + } + bg = gc.getMeta().BackgroundSettings + } + if bg == nil || len(requestResource) == 0 || len(bg.JobTypes) == 0 { + return false + } + + if idx := strings.LastIndex(requestResource, "_"); idx != -1 { + return slices.Contains(bg.JobTypes, requestResource[idx+1:]) + } + + return false +} + +// GetResourceGroup returns the meta setting of the given resource group name. +func (c *ResourceGroupsController) GetResourceGroup(resourceGroupName string) (*rmpb.ResourceGroup, error) { + gc, err := c.tryGetResourceGroup(c.loopCtx, resourceGroupName) + if err != nil { + return nil, err + } + return gc.getMeta(), nil +} + +type groupCostController struct { + // invariant attributes + name string + mode rmpb.GroupMode + mainCfg *RUConfig + // meta info + meta *rmpb.ResourceGroup + metaLock sync.RWMutex + + // following fields are used for token limiter. + calculators []ResourceCalculator handleRespFunc func(*rmpb.TokenBucketResponse) + successfulRequestDuration prometheus.Observer + requestRetryCounter prometheus.Counter + failedRequestCounter prometheus.Counter + tokenRequestCounter prometheus.Counter + mu struct { sync.Mutex - consumption *rmpb.Consumption + consumption *rmpb.Consumption + storeCounter map[uint64]*rmpb.Consumption + globalCounter *rmpb.Consumption } // fast path to make once token limit with un-limit burst. @@ -421,7 +552,13 @@ type groupCostController struct { // run contains the state that is updated by the main loop. run struct { - now time.Time + now time.Time + lastRequestTime time.Time + + // requestInProgress is set true when sending token bucket request. + // And it is set false when receiving token bucket response. + // This triggers a retry attempt on the next tick. + requestInProgress bool // targetPeriod stores the value of the TargetPeriodSetting setting at the // last update. @@ -443,9 +580,13 @@ type groupCostController struct { resourceTokens map[rmpb.RawResourceType]*tokenCounter requestUnitTokens map[rmpb.RequestUnitType]*tokenCounter } + + tombstone bool } type tokenCounter struct { + getTokenBucketFunc func() *rmpb.TokenBucket + // avgRUPerSec is an exponentially-weighted moving average of the RU // consumption per second; used to estimate the RU requirements for the next // request. @@ -465,11 +606,13 @@ type tokenCounter struct { lastRate float64 limiter *Limiter + + inDegradedMode bool } func newGroupCostController( group *rmpb.ResourceGroup, - mainCfg *Config, + mainCfg *RUConfig, lowRUNotifyChan chan struct{}, tokenBucketUpdateChan chan *groupCostController, ) (*groupCostController, error) { @@ -481,15 +624,19 @@ func newGroupCostController( default: return nil, errs.ErrClientResourceGroupConfigUnavailable.FastGenByArgs("not supports the resource type") } - gc := &groupCostController{ - ResourceGroup: group, - mainCfg: mainCfg, + meta: group, + name: group.Name, + mainCfg: mainCfg, + mode: group.GetMode(), + successfulRequestDuration: successfulRequestDuration.WithLabelValues(group.Name), + failedRequestCounter: failedRequestCounter.WithLabelValues(group.Name), + requestRetryCounter: requestRetryCounter.WithLabelValues(group.Name), + tokenRequestCounter: resourceGroupTokenRequestCounter.WithLabelValues(group.Name), calculators: []ResourceCalculator{ newKVCalculator(mainCfg), newSQLCalculator(mainCfg), }, - mode: group.GetMode(), tokenBucketUpdateChan: tokenBucketUpdateChan, lowRUNotifyChan: lowRUNotifyChan, burstable: &atomic.Bool{}, @@ -503,57 +650,80 @@ func newGroupCostController( } gc.mu.consumption = &rmpb.Consumption{} + gc.mu.storeCounter = make(map[uint64]*rmpb.Consumption) + gc.mu.globalCounter = &rmpb.Consumption{} return gc, nil } func (gc *groupCostController) initRunState() { now := time.Now() gc.run.now = now + gc.run.lastRequestTime = now.Add(-defaultTargetPeriod) gc.run.targetPeriod = defaultTargetPeriod gc.run.consumption = &rmpb.Consumption{} gc.run.lastRequestConsumption = &rmpb.Consumption{SqlLayerCpuTimeMs: getSQLProcessCPUTime(gc.mainCfg.isSingleGroupByKeyspace)} + isBurstable := true cfgFunc := func(tb *rmpb.TokenBucket) tokenBucketReconfigureArgs { + initialToken := float64(tb.Settings.FillRate) cfg := tokenBucketReconfigureArgs{ - NewTokens: initialRequestUnits, + NewTokens: initialToken, NewBurst: tb.Settings.BurstLimit, // This is to trigger token requests as soon as resource group start consuming tokens. - NotifyThreshold: math.Max(initialRequestUnits-float64(tb.Settings.FillRate)*0.2, 1), + NotifyThreshold: math.Max(initialToken*tokenReserveFraction, 1), } if cfg.NewBurst >= 0 { cfg.NewBurst = 0 } + if tb.Settings.BurstLimit >= 0 { + isBurstable = false + } return cfg } + gc.metaLock.RLock() + defer gc.metaLock.RUnlock() switch gc.mode { case rmpb.GroupMode_RUMode: gc.run.requestUnitTokens = make(map[rmpb.RequestUnitType]*tokenCounter) for typ := range requestUnitLimitTypeList { - tb := getRUTokenBucketSetting(gc.ResourceGroup, typ) - cfg := cfgFunc(tb) - limiter := NewLimiterWithCfg(now, cfg, gc.lowRUNotifyChan) + limiter := NewLimiterWithCfg(now, cfgFunc(getRUTokenBucketSetting(gc.meta, typ)), gc.lowRUNotifyChan) counter := &tokenCounter{ limiter: limiter, avgRUPerSec: 0, avgLastTime: now, + getTokenBucketFunc: func() *rmpb.TokenBucket { + return getRUTokenBucketSetting(gc.meta, typ) + }, } gc.run.requestUnitTokens[typ] = counter } case rmpb.GroupMode_RawMode: gc.run.resourceTokens = make(map[rmpb.RawResourceType]*tokenCounter) for typ := range requestResourceLimitTypeList { - tb := getRawResourceTokenBucketSetting(gc.ResourceGroup, typ) - cfg := cfgFunc(tb) - limiter := NewLimiterWithCfg(now, cfg, gc.lowRUNotifyChan) + limiter := NewLimiterWithCfg(now, cfgFunc(getRawResourceTokenBucketSetting(gc.meta, typ)), gc.lowRUNotifyChan) counter := &tokenCounter{ limiter: limiter, avgRUPerSec: 0, avgLastTime: now, + getTokenBucketFunc: func() *rmpb.TokenBucket { + return getRawResourceTokenBucketSetting(gc.meta, typ) + }, } gc.run.resourceTokens[typ] = counter } } + gc.burstable.Store(isBurstable) +} + +// applyDegradedMode is used to apply degraded mode for resource group which is in low-process. +func (gc *groupCostController) applyDegradedMode() { + switch gc.mode { + case rmpb.GroupMode_RawMode: + gc.applyBasicConfigForRawResourceTokenCounter() + case rmpb.GroupMode_RUMode: + gc.applyBasicConfigForRUTokenCounters() + } } func (gc *groupCostController) updateRunState() { @@ -564,7 +734,7 @@ func (gc *groupCostController) updateRunState() { } *gc.run.consumption = *gc.mu.consumption gc.mu.Unlock() - log.Debug("[resource group controller] update run state", zap.Any("request unit consumption", gc.run.consumption)) + log.Debug("[resource group controller] update run state", zap.Any("request-unit-consumption", gc.run.consumption)) gc.run.now = newTime } @@ -577,6 +747,19 @@ func (gc *groupCostController) updateAvgRequestResourcePerSec() { } } +func (gc *groupCostController) resetEmergencyTokenAcquisition() { + switch gc.mode { + case rmpb.GroupMode_RawMode: + for _, counter := range gc.run.resourceTokens { + counter.limiter.ResetRemainingNotifyTimes() + } + case rmpb.GroupMode_RUMode: + for _, counter := range gc.run.requestUnitTokens { + counter.limiter.ResetRemainingNotifyTimes() + } + } +} + func (gc *groupCostController) handleTokenBucketUpdateEvent(ctx context.Context, now time.Time) { switch gc.mode { case rmpb.GroupMode_RawMode: @@ -632,7 +815,7 @@ func (gc *groupCostController) updateAvgRaWResourcePerSec() { if !gc.calcAvg(counter, getRawResourceValueFromConsumption(gc.run.consumption, typ)) { continue } - log.Debug("[resource group controller] update avg raw resource per sec", zap.String("name", gc.Name), zap.String("type", rmpb.RawResourceType_name[int32(typ)]), zap.Float64("avgRUPerSec", counter.avgRUPerSec)) + log.Debug("[resource group controller] update avg raw resource per sec", zap.String("name", gc.name), zap.String("type", rmpb.RawResourceType_name[int32(typ)]), zap.Float64("avg-ru-per-sec", counter.avgRUPerSec)) } gc.burstable.Store(isBurstable) } @@ -646,35 +829,57 @@ func (gc *groupCostController) updateAvgRUPerSec() { if !gc.calcAvg(counter, getRUValueFromConsumption(gc.run.consumption, typ)) { continue } - log.Debug("[resource group controller] update avg ru per sec", zap.String("name", gc.Name), zap.String("type", rmpb.RequestUnitType_name[int32(typ)]), zap.Float64("avgRUPerSec", counter.avgRUPerSec)) + log.Debug("[resource group controller] update avg ru per sec", zap.String("name", gc.name), zap.String("type", rmpb.RequestUnitType_name[int32(typ)]), zap.Float64("avg-ru-per-sec", counter.avgRUPerSec)) } gc.burstable.Store(isBurstable) } func (gc *groupCostController) calcAvg(counter *tokenCounter, new float64) bool { deltaDuration := gc.run.now.Sub(counter.avgLastTime) - if deltaDuration <= 500*time.Millisecond { - return false - } + failpoint.Inject("acceleratedReportingPeriod", func() { + deltaDuration = 100 * time.Millisecond + }) delta := (new - counter.avgRUPerSecLastRU) / deltaDuration.Seconds() counter.avgRUPerSec = movingAvgFactor*counter.avgRUPerSec + (1-movingAvgFactor)*delta + failpoint.Inject("acceleratedSpeedTrend", func() { + if delta > 0 { + counter.avgRUPerSec = 1000 + } else { + counter.avgRUPerSec = 0 + } + }) counter.avgLastTime = gc.run.now counter.avgRUPerSecLastRU = new return true } func (gc *groupCostController) shouldReportConsumption() bool { - switch gc.Mode { - case rmpb.GroupMode_RUMode: - for typ := range requestUnitLimitTypeList { - if getRUValueFromConsumption(gc.run.consumption, typ)-getRUValueFromConsumption(gc.run.lastRequestConsumption, typ) >= consumptionsReportingThreshold { - return true - } + if !gc.run.initialRequestCompleted { + return true + } + timeSinceLastRequest := gc.run.now.Sub(gc.run.lastRequestTime) + failpoint.Inject("acceleratedReportingPeriod", func() { + timeSinceLastRequest = extendedReportingPeriodFactor * defaultTargetPeriod + }) + // Due to `gc.run.lastRequestTime` update operations late in this logic, + // so `timeSinceLastRequest` is less than defaultGroupStateUpdateInterval a little bit, lead to actual report period is greater than defaultTargetPeriod. + // Add defaultGroupStateUpdateInterval/2 as duration buffer to avoid it. + if timeSinceLastRequest+defaultGroupStateUpdateInterval/2 >= defaultTargetPeriod { + if timeSinceLastRequest >= extendedReportingPeriodFactor*defaultTargetPeriod { + return true } - case rmpb.GroupMode_RawMode: - for typ := range requestResourceLimitTypeList { - if getRawResourceValueFromConsumption(gc.run.consumption, typ)-getRawResourceValueFromConsumption(gc.run.lastRequestConsumption, typ) >= consumptionsReportingThreshold { - return true + switch gc.mode { + case rmpb.GroupMode_RUMode: + for typ := range requestUnitLimitTypeList { + if getRUValueFromConsumption(gc.run.consumption, typ)-getRUValueFromConsumption(gc.run.lastRequestConsumption, typ) >= consumptionsReportingThreshold { + return true + } + } + case rmpb.GroupMode_RawMode: + for typ := range requestResourceLimitTypeList { + if getRawResourceValueFromConsumption(gc.run.consumption, typ)-getRawResourceValueFromConsumption(gc.run.lastRequestConsumption, typ) >= consumptionsReportingThreshold { + return true + } } } } @@ -682,18 +887,9 @@ func (gc *groupCostController) shouldReportConsumption() bool { } func (gc *groupCostController) handleTokenBucketResponse(resp *rmpb.TokenBucketResponse) { + gc.run.requestInProgress = false gc.handleRespFunc(resp) - if !gc.run.initialRequestCompleted { - gc.run.initialRequestCompleted = true - // This is the first successful request. Take back the initial RUs that we - // used to pre-fill the bucket. - for _, counter := range gc.run.resourceTokens { - counter.limiter.RemoveTokens(gc.run.now, initialRequestUnits) - } - for _, counter := range gc.run.requestUnitTokens { - counter.limiter.RemoveTokens(gc.run.now, initialRequestUnits) - } - } + gc.run.initialRequestCompleted = true } func (gc *groupCostController) handleRawResourceTokenResponse(resp *rmpb.TokenBucketResponse) { @@ -720,8 +916,44 @@ func (gc *groupCostController) handleRUTokenResponse(resp *rmpb.TokenBucketRespo } } +func (gc *groupCostController) applyBasicConfigForRUTokenCounters() { + for typ, counter := range gc.run.requestUnitTokens { + if !counter.limiter.IsLowTokens() { + continue + } + if counter.inDegradedMode { + continue + } + counter.inDegradedMode = true + initCounterNotify(counter) + var cfg tokenBucketReconfigureArgs + fillRate := counter.getTokenBucketFunc().Settings.FillRate + cfg.NewBurst = int64(fillRate) + cfg.NewRate = float64(fillRate) + failpoint.Inject("degradedModeRU", func() { + cfg.NewRate = 99999999 + }) + counter.limiter.Reconfigure(gc.run.now, cfg, resetLowProcess()) + log.Info("[resource group controller] resource token bucket enter degraded mode", zap.String("resource-group", gc.name), zap.String("type", rmpb.RequestUnitType_name[int32(typ)])) + } +} + +func (gc *groupCostController) applyBasicConfigForRawResourceTokenCounter() { + for _, counter := range gc.run.resourceTokens { + if !counter.limiter.IsLowTokens() { + continue + } + initCounterNotify(counter) + var cfg tokenBucketReconfigureArgs + fillRate := counter.getTokenBucketFunc().Settings.FillRate + cfg.NewBurst = int64(fillRate) + cfg.NewRate = float64(fillRate) + counter.limiter.Reconfigure(gc.run.now, cfg, resetLowProcess()) + } +} + func (gc *groupCostController) modifyTokenCounter(counter *tokenCounter, bucket *rmpb.TokenBucket, trickleTimeMs int64) { - granted := bucket.Tokens + granted := bucket.GetTokens() if !counter.lastDeadline.IsZero() { // If last request came with a trickle duration, we may have RUs that were // not made available to the bucket yet; throw them together with the newly @@ -730,30 +962,19 @@ func (gc *groupCostController) modifyTokenCounter(counter *tokenCounter, bucket granted += counter.lastRate * since.Seconds() } } - counter.notify.mu.Lock() - if counter.notify.setupNotificationTimer != nil { - counter.notify.setupNotificationTimer.Stop() - counter.notify.setupNotificationTimer = nil - counter.notify.setupNotificationCh = nil - } - counter.notify.mu.Unlock() - notifyThreshold := granted * notifyFraction - if notifyThreshold < bufferRUs { - notifyThreshold = bufferRUs - } - + initCounterNotify(counter) + counter.inDegradedMode = false var cfg tokenBucketReconfigureArgs cfg.NewBurst = bucket.GetSettings().GetBurstLimit() - // when trickleTimeMs equals zero, server has enough tokens and does not need to + // When trickleTimeMs equals zero, server has enough tokens and does not need to // limit client consume token. So all token is granted to client right now. if trickleTimeMs == 0 { cfg.NewTokens = granted cfg.NewRate = float64(bucket.GetSettings().FillRate) - cfg.NotifyThreshold = notifyThreshold counter.lastDeadline = time.Time{} - // In the non-trickle case, clients can be allowed to accumulate more tokens. - if cfg.NewBurst >= 0 { - cfg.NewBurst = 0 + cfg.NotifyThreshold = math.Min(granted+counter.limiter.AvailableTokens(gc.run.now), counter.avgRUPerSec*defaultTargetPeriod.Seconds()) * notifyFraction + if cfg.NewBurst < 0 { + cfg.NewTokens = float64(counter.getTokenBucketFunc().Settings.FillRate) } } else { // Otherwise the granted token is delivered to the client by fill rate. @@ -762,14 +983,14 @@ func (gc *groupCostController) modifyTokenCounter(counter *tokenCounter, bucket deadline := gc.run.now.Add(trickleDuration) cfg.NewRate = float64(bucket.GetSettings().FillRate) + granted/trickleDuration.Seconds() - timerDuration := trickleDuration - time.Second + timerDuration := trickleDuration - trickleReserveDuration if timerDuration <= 0 { - timerDuration = (trickleDuration + time.Second) / 2 + timerDuration = (trickleDuration + trickleReserveDuration) / 2 } counter.notify.mu.Lock() counter.notify.setupNotificationTimer = time.NewTimer(timerDuration) counter.notify.setupNotificationCh = counter.notify.setupNotificationTimer.C - counter.notify.setupNotificationThreshold = notifyThreshold + counter.notify.setupNotificationThreshold = 1 counter.notify.mu.Unlock() counter.lastDeadline = deadline select { @@ -779,21 +1000,37 @@ func (gc *groupCostController) modifyTokenCounter(counter *tokenCounter, bucket } counter.lastRate = cfg.NewRate - counter.limiter.Reconfigure(gc.run.now, cfg) + counter.limiter.Reconfigure(gc.run.now, cfg, resetLowProcess()) } -func (gc *groupCostController) collectRequestAndConsumption(onlySelectLow bool) *rmpb.TokenBucketRequest { +func initCounterNotify(counter *tokenCounter) { + counter.notify.mu.Lock() + if counter.notify.setupNotificationTimer != nil { + counter.notify.setupNotificationTimer.Stop() + counter.notify.setupNotificationTimer = nil + counter.notify.setupNotificationCh = nil + } + counter.notify.mu.Unlock() +} + +func (gc *groupCostController) collectRequestAndConsumption(selectTyp selectType) *rmpb.TokenBucketRequest { req := &rmpb.TokenBucketRequest{ - ResourceGroupName: gc.ResourceGroup.GetName(), + ResourceGroupName: gc.name, } // collect request resource - selected := !onlySelectLow + selected := gc.run.requestInProgress switch gc.mode { case rmpb.GroupMode_RawMode: requests := make([]*rmpb.RawResourceItem, 0, len(requestResourceLimitTypeList)) for typ, counter := range gc.run.resourceTokens { - if onlySelectLow && counter.limiter.IsLowTokens() { - selected = true + switch selectTyp { + case periodicReport: + selected = selected || gc.shouldReportConsumption() + fallthrough + case lowToken: + if counter.limiter.IsLowTokens() { + selected = true + } } request := &rmpb.RawResourceItem{ Type: typ, @@ -809,8 +1046,14 @@ func (gc *groupCostController) collectRequestAndConsumption(onlySelectLow bool) case rmpb.GroupMode_RUMode: requests := make([]*rmpb.RequestUnitItem, 0, len(requestUnitLimitTypeList)) for typ, counter := range gc.run.requestUnitTokens { - if onlySelectLow && counter.limiter.IsLowTokens() { - selected = true + switch selectTyp { + case periodicReport: + selected = selected || gc.shouldReportConsumption() + fallthrough + case lowToken: + if counter.limiter.IsLowTokens() { + selected = true + } } request := &rmpb.RequestUnitItem{ Type: typ, @@ -834,11 +1077,28 @@ func (gc *groupCostController) collectRequestAndConsumption(onlySelectLow bool) req.ConsumptionSinceLastRequest = deltaConsumption *gc.run.lastRequestConsumption = *gc.run.consumption + gc.run.lastRequestTime = time.Now() + gc.run.requestInProgress = true return req } +func (gc *groupCostController) getMeta() *rmpb.ResourceGroup { + gc.metaLock.RLock() + defer gc.metaLock.RUnlock() + return gc.meta +} + +func (gc *groupCostController) modifyMeta(newMeta *rmpb.ResourceGroup) { + gc.metaLock.Lock() + defer gc.metaLock.Unlock() + gc.meta = newMeta +} + func (gc *groupCostController) calcRequest(counter *tokenCounter) float64 { - value := counter.avgRUPerSec*gc.run.targetPeriod.Seconds() + bufferRUs + // `needTokensAmplification` is used to properly amplify a need. The reason is that in the current implementation, + // the token returned from the server determines the average consumption speed. + // Therefore, when the fillrate of resource group increases, `needTokensAmplification` can enable the client to obtain more tokens. + value := counter.avgRUPerSec * gc.run.targetPeriod.Seconds() * needTokensAmplification value -= counter.limiter.AvailableTokens(gc.run.now) if value < 0 { value = 0 @@ -848,76 +1108,132 @@ func (gc *groupCostController) calcRequest(counter *tokenCounter) float64 { func (gc *groupCostController) onRequestWait( ctx context.Context, info RequestInfo, -) (err error) { +) (*rmpb.Consumption, *rmpb.Consumption, error) { delta := &rmpb.Consumption{} for _, calc := range gc.calculators { calc.BeforeKVRequest(delta, info) } - now := time.Now() - if gc.burstable.Load() { - goto ret + + gc.mu.Lock() + add(gc.mu.consumption, delta) + gc.mu.Unlock() + + if !gc.burstable.Load() { + var err error + now := time.Now() + var i int + var d time.Duration + retryLoop: + for i = 0; i < maxRetry; i++ { + switch gc.mode { + case rmpb.GroupMode_RawMode: + res := make([]*Reservation, 0, len(requestResourceLimitTypeList)) + for typ, counter := range gc.run.resourceTokens { + if v := getRawResourceValueFromConsumption(delta, typ); v > 0 { + res = append(res, counter.limiter.Reserve(ctx, gc.mainCfg.maxWaitDuration, now, v)) + } + } + if d, err = WaitReservations(ctx, now, res); err == nil { + break retryLoop + } + case rmpb.GroupMode_RUMode: + res := make([]*Reservation, 0, len(requestUnitLimitTypeList)) + for typ, counter := range gc.run.requestUnitTokens { + if v := getRUValueFromConsumption(delta, typ); v > 0 { + res = append(res, counter.limiter.Reserve(ctx, gc.mainCfg.maxWaitDuration, now, v)) + } + } + if d, err = WaitReservations(ctx, now, res); err == nil { + break retryLoop + } + } + gc.requestRetryCounter.Inc() + time.Sleep(retryInterval) + } + if err != nil { + gc.failedRequestCounter.Inc() + gc.mu.Lock() + sub(gc.mu.consumption, delta) + gc.mu.Unlock() + return nil, nil, err + } + gc.successfulRequestDuration.Observe(d.Seconds()) + } + + gc.mu.Lock() + // Calculate the penalty of the store + penalty := &rmpb.Consumption{} + if storeCounter, exist := gc.mu.storeCounter[info.StoreID()]; exist { + *penalty = *gc.mu.globalCounter + sub(penalty, storeCounter) + } else { + gc.mu.storeCounter[info.StoreID()] = &rmpb.Consumption{} + } + // More accurately, it should be reset when the request succeed. But it would cause all concurrent requests piggyback large delta which inflates penalty. + // So here resets it directly as failure is rare. + *gc.mu.storeCounter[info.StoreID()] = *gc.mu.globalCounter + gc.mu.Unlock() + + return delta, penalty, nil +} + +func (gc *groupCostController) onResponse( + req RequestInfo, resp ResponseInfo, +) (*rmpb.Consumption, error) { + delta := &rmpb.Consumption{} + for _, calc := range gc.calculators { + calc.AfterKVRequest(delta, req, resp) } - // retry -retryLoop: - for i := 0; i < maxRetry; i++ { + if !gc.burstable.Load() { switch gc.mode { case rmpb.GroupMode_RawMode: - res := make([]*Reservation, 0, len(requestResourceLimitTypeList)) for typ, counter := range gc.run.resourceTokens { if v := getRawResourceValueFromConsumption(delta, typ); v > 0 { - res = append(res, counter.limiter.Reserve(ctx, defaultMaxWaitDuration, now, v)) + counter.limiter.RemoveTokens(time.Now(), v) } } - if err = WaitReservations(ctx, now, res); err == nil { - break retryLoop - } case rmpb.GroupMode_RUMode: - res := make([]*Reservation, 0, len(requestUnitLimitTypeList)) for typ, counter := range gc.run.requestUnitTokens { if v := getRUValueFromConsumption(delta, typ); v > 0 { - res = append(res, counter.limiter.Reserve(ctx, defaultMaxWaitDuration, now, v)) + counter.limiter.RemoveTokens(time.Now(), v) } } - if err = WaitReservations(ctx, now, res); err == nil { - break retryLoop - } } - time.Sleep(100 * time.Millisecond) - } - if err != nil { - return err } -ret: + gc.mu.Lock() + // Record the consumption of the request add(gc.mu.consumption, delta) + // Record the consumption of the request by store + count := &rmpb.Consumption{} + *count = *delta + // As the penalty is only counted when the request is completed, so here needs to calculate the write cost which is added in `BeforeKVRequest` + for _, calc := range gc.calculators { + calc.BeforeKVRequest(count, req) + } + add(gc.mu.storeCounter[req.StoreID()], count) + add(gc.mu.globalCounter, count) gc.mu.Unlock() - return nil + + return delta, nil } -func (gc *groupCostController) onResponse(req RequestInfo, resp ResponseInfo) { - delta := &rmpb.Consumption{} - for _, calc := range gc.calculators { - calc.AfterKVRequest(delta, req, resp) - } - if gc.burstable.Load() { - goto ret +// GetActiveResourceGroup is used to get action resource group. +// This is used for test only. +func (c *ResourceGroupsController) GetActiveResourceGroup(resourceGroupName string) *rmpb.ResourceGroup { + tmp, ok := c.groupsController.Load(resourceGroupName) + if !ok { + return nil } - switch gc.mode { - case rmpb.GroupMode_RawMode: - for typ, counter := range gc.run.resourceTokens { - if v := getRawResourceValueFromConsumption(delta, typ); v > 0 { - counter.limiter.RemoveTokens(time.Now(), v) - } - } - case rmpb.GroupMode_RUMode: - for typ, counter := range gc.run.requestUnitTokens { - if v := getRUValueFromConsumption(delta, typ); v > 0 { - counter.limiter.RemoveTokens(time.Now(), v) - } + return tmp.(*groupCostController).getMeta() +} + +// This is used for test only. +func (gc *groupCostController) getKVCalculator() *KVCalculator { + for _, calc := range gc.calculators { + if kvCalc, ok := calc.(*KVCalculator); ok { + return kvCalc } } -ret: - gc.mu.Lock() - add(gc.mu.consumption, delta) - gc.mu.Unlock() + return nil } diff --git a/client/resource_group/controller/controller_test.go b/client/resource_group/controller/controller_test.go index 9424aec1f11..6877f8206f3 100644 --- a/client/resource_group/controller/controller_test.go +++ b/client/resource_group/controller/controller_test.go @@ -19,6 +19,8 @@ package controller import ( + "context" + "fmt" "testing" "time" @@ -26,8 +28,7 @@ import ( "github.com/stretchr/testify/require" ) -func TestGroupControlBurstable(t *testing.T) { - re := require.New(t) +func createTestGroupCostController(re *require.Assertions) *groupCostController { group := &rmpb.ResourceGroup{ Name: "test", Mode: rmpb.GroupMode_RUMode, @@ -38,11 +39,20 @@ func TestGroupControlBurstable(t *testing.T) { }, }, }, + BackgroundSettings: &rmpb.BackgroundSettings{ + JobTypes: []string{"lightning", "br"}, + }, } ch1 := make(chan struct{}) ch2 := make(chan *groupCostController) - gc, err := newGroupCostController(group, DefaultConfig(), ch1, ch2) + gc, err := newGroupCostController(group, DefaultRUConfig(), ch1, ch2) re.NoError(err) + return gc +} + +func TestGroupControlBurstable(t *testing.T) { + re := require.New(t) + gc := createTestGroupCostController(re) gc.initRunState() args := tokenBucketReconfigureArgs{ NewRate: 1000, @@ -54,3 +64,54 @@ func TestGroupControlBurstable(t *testing.T) { gc.updateAvgRequestResourcePerSec() re.Equal(gc.burstable.Load(), true) } + +func TestRequestAndResponseConsumption(t *testing.T) { + re := require.New(t) + gc := createTestGroupCostController(re) + gc.initRunState() + testCases := []struct { + req *TestRequestInfo + resp *TestResponseInfo + }{ + // Write request + { + req: &TestRequestInfo{ + isWrite: true, + writeBytes: 100, + }, + resp: &TestResponseInfo{ + readBytes: 100, + succeed: true, + }, + }, + // Read request + { + req: &TestRequestInfo{ + isWrite: false, + writeBytes: 0, + }, + resp: &TestResponseInfo{ + readBytes: 100, + kvCPU: 100 * time.Millisecond, + succeed: true, + }, + }, + } + kvCalculator := gc.getKVCalculator() + for idx, testCase := range testCases { + caseNum := fmt.Sprintf("case %d", idx) + consumption, _, err := gc.onRequestWait(context.TODO(), testCase.req) + re.NoError(err, caseNum) + expectedConsumption := &rmpb.Consumption{} + if testCase.req.IsWrite() { + kvCalculator.calculateWriteCost(expectedConsumption, testCase.req) + re.Equal(expectedConsumption.WRU, consumption.WRU) + } + consumption, err = gc.onResponse(testCase.req, testCase.resp) + re.NoError(err, caseNum) + kvCalculator.calculateReadCost(expectedConsumption, testCase.resp) + kvCalculator.calculateCPUCost(expectedConsumption, testCase.resp) + re.Equal(expectedConsumption.RRU, consumption.RRU, caseNum) + re.Equal(expectedConsumption.TotalCpuTimeMs, consumption.TotalCpuTimeMs, caseNum) + } +} diff --git a/client/resource_group/controller/limiter.go b/client/resource_group/controller/limiter.go index af52ae98478..f89ab17514c 100644 --- a/client/resource_group/controller/limiter.go +++ b/client/resource_group/controller/limiter.go @@ -64,7 +64,7 @@ func Every(interval time.Duration) Limit { // // Some changes about burst(b): // - If b == 0, that means the limiter is unlimited capacity. default use in resource controller (burst with a rate within an unlimited capacity). -// - If b < 0, that means the limiter is unlimited capacity and r is ignored, can be seen as r == Inf (burst within a unlimited capacity). +// - If b < 0, that means the limiter is unlimited capacity and r is ignored, can be seen as r == Inf (burst within an unlimited capacity). // - If b > 0, that means the limiter is limited capacity. type Limiter struct { mu sync.Mutex @@ -79,6 +79,8 @@ type Limiter struct { // So the notifyThreshold cannot show whether the limiter is in the low token state, // isLowProcess is used to check it. isLowProcess bool + // remainingNotifyTimes is used to limit notify when the speed limit is already set. + remainingNotifyTimes int } // Limit returns the maximum overall event rate. @@ -168,7 +170,7 @@ func (r *Reservation) CancelAt(now time.Time) { r.lim.mu.Lock() defer r.lim.mu.Unlock() - if r.lim.limit == Inf || r.tokens == 0 { + if r.tokens == 0 || r.lim.burst < 0 || r.lim.limit == Inf { return } // advance time to now @@ -183,7 +185,7 @@ func (r *Reservation) CancelAt(now time.Time) { // Reserve returns a Reservation that indicates how long the caller must wait before n events happen. // The Limiter takes this Reservation into account when allowing future events. -// The returned Reservation’s OK() method returns false if wait duration exceeds deadline. +// The returned Reservation's OK() method returns false if wait duration exceeds deadline. // Usage example: // // r := lim.Reserve(time.Now(), 1) @@ -217,7 +219,6 @@ func (lim *Limiter) Reserve(ctx context.Context, waitDuration time.Duration, now func (lim *Limiter) SetupNotificationThreshold(now time.Time, threshold float64) { lim.mu.Lock() defer lim.mu.Unlock() - lim.advance(now) lim.notifyThreshold = threshold } @@ -244,7 +245,7 @@ func (lim *Limiter) maybeNotify() { } func (lim *Limiter) isLowTokensLocked() bool { - if lim.isLowProcess || (lim.notifyThreshold > 0 && lim.tokens < lim.notifyThreshold) { + if lim.burst >= 0 && (lim.isLowProcess || (lim.notifyThreshold > 0 && lim.tokens < lim.notifyThreshold)) { return true } return false @@ -268,6 +269,9 @@ func (lim *Limiter) GetBurst() int64 { func (lim *Limiter) RemoveTokens(now time.Time, amount float64) { lim.mu.Lock() defer lim.mu.Unlock() + if lim.burst < 0 || lim.limit == Inf { + return + } now, _, tokens := lim.advance(now) lim.last = now lim.tokens = tokens - amount @@ -281,20 +285,39 @@ type tokenBucketReconfigureArgs struct { NotifyThreshold float64 } +// LimiterOption configures Limiter. +type LimiterOption func(*Limiter) + +func resetLowProcess() func(*Limiter) { + return func(limiter *Limiter) { + limiter.isLowProcess = false + } +} + // Reconfigure modifies all setting for limiter -func (lim *Limiter) Reconfigure(now time.Time, args tokenBucketReconfigureArgs) { +func (lim *Limiter) Reconfigure(now time.Time, + args tokenBucketReconfigureArgs, + opts ...LimiterOption, +) { lim.mu.Lock() defer lim.mu.Unlock() - log.Debug("[resource group controller] before reconfigure", zap.Float64("NewTokens", lim.tokens), zap.Float64("NewRate", float64(lim.limit)), zap.Float64("NotifyThreshold", args.NotifyThreshold)) - now, _, tokens := lim.advance(now) - lim.last = now - lim.tokens = tokens + args.NewTokens + log.Debug("[resource group controller] before reconfigure", zap.Float64("old-tokens", lim.tokens), zap.Float64("old-rate", float64(lim.limit)), zap.Float64("old-notify-threshold", args.NotifyThreshold), zap.Int64("old-burst", lim.burst)) + if args.NewBurst < 0 { + lim.last = now + lim.tokens = args.NewTokens + } else { + now, _, tokens := lim.advance(now) + lim.last = now + lim.tokens = tokens + args.NewTokens + } lim.limit = Limit(args.NewRate) lim.burst = args.NewBurst lim.notifyThreshold = args.NotifyThreshold - lim.isLowProcess = false + for _, opt := range opts { + opt(lim) + } lim.maybeNotify() - log.Debug("[resource group controller] after reconfigure", zap.Float64("NewTokens", lim.tokens), zap.Float64("NewRate", float64(lim.limit)), zap.Float64("NotifyThreshold", args.NotifyThreshold)) + log.Debug("[resource group controller] after reconfigure", zap.Float64("tokens", lim.tokens), zap.Float64("rate", float64(lim.limit)), zap.Float64("notify-threshold", args.NotifyThreshold), zap.Int64("burst", lim.burst)) } // AvailableTokens decreases the amount of tokens currently available. @@ -312,7 +335,7 @@ func (lim *Limiter) reserveN(now time.Time, n float64, maxFutureReserve time.Dur lim.mu.Lock() defer lim.mu.Unlock() - if lim.limit == Inf || lim.burst < 0 { + if lim.burst < 0 || lim.limit == Inf { return Reservation{ ok: true, lim: lim, @@ -349,11 +372,28 @@ func (lim *Limiter) reserveN(now time.Time, n float64, maxFutureReserve time.Dur lim.tokens = tokens lim.maybeNotify() } else { + log.Debug("[resource group controller]", zap.Float64("current-tokens", lim.tokens), zap.Float64("current-rate", float64(lim.limit)), zap.Float64("request-tokens", n), zap.Int64("burst", lim.burst), zap.Int("remaining-notify-times", lim.remainingNotifyTimes)) lim.last = last + if lim.limit == 0 { + lim.notify() + } else if lim.remainingNotifyTimes > 0 { + // When fillrate is greater than 0, the speed limit is already set. + // If limiter are in limit state, the server has allocated tokens as much as possible. Don't need to request tokens. + // But there is a special case, see issue https://github.com/tikv/pd/issues/6300. + lim.remainingNotifyTimes-- + lim.notify() + } } return r } +// ResetRemainingNotifyTimes resets the remaining notify times to 3. +func (lim *Limiter) ResetRemainingNotifyTimes() { + lim.mu.Lock() + defer lim.mu.Unlock() + lim.remainingNotifyTimes = 3 +} + // advance calculates and returns an updated state for lim resulting from the passage of time. // lim is not changed. // advance requires that lim.mu is held. @@ -362,12 +402,15 @@ func (lim *Limiter) advance(now time.Time) (newNow time.Time, newLast time.Time, if now.Before(last) { last = now } + if lim.burst < 0 { + return now, last, lim.tokens + } // Calculate the new number of tokens, due to time that passed. elapsed := now.Sub(last) delta := lim.limit.tokensFromDuration(elapsed) tokens := lim.tokens + delta - if lim.burst != 0 { + if lim.burst > 0 { if burst := float64(lim.burst); tokens > burst { tokens = burst } @@ -396,9 +439,9 @@ func (limit Limit) tokensFromDuration(d time.Duration) float64 { // WaitReservations is used to process a series of reservations // so that all limiter tokens are returned if one reservation fails -func WaitReservations(ctx context.Context, now time.Time, reservations []*Reservation) error { +func WaitReservations(ctx context.Context, now time.Time, reservations []*Reservation) (time.Duration, error) { if len(reservations) == 0 { - return nil + return 0, nil } cancel := func() { for _, res := range reservations { @@ -409,7 +452,7 @@ func WaitReservations(ctx context.Context, now time.Time, reservations []*Reserv for _, res := range reservations { if !res.ok { cancel() - return errs.ErrClientResourceGroupThrottled + return 0, errs.ErrClientResourceGroupThrottled } delay := res.DelayFrom(now) if delay > longestDelayDuration { @@ -417,7 +460,7 @@ func WaitReservations(ctx context.Context, now time.Time, reservations []*Reserv } } if longestDelayDuration <= 0 { - return nil + return 0, nil } t := time.NewTimer(longestDelayDuration) defer t.Stop() @@ -425,11 +468,11 @@ func WaitReservations(ctx context.Context, now time.Time, reservations []*Reserv select { case <-t.C: // We can proceed. - return nil + return longestDelayDuration, nil case <-ctx.Done(): // Context was canceled before we could proceed. Cancel the // reservation, which may permit other events to proceed sooner. cancel() - return ctx.Err() + return 0, ctx.Err() } } diff --git a/client/resource_group/controller/limiter_test.go b/client/resource_group/controller/limiter_test.go index c3620a9addd..b8b96ae13d6 100644 --- a/client/resource_group/controller/limiter_test.go +++ b/client/resource_group/controller/limiter_test.go @@ -109,11 +109,19 @@ func TestReconfig(t *testing.T) { args := tokenBucketReconfigureArgs{ NewTokens: 6., NewRate: 2, - NewBurst: -1, } lim.Reconfigure(t1, args) checkTokens(re, lim, t1, 5) checkTokens(re, lim, t2, 7) + + args = tokenBucketReconfigureArgs{ + NewTokens: 6., + NewRate: 2, + NewBurst: -1, + } + lim.Reconfigure(t1, args) + checkTokens(re, lim, t1, 6) + checkTokens(re, lim, t2, 6) re.Equal(int64(-1), lim.GetBurst()) } @@ -152,7 +160,8 @@ func TestCancel(t *testing.T) { r2 := lim2.Reserve(ctx1, InfDuration, t1, 5) checkTokens(re, lim1, t2, 7) checkTokens(re, lim2, t2, 2) - err := WaitReservations(ctx, t2, []*Reservation{r1, r2}) + d, err := WaitReservations(ctx, t2, []*Reservation{r1, r2}) + re.Equal(d, time.Duration(0)) re.Error(err) checkTokens(re, lim1, t3, 13) checkTokens(re, lim2, t3, 3) @@ -166,7 +175,7 @@ func TestCancel(t *testing.T) { var wg sync.WaitGroup wg.Add(1) go func() { - err := WaitReservations(ctx2, t3, []*Reservation{r1, r2}) + _, err := WaitReservations(ctx2, t3, []*Reservation{r1, r2}) re.Error(err) wg.Done() }() diff --git a/client/resource_group/controller/metrics.go b/client/resource_group/controller/metrics.go new file mode 100644 index 00000000000..68eb26d0312 --- /dev/null +++ b/client/resource_group/controller/metrics.go @@ -0,0 +1,91 @@ +// Copyright 2023 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controller + +import "github.com/prometheus/client_golang/prometheus" + +const ( + namespace = "resource_manager_client" + requestSubsystem = "request" + tokenRequestSubsystem = "token_request" + + resourceGroupNameLabel = "name" +) + +var ( + resourceGroupStatusGauge = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: "resource_group", + Name: "status", + Help: "Status of the resource group.", + }, []string{resourceGroupNameLabel}) + + successfulRequestDuration = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Namespace: namespace, + Subsystem: requestSubsystem, + Name: "success", + Buckets: []float64{.005, .01, .05, .1, .5, 1, 5, 10, 20, 25, 30}, // 0.005 ~ 30 + Help: "Bucketed histogram of wait duration of successful request.", + }, []string{resourceGroupNameLabel}) + + failedRequestCounter = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: requestSubsystem, + Name: "fail", + Help: "Counter of failed request.", + }, []string{resourceGroupNameLabel}) + + requestRetryCounter = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: requestSubsystem, + Name: "retry", + Help: "Counter of retry time for request.", + }, []string{resourceGroupNameLabel}) + + tokenRequestDuration = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Namespace: namespace, + Subsystem: tokenRequestSubsystem, + Name: "duration", + Help: "Bucketed histogram of latency(s) of token request.", + }, []string{"type"}) + + resourceGroupTokenRequestCounter = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: tokenRequestSubsystem, + Name: "resource_group", + Help: "Counter of token request by every resource group.", + }, []string{resourceGroupNameLabel}) +) + +var ( + // WithLabelValues is a heavy operation, define variable to avoid call it every time. + failedTokenRequestDuration = tokenRequestDuration.WithLabelValues("fail") + successfulTokenRequestDuration = tokenRequestDuration.WithLabelValues("success") +) + +func init() { + prometheus.MustRegister(resourceGroupStatusGauge) + prometheus.MustRegister(successfulRequestDuration) + prometheus.MustRegister(failedRequestCounter) + prometheus.MustRegister(requestRetryCounter) + prometheus.MustRegister(tokenRequestDuration) + prometheus.MustRegister(resourceGroupTokenRequestCounter) +} diff --git a/client/resource_group/controller/model.go b/client/resource_group/controller/model.go index 0375b34c8b0..cf89aa535f9 100644 --- a/client/resource_group/controller/model.go +++ b/client/resource_group/controller/model.go @@ -18,11 +18,11 @@ import ( "os" "time" - "github.com/elastic/gosigar" - "github.com/pingcap/log" + sigar "github.com/cloudfoundry/gosigar" "go.uber.org/zap" rmpb "github.com/pingcap/kvproto/pkg/resource_manager" + "github.com/pingcap/log" ) // RequestUnit is the basic unit of the resource request management, which has two types: @@ -35,6 +35,8 @@ type RequestUnit float64 type RequestInfo interface { IsWrite() bool WriteBytes() uint64 + ReplicaNumber() int64 + StoreID() uint64 } // ResponseInfo is the interface of the response information provider. A response should be @@ -63,13 +65,13 @@ type ResourceCalculator interface { // KVCalculator is used to calculate the KV-side consumption. type KVCalculator struct { - *Config + *RUConfig } var _ ResourceCalculator = (*KVCalculator)(nil) -func newKVCalculator(cfg *Config) *KVCalculator { - return &KVCalculator{Config: cfg} +func newKVCalculator(cfg *RUConfig) *KVCalculator { + return &KVCalculator{RUConfig: cfg} } // Trickle ... @@ -86,14 +88,19 @@ func (kc *KVCalculator) BeforeKVRequest(consumption *rmpb.Consumption, req Reque consumption.KvReadRpcCount += 1 // Read bytes could not be known before the request is executed, // so we only add the base cost here. - consumption.RRU += float64(kc.ReadBaseCost) + consumption.RRU += float64(kc.ReadBaseCost) + float64(kc.ReadPerBatchBaseCost)*defaultAvgBatchProportion } } func (kc *KVCalculator) calculateWriteCost(consumption *rmpb.Consumption, req RequestInfo) { writeBytes := float64(req.WriteBytes()) consumption.WriteBytes += writeBytes - consumption.WRU += float64(kc.WriteBaseCost) + float64(kc.WriteBytesCost)*writeBytes + // write request cost need consider the replicas, due to write data will be replicate to all replicas. + replicaNums := float64(req.ReplicaNumber()) + if replicaNums == 0 { + replicaNums = 1 + } + consumption.WRU += (float64(kc.WriteBaseCost) + float64(kc.WritePerBatchBaseCost)*defaultAvgBatchProportion + float64(kc.WriteBytesCost)*writeBytes) * replicaNums } // AfterKVRequest ... @@ -111,18 +118,27 @@ func (kc *KVCalculator) AfterKVRequest(consumption *rmpb.Consumption, req Reques } func (kc *KVCalculator) calculateReadCost(consumption *rmpb.Consumption, res ResponseInfo) { + if consumption == nil { + return + } readBytes := float64(res.ReadBytes()) consumption.ReadBytes += readBytes consumption.RRU += float64(kc.ReadBytesCost) * readBytes } func (kc *KVCalculator) calculateCPUCost(consumption *rmpb.Consumption, res ResponseInfo) { + if consumption == nil { + return + } kvCPUMs := float64(res.KVCPU().Nanoseconds()) / 1000000.0 consumption.TotalCpuTimeMs += kvCPUMs consumption.RRU += float64(kc.CPUMsCost) * kvCPUMs } func (kc *KVCalculator) payBackWriteCost(consumption *rmpb.Consumption, req RequestInfo) { + if consumption == nil { + return + } writeBytes := float64(req.WriteBytes()) consumption.WriteBytes -= writeBytes consumption.WRU -= float64(kc.WriteBaseCost) + float64(kc.WriteBytesCost)*writeBytes @@ -130,17 +146,20 @@ func (kc *KVCalculator) payBackWriteCost(consumption *rmpb.Consumption, req Requ // SQLCalculator is used to calculate the SQL-side consumption. type SQLCalculator struct { - *Config + *RUConfig } var _ ResourceCalculator = (*SQLCalculator)(nil) -func newSQLCalculator(cfg *Config) *SQLCalculator { - return &SQLCalculator{Config: cfg} +func newSQLCalculator(cfg *RUConfig) *SQLCalculator { + return &SQLCalculator{RUConfig: cfg} } // Trickle update sql layer CPU consumption. func (dsc *SQLCalculator) Trickle(consumption *rmpb.Consumption) { + if consumption == nil { + return + } delta := getSQLProcessCPUTime(dsc.isSingleGroupByKeyspace) - consumption.SqlLayerCpuTimeMs consumption.TotalCpuTimeMs += delta consumption.SqlLayerCpuTimeMs += delta @@ -155,44 +174,59 @@ func (dsc *SQLCalculator) AfterKVRequest(consumption *rmpb.Consumption, req Requ } func getRUValueFromConsumption(custom *rmpb.Consumption, typ rmpb.RequestUnitType) float64 { - if typ == 0 { + if custom == nil { + return 0 + } + if typ == rmpb.RequestUnitType_RU { return custom.RRU + custom.WRU } return 0 } func getRUTokenBucketSetting(group *rmpb.ResourceGroup, typ rmpb.RequestUnitType) *rmpb.TokenBucket { - if typ == 0 { + if group == nil { + return nil + } + if typ == rmpb.RequestUnitType_RU { return group.RUSettings.RU } return nil } func getRawResourceValueFromConsumption(custom *rmpb.Consumption, typ rmpb.RawResourceType) float64 { + if custom == nil { + return 0 + } switch typ { - case 0: + case rmpb.RawResourceType_CPU: return custom.TotalCpuTimeMs - case 1: + case rmpb.RawResourceType_IOReadFlow: return custom.ReadBytes - case 2: + case rmpb.RawResourceType_IOWriteFlow: return custom.WriteBytes } return 0 } func getRawResourceTokenBucketSetting(group *rmpb.ResourceGroup, typ rmpb.RawResourceType) *rmpb.TokenBucket { + if group == nil { + return nil + } switch typ { - case 0: + case rmpb.RawResourceType_CPU: return group.RawResourceSettings.Cpu - case 1: + case rmpb.RawResourceType_IOReadFlow: return group.RawResourceSettings.IoRead - case 2: + case rmpb.RawResourceType_IOWriteFlow: return group.RawResourceSettings.IoWrite } return nil } func add(custom1 *rmpb.Consumption, custom2 *rmpb.Consumption) { + if custom1 == nil || custom2 == nil { + return + } custom1.RRU += custom2.RRU custom1.WRU += custom2.WRU custom1.ReadBytes += custom2.ReadBytes @@ -204,6 +238,9 @@ func add(custom1 *rmpb.Consumption, custom2 *rmpb.Consumption) { } func sub(custom1 *rmpb.Consumption, custom2 *rmpb.Consumption) { + if custom1 == nil || custom2 == nil { + return + } custom1.RRU -= custom2.RRU custom1.WRU -= custom2.WRU custom1.ReadBytes -= custom2.ReadBytes @@ -214,6 +251,10 @@ func sub(custom1 *rmpb.Consumption, custom2 *rmpb.Consumption) { custom1.KvWriteRpcCount -= custom2.KvWriteRpcCount } +func equalRU(custom1 rmpb.Consumption, custom2 rmpb.Consumption) bool { + return custom1.RRU == custom2.RRU && custom1.WRU == custom2.WRU +} + // getSQLProcessCPUTime returns the cumulative user+system time (in ms) since the process start. func getSQLProcessCPUTime(isSingleGroupByKeyspace bool) float64 { if isSingleGroupByKeyspace { @@ -224,7 +265,7 @@ func getSQLProcessCPUTime(isSingleGroupByKeyspace bool) float64 { func getSysProcessCPUTime() float64 { pid := os.Getpid() - cpuTime := gosigar.ProcTime{} + cpuTime := sigar.ProcTime{} if err := cpuTime.Get(pid); err != nil { log.Error("getCPUTime get pid failed", zap.Error(err)) } diff --git a/client/resource_group/controller/model_test.go b/client/resource_group/controller/model_test.go new file mode 100644 index 00000000000..594091da364 --- /dev/null +++ b/client/resource_group/controller/model_test.go @@ -0,0 +1,198 @@ +// Copyright 2023 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controller + +import ( + "testing" + + rmpb "github.com/pingcap/kvproto/pkg/resource_manager" + "github.com/stretchr/testify/require" +) + +func TestGetRUValueFromConsumption(t *testing.T) { + // Positive test case + re := require.New(t) + custom := &rmpb.Consumption{RRU: 2.5, WRU: 3.5} + typ := rmpb.RequestUnitType_RU + expected := float64(6) + + result := getRUValueFromConsumption(custom, typ) + re.Equal(expected, result) + + // When custom is nil + custom = nil + expected = float64(0) + + result = getRUValueFromConsumption(custom, typ) + re.Equal(expected, result) +} + +func TestGetRUTokenBucketSetting(t *testing.T) { + // Positive test case + re := require.New(t) + group := &rmpb.ResourceGroup{ + RUSettings: &rmpb.GroupRequestUnitSettings{ + RU: &rmpb.TokenBucket{Settings: &rmpb.TokenLimitSettings{FillRate: 100}}, + }, + } + typ := rmpb.RequestUnitType_RU + expected := &rmpb.TokenBucket{Settings: &rmpb.TokenLimitSettings{FillRate: 100}} + + result := getRUTokenBucketSetting(group, typ) + re.Equal(expected.GetSettings().GetFillRate(), result.GetSettings().GetFillRate()) + + // When group is nil + group = nil + expected = nil + + result = getRUTokenBucketSetting(group, typ) + if result != expected { + t.Errorf("Expected nil but got %v", result) + } +} + +func TestGetRawResourceValueFromConsumption(t *testing.T) { + // Positive test case + re := require.New(t) + custom := &rmpb.Consumption{TotalCpuTimeMs: 50} + typ := rmpb.RawResourceType_CPU + expected := float64(50) + + result := getRawResourceValueFromConsumption(custom, typ) + re.Equal(expected, result) + + // When custom is nil + custom = nil + expected = float64(0) + + result = getRawResourceValueFromConsumption(custom, typ) + re.Equal(expected, result) + + // When typ is IOReadFlow + custom = &rmpb.Consumption{ReadBytes: 200} + typ = rmpb.RawResourceType_IOReadFlow + expected = float64(200) + + result = getRawResourceValueFromConsumption(custom, typ) + re.Equal(expected, result) +} + +func TestGetRawResourceTokenBucketSetting(t *testing.T) { + // Positive test case + re := require.New(t) + group := &rmpb.ResourceGroup{ + RawResourceSettings: &rmpb.GroupRawResourceSettings{ + Cpu: &rmpb.TokenBucket{Settings: &rmpb.TokenLimitSettings{FillRate: 100}}, + }, + } + typ := rmpb.RawResourceType_CPU + expected := &rmpb.TokenBucket{Settings: &rmpb.TokenLimitSettings{FillRate: 100}} + + result := getRawResourceTokenBucketSetting(group, typ) + + re.Equal(expected.GetSettings().GetFillRate(), result.GetSettings().GetFillRate()) + + // When group is nil + group = nil + expected = nil + + result = getRawResourceTokenBucketSetting(group, typ) + if result != expected { + t.Errorf("Expected nil but got %v", result) + } + + // When typ is IOReadFlow + group = &rmpb.ResourceGroup{ + RawResourceSettings: &rmpb.GroupRawResourceSettings{ + IoRead: &rmpb.TokenBucket{Settings: &rmpb.TokenLimitSettings{FillRate: 200}}, + }, + } + typ = rmpb.RawResourceType_IOReadFlow + expected = &rmpb.TokenBucket{Settings: &rmpb.TokenLimitSettings{FillRate: 200}} + + result = getRawResourceTokenBucketSetting(group, typ) + re.Equal(expected.GetSettings().GetFillRate(), result.GetSettings().GetFillRate()) +} + +func TestAdd(t *testing.T) { + // Positive test case + re := require.New(t) + custom1 := &rmpb.Consumption{RRU: 2.5, WRU: 3.5} + custom2 := &rmpb.Consumption{RRU: 1.5, WRU: 2.5} + expected := &rmpb.Consumption{ + RRU: 4, + WRU: 6, + ReadBytes: 0, + WriteBytes: 0, + TotalCpuTimeMs: 0, + SqlLayerCpuTimeMs: 0, + KvReadRpcCount: 0, + KvWriteRpcCount: 0, + } + + add(custom1, custom2) + re.Equal(expected, custom1) + + // When custom1 is nil + custom1 = nil + custom2 = &rmpb.Consumption{RRU: 1.5, WRU: 2.5} + expected = nil + + add(custom1, custom2) + re.Equal(expected, custom1) + + // When custom2 is nil + custom1 = &rmpb.Consumption{RRU: 2.5, WRU: 3.5} + custom2 = nil + expected = &rmpb.Consumption{RRU: 2.5, WRU: 3.5} + + add(custom1, custom2) + re.Equal(expected, custom1) +} + +func TestSub(t *testing.T) { + // Positive test case + re := require.New(t) + custom1 := &rmpb.Consumption{RRU: 2.5, WRU: 3.5} + custom2 := &rmpb.Consumption{RRU: 1.5, WRU: 2.5} + expected := &rmpb.Consumption{ + RRU: 1, + WRU: 1, + ReadBytes: 0, + WriteBytes: 0, + TotalCpuTimeMs: 0, + SqlLayerCpuTimeMs: 0, + KvReadRpcCount: 0, + KvWriteRpcCount: 0, + } + + sub(custom1, custom2) + re.Equal(expected, custom1) + // When custom1 is nil + custom1 = nil + custom2 = &rmpb.Consumption{RRU: 1.5, WRU: 2.5} + expected = nil + + sub(custom1, custom2) + re.Equal(expected, custom1) + + // When custom2 is nil + custom1 = &rmpb.Consumption{RRU: 2.5, WRU: 3.5} + custom2 = nil + expected = &rmpb.Consumption{RRU: 2.5, WRU: 3.5} + + sub(custom1, custom2) + re.Equal(expected, custom1) +} diff --git a/client/resource_group/controller/testutil.go b/client/resource_group/controller/testutil.go new file mode 100644 index 00000000000..4df8c9bba0d --- /dev/null +++ b/client/resource_group/controller/testutil.go @@ -0,0 +1,88 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Copyright 2023 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS,g +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controller + +import "time" + +// TestRequestInfo is used to test the request info interface. +type TestRequestInfo struct { + isWrite bool + writeBytes uint64 + storeID uint64 +} + +// NewTestRequestInfo creates a new TestRequestInfo. +func NewTestRequestInfo(isWrite bool, writeBytes uint64, storeID uint64) *TestRequestInfo { + return &TestRequestInfo{ + isWrite: isWrite, + writeBytes: writeBytes, + storeID: storeID, + } +} + +// IsWrite implements the RequestInfo interface. +func (tri *TestRequestInfo) IsWrite() bool { + return tri.isWrite +} + +// WriteBytes implements the RequestInfo interface. +func (tri *TestRequestInfo) WriteBytes() uint64 { + return tri.writeBytes +} + +// StoreID implements the RequestInfo interface. +func (tri *TestRequestInfo) StoreID() uint64 { + return tri.storeID +} + +// ReplicaNumber implements the RequestInfo interface. +func (tri *TestRequestInfo) ReplicaNumber() int64 { + return 1 +} + +// TestResponseInfo is used to test the response info interface. +type TestResponseInfo struct { + readBytes uint64 + kvCPU time.Duration + succeed bool +} + +// NewTestResponseInfo creates a new TestResponseInfo. +func NewTestResponseInfo(readBytes uint64, kvCPU time.Duration, succeed bool) *TestResponseInfo { + return &TestResponseInfo{ + readBytes: readBytes, + kvCPU: kvCPU, + succeed: succeed, + } +} + +// ReadBytes implements the ResponseInfo interface. +func (tri *TestResponseInfo) ReadBytes() uint64 { + return tri.readBytes +} + +// KVCPU implements the ResponseInfo interface. +func (tri *TestResponseInfo) KVCPU() time.Duration { + return tri.kvCPU +} + +// Succeed implements the ResponseInfo interface. +func (tri *TestResponseInfo) Succeed() bool { + return tri.succeed +} diff --git a/client/resource_manager_client.go b/client/resource_manager_client.go index 7452ca4a154..4544c288896 100644 --- a/client/resource_manager_client.go +++ b/client/resource_manager_client.go @@ -21,7 +21,7 @@ import ( "github.com/gogo/protobuf/proto" "github.com/pingcap/errors" - "github.com/pingcap/kvproto/pkg/pdpb" + "github.com/pingcap/kvproto/pkg/meta_storagepb" rmpb "github.com/pingcap/kvproto/pkg/resource_manager" "github.com/pingcap/log" "github.com/tikv/pd/client/errs" @@ -36,8 +36,13 @@ const ( groupSettingsPathPrefix = "resource_group/settings" // errNotPrimary is returned when the requested server is not primary. errNotPrimary = "not primary" + // errNotLeader is returned when the requested server is not pd leader. + errNotLeader = "not leader" ) +// GroupSettingsPathPrefixBytes is used to watch or get resource groups. +var GroupSettingsPathPrefixBytes = []byte(groupSettingsPathPrefix) + // ResourceManagerClient manages resource group info and token request. type ResourceManagerClient interface { ListResourceGroups(ctx context.Context) ([]*rmpb.ResourceGroup, error) @@ -45,13 +50,15 @@ type ResourceManagerClient interface { AddResourceGroup(ctx context.Context, metaGroup *rmpb.ResourceGroup) (string, error) ModifyResourceGroup(ctx context.Context, metaGroup *rmpb.ResourceGroup) (string, error) DeleteResourceGroup(ctx context.Context, resourceGroupName string) (string, error) + LoadResourceGroups(ctx context.Context) ([]*rmpb.ResourceGroup, int64, error) WatchResourceGroup(ctx context.Context, revision int64) (chan []*rmpb.ResourceGroup, error) AcquireTokenBuckets(ctx context.Context, request *rmpb.TokenBucketsRequest) ([]*rmpb.TokenBucketResponse, error) + Watch(ctx context.Context, key []byte, opts ...OpOption) (chan []*meta_storagepb.Event, error) } // resourceManagerClient gets the ResourceManager client of current PD leader. func (c *client) resourceManagerClient() (rmpb.ResourceManagerClient, error) { - cc, err := c.svcDiscovery.GetOrCreateGRPCConn(c.GetLeaderAddr()) + cc, err := c.pdSvcDiscovery.GetOrCreateGRPCConn(c.GetLeaderAddr()) if err != nil { return nil, err } @@ -60,8 +67,8 @@ func (c *client) resourceManagerClient() (rmpb.ResourceManagerClient, error) { // gRPCErrorHandler is used to handle the gRPC error returned by the resource manager service. func (c *client) gRPCErrorHandler(err error) { - if strings.Contains(err.Error(), errNotPrimary) { - c.svcDiscovery.ScheduleCheckMemberChanged() + if strings.Contains(err.Error(), errNotPrimary) || strings.Contains(err.Error(), errNotLeader) { + c.pdSvcDiscovery.ScheduleCheckMemberChanged() } } @@ -158,12 +165,31 @@ func (c *client) DeleteResourceGroup(ctx context.Context, resourceGroupName stri return resp.GetBody(), nil } +func (c *client) LoadResourceGroups(ctx context.Context) ([]*rmpb.ResourceGroup, int64, error) { + resp, err := c.Get(ctx, GroupSettingsPathPrefixBytes, WithPrefix()) + if err != nil { + return nil, 0, err + } + if resp.Header.Error != nil { + return nil, resp.Header.Revision, errors.Errorf(resp.Header.Error.Message) + } + groups := make([]*rmpb.ResourceGroup, 0, len(resp.Kvs)) + for _, item := range resp.Kvs { + group := &rmpb.ResourceGroup{} + if err := proto.Unmarshal(item.Value, group); err != nil { + continue + } + groups = append(groups, group) + } + return groups, resp.Header.Revision, nil +} + // WatchResourceGroup [just for TEST] watches resource groups changes. // It returns a stream of slices of resource groups. // The first message in stream contains all current resource groups, // all subsequent messages contains new events[PUT/DELETE] for all resource groups. func (c *client) WatchResourceGroup(ctx context.Context, revision int64) (chan []*rmpb.ResourceGroup, error) { - configChan, err := c.WatchGlobalConfig(ctx, groupSettingsPathPrefix, revision) + configChan, err := c.Watch(ctx, GroupSettingsPathPrefixBytes, WithRev(revision), WithPrefix()) if err != nil { return nil, err } @@ -186,14 +212,14 @@ func (c *client) WatchResourceGroup(ctx context.Context, revision int64) (chan [ } groups := make([]*rmpb.ResourceGroup, 0, len(res)) for _, item := range res { - switch item.EventType { - case pdpb.EventType_PUT: + switch item.Type { + case meta_storagepb.Event_PUT: group := &rmpb.ResourceGroup{} - if err := proto.Unmarshal([]byte(item.Value), group); err != nil { + if err := proto.Unmarshal(item.Kv.Value, group); err != nil { return } groups = append(groups, group) - case pdpb.EventType_DELETE: + case meta_storagepb.Event_DELETE: continue } } @@ -319,7 +345,7 @@ func (c *client) handleResourceTokenDispatcher(dispatcherCtx context.Context, tb // If the stream is still nil, return an error. if stream == nil { firstRequest.done <- errors.Errorf("failed to get the stream connection") - c.svcDiscovery.ScheduleCheckMemberChanged() + c.pdSvcDiscovery.ScheduleCheckMemberChanged() connection.reset() continue } @@ -331,7 +357,7 @@ func (c *client) handleResourceTokenDispatcher(dispatcherCtx context.Context, tb default: } if err = c.processTokenRequests(stream, firstRequest); err != nil { - c.svcDiscovery.ScheduleCheckMemberChanged() + c.pdSvcDiscovery.ScheduleCheckMemberChanged() connection.reset() log.Info("[resource_manager] token request error", zap.Error(err)) } @@ -365,6 +391,8 @@ func (c *client) tryResourceManagerConnect(ctx context.Context, connection *reso err error stream rmpb.ResourceManager_AcquireTokenBucketsClient ) + ticker := time.NewTicker(retryInterval) + defer ticker.Stop() for i := 0; i < maxRetryTimes; i++ { cc, err := c.resourceManagerClient() if err != nil { @@ -382,7 +410,7 @@ func (c *client) tryResourceManagerConnect(ctx context.Context, connection *reso select { case <-ctx.Done(): return err - case <-time.After(retryInterval): + case <-ticker.C: } } return err diff --git a/client/timerpool/pool.go b/client/timerpool/pool.go new file mode 100644 index 00000000000..28ffacfc629 --- /dev/null +++ b/client/timerpool/pool.go @@ -0,0 +1,43 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Note: This file is copied from https://go-review.googlesource.com/c/go/+/276133 + +package timerpool + +import ( + "sync" + "time" +) + +// GlobalTimerPool is a global pool for reusing *time.Timer. +var GlobalTimerPool TimerPool + +// TimerPool is a wrapper of sync.Pool which caches *time.Timer for reuse. +type TimerPool struct { + pool sync.Pool +} + +// Get returns a timer with a given duration. +func (tp *TimerPool) Get(d time.Duration) *time.Timer { + if v := tp.pool.Get(); v != nil { + timer := v.(*time.Timer) + timer.Reset(d) + return timer + } + return time.NewTimer(d) +} + +// Put tries to call timer.Stop() before putting it back into pool, +// if the timer.Stop() returns false (it has either already expired or been stopped), +// have a shot at draining the channel with residual time if there is one. +func (tp *TimerPool) Put(timer *time.Timer) { + if !timer.Stop() { + select { + case <-timer.C: + default: + } + } + tp.pool.Put(timer) +} diff --git a/client/timerpool/pool_test.go b/client/timerpool/pool_test.go new file mode 100644 index 00000000000..d6dffc723a9 --- /dev/null +++ b/client/timerpool/pool_test.go @@ -0,0 +1,70 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Note: This file is copied from https://go-review.googlesource.com/c/go/+/276133 + +package timerpool + +import ( + "testing" + "time" +) + +func TestTimerPool(t *testing.T) { + var tp TimerPool + + for i := 0; i < 100; i++ { + timer := tp.Get(20 * time.Millisecond) + + select { + case <-timer.C: + t.Errorf("timer expired too early") + continue + default: + } + + select { + case <-time.After(100 * time.Millisecond): + t.Errorf("timer didn't expire on time") + case <-timer.C: + } + + tp.Put(timer) + } +} + +const timeout = 10 * time.Millisecond + +func BenchmarkTimerUtilization(b *testing.B) { + b.Run("TimerWithPool", func(b *testing.B) { + for i := 0; i < b.N; i++ { + t := GlobalTimerPool.Get(timeout) + GlobalTimerPool.Put(t) + } + }) + b.Run("TimerWithoutPool", func(b *testing.B) { + for i := 0; i < b.N; i++ { + t := time.NewTimer(timeout) + t.Stop() + } + }) +} + +func BenchmarkTimerPoolParallel(b *testing.B) { + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + t := GlobalTimerPool.Get(timeout) + GlobalTimerPool.Put(t) + } + }) +} + +func BenchmarkTimerNativeParallel(b *testing.B) { + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + t := time.NewTimer(timeout) + t.Stop() + } + }) +} diff --git a/client/tso_batch_controller.go b/client/tso_batch_controller.go index 3ad05ca7cba..842c772abd9 100644 --- a/client/tso_batch_controller.go +++ b/client/tso_batch_controller.go @@ -44,16 +44,16 @@ func newTSOBatchController(tsoRequestCh chan *tsoRequest, maxBatchSize int) *tso // fetchPendingRequests will start a new round of the batch collecting from the channel. // It returns true if everything goes well, otherwise false which means we should stop the service. func (tbc *tsoBatchController) fetchPendingRequests(ctx context.Context, maxBatchWaitInterval time.Duration) error { - var firstTSORequest *tsoRequest + var firstRequest *tsoRequest select { case <-ctx.Done(): return ctx.Err() - case firstTSORequest = <-tbc.tsoRequestCh: + case firstRequest = <-tbc.tsoRequestCh: } // Start to batch when the first TSO request arrives. tbc.batchStartTime = time.Now() tbc.collectedRequestCount = 0 - tbc.pushRequest(firstTSORequest) + tbc.pushRequest(firstRequest) // This loop is for trying best to collect more requests, so we use `tbc.maxBatchSize` here. fetchPendingRequestsLoop: @@ -130,7 +130,7 @@ func (tbc *tsoBatchController) adjustBestBatchSize() { } } -func (tbc *tsoBatchController) revokePendingTSORequest(err error) { +func (tbc *tsoBatchController) revokePendingRequest(err error) { for i := 0; i < len(tbc.tsoRequestCh); i++ { req := <-tbc.tsoRequestCh req.done <- err diff --git a/client/tso_client.go b/client/tso_client.go index 871a8da4847..35d9388c72b 100644 --- a/client/tso_client.go +++ b/client/tso_client.go @@ -29,11 +29,19 @@ import ( healthpb "google.golang.org/grpc/health/grpc_health_v1" ) -// TSOClient defines basic interface of the TSO client -// For test only +// TSOClient is the client used to get timestamps. type TSOClient interface { - // GetTSOAllocators returns {dc-location -> TSO allocator serving URL} connection map - GetTSOAllocators() *sync.Map + // GetTS gets a timestamp from PD or TSO microservice. + GetTS(ctx context.Context) (int64, int64, error) + // GetTSAsync gets a timestamp from PD or TSO microservice, without block the caller. + GetTSAsync(ctx context.Context) TSFuture + // GetLocalTS gets a local timestamp from PD or TSO microservice. + GetLocalTS(ctx context.Context, dcLocation string) (int64, int64, error) + // GetLocalTSAsync gets a local timestamp from PD or TSO microservice, without block the caller. + GetLocalTSAsync(ctx context.Context, dcLocation string) TSFuture + // GetMinTS gets a timestamp from PD or the minimal timestamp across all keyspace groups from + // the TSO microservice. + GetMinTS(ctx context.Context) (int64, int64, error) } type tsoRequest struct { @@ -43,7 +51,6 @@ type tsoRequest struct { done chan error physical int64 logical int64 - keyspaceID uint32 dcLocation string } @@ -60,10 +67,9 @@ var tsoReqPool = sync.Pool{ type tsoClient struct { ctx context.Context cancel context.CancelFunc - wg *sync.WaitGroup + wg sync.WaitGroup option *option - keyspaceID uint32 svcDiscovery ServiceDiscovery tsoStreamBuilderFactory // tsoAllocators defines the mapping {dc-location -> TSO allocator leader URL} @@ -77,8 +83,8 @@ type tsoClient struct { tsoDispatcher sync.Map // Same as map[string]chan *tsoRequest // dc-location -> deadline tsDeadline sync.Map // Same as map[string]chan deadline - // dc-location -> *lastTSO - lastTSMap sync.Map // Same as map[string]*lastTSO + // dc-location -> *tsoInfo while the tsoInfo is the last TSO info + lastTSOInfoMap sync.Map // Same as map[string]*tsoInfo checkTSDeadlineCh chan struct{} checkTSODispatcherCh chan struct{} @@ -86,14 +92,15 @@ type tsoClient struct { } // newTSOClient returns a new TSO client. -func newTSOClient(ctx context.Context, cancel context.CancelFunc, wg *sync.WaitGroup, option *option, - keyspaceID uint32, svcDiscovery ServiceDiscovery, eventSrc tsoAllocatorEventSource, factory tsoStreamBuilderFactory) *tsoClient { +func newTSOClient( + ctx context.Context, option *option, + svcDiscovery ServiceDiscovery, factory tsoStreamBuilderFactory, +) *tsoClient { + ctx, cancel := context.WithCancel(ctx) c := &tsoClient{ ctx: ctx, cancel: cancel, - wg: wg, option: option, - keyspaceID: keyspaceID, svcDiscovery: svcDiscovery, tsoStreamBuilderFactory: factory, checkTSDeadlineCh: make(chan struct{}), @@ -101,6 +108,7 @@ func newTSOClient(ctx context.Context, cancel context.CancelFunc, wg *sync.WaitG updateTSOConnectionCtxsCh: make(chan struct{}, 1), } + eventSrc := svcDiscovery.(tsoAllocatorEventSource) eventSrc.SetTSOLocalServAddrsUpdatedCallback(c.updateTSOLocalServAddrs) eventSrc.SetTSOGlobalServAddrUpdatedCallback(c.updateTSOGlobalServAddr) c.svcDiscovery.AddServiceAddrsSwitchedCallback(c.scheduleUpdateTSOConnectionCtxs) @@ -108,31 +116,38 @@ func newTSOClient(ctx context.Context, cancel context.CancelFunc, wg *sync.WaitG return c } -func (c *tsoClient) setup() error { - if err := c.svcDiscovery.Init(); err != nil { - return err - } +func (c *tsoClient) Setup() { + c.svcDiscovery.CheckMemberChanged() + c.updateTSODispatcher() // Start the daemons. c.wg.Add(2) go c.tsoDispatcherCheckLoop() go c.tsCancelLoop() - return nil } // Close closes the TSO client func (c *tsoClient) Close() { + if c == nil { + return + } + log.Info("closing tso client") + + c.cancel() + c.wg.Wait() + log.Info("close tso client") c.tsoDispatcher.Range(func(_, dispatcherInterface interface{}) bool { if dispatcherInterface != nil { dispatcher := dispatcherInterface.(*tsoDispatcher) tsoErr := errors.WithStack(errClosing) - dispatcher.tsoBatchController.revokePendingTSORequest(tsoErr) + dispatcher.tsoBatchController.revokePendingRequest(tsoErr) dispatcher.dispatcherCancel() } return true }) - c.svcDiscovery.Close() + + log.Info("tso client is closed") } // GetTSOAllocators returns {dc-location -> TSO allocator leader URL} connection map @@ -194,7 +209,7 @@ func (c *tsoClient) updateTSOLocalServAddrs(allocatorMap map[string]string) erro return err } c.tsoAllocators.Store(dcLocation, addr) - log.Info("[tso] switch dc tso allocator serving address", + log.Info("[tso] switch dc tso local allocator serving address", zap.String("dc-location", dcLocation), zap.String("new-address", addr), zap.String("old-address", oldAddr)) @@ -212,7 +227,7 @@ func (c *tsoClient) updateTSOLocalServAddrs(allocatorMap map[string]string) erro func (c *tsoClient) updateTSOGlobalServAddr(addr string) error { c.tsoAllocators.Store(globalDCLocation, addr) - log.Info("[tso] switch dc tso allocator serving address", + log.Info("[tso] switch dc tso global allocator serving address", zap.String("dc-location", globalDCLocation), zap.String("new-address", addr)) c.scheduleCheckTSODispatcher() diff --git a/client/tso_dispatcher.go b/client/tso_dispatcher.go index cbf02510c59..a1e4e431e78 100644 --- a/client/tso_dispatcher.go +++ b/client/tso_dispatcher.go @@ -18,6 +18,7 @@ import ( "context" "fmt" "math/rand" + "runtime/trace" "sync" "time" @@ -27,6 +28,8 @@ import ( "github.com/pingcap/log" "github.com/tikv/pd/client/errs" "github.com/tikv/pd/client/grpcutil" + "github.com/tikv/pd/client/timerpool" + "github.com/tikv/pd/client/tsoutil" "go.uber.org/zap" "google.golang.org/grpc" "google.golang.org/grpc/codes" @@ -39,14 +42,16 @@ type tsoDispatcher struct { tsoBatchController *tsoBatchController } -type lastTSO struct { - physical int64 - logical int64 +type tsoInfo struct { + tsoServer string + reqKeyspaceGroupID uint32 + respKeyspaceGroupID uint32 + respReceivedAt time.Time + physical int64 + logical int64 } const ( - dialTimeout = 3 * time.Second - updateMemberTimeout = time.Second // Use a shorter timeout to recover faster from network isolation. tsLoopDCCheckInterval = time.Minute defaultMaxTSOBatchSize = 10000 // should be higher if client is sending requests in burst retryInterval = 500 * time.Millisecond @@ -75,6 +80,8 @@ func (c *tsoClient) dispatchRequest(dcLocation string, request *tsoRequest) erro c.svcDiscovery.ScheduleCheckMemberChanged() return err } + + defer trace.StartRegion(request.requestCtx, "tsoReqEnqueue").End() dispatcher.(*tsoDispatcher).tsoBatchController.tsoRequestCh <- request return nil } @@ -92,6 +99,7 @@ func (req *tsoRequest) Wait() (physical int64, logical int64, err error) { cmdDurationTSOAsyncWait.Observe(start.Sub(req.start).Seconds()) select { case err = <-req.done: + defer trace.StartRegion(req.requestCtx, "tsoReqDone").End() err = errors.WithStack(err) defer tsoReqPool.Put(req) if err != nil { @@ -136,11 +144,24 @@ func (c *tsoClient) updateTSODispatcher() { } type deadline struct { - timer <-chan time.Time + timer *time.Timer done chan struct{} cancel context.CancelFunc } +func newTSDeadline( + timeout time.Duration, + done chan struct{}, + cancel context.CancelFunc, +) *deadline { + timer := timerpool.GlobalTimerPool.Get(timeout) + return &deadline{ + timer: timer, + done: done, + cancel: cancel, + } +} + func (c *tsoClient) tsCancelLoop() { defer c.wg.Done() @@ -161,6 +182,7 @@ func (c *tsoClient) tsCancelLoop() { case <-ticker.C: continue case <-tsCancelLoopCtx.Done(): + log.Info("exit tso requests cancel loop") return } } @@ -168,19 +190,21 @@ func (c *tsoClient) tsCancelLoop() { func (c *tsoClient) watchTSDeadline(ctx context.Context, dcLocation string) { if _, exist := c.tsDeadline.Load(dcLocation); !exist { - tsDeadlineCh := make(chan deadline, 1) + tsDeadlineCh := make(chan *deadline, 1) c.tsDeadline.Store(dcLocation, tsDeadlineCh) - go func(dc string, tsDeadlineCh <-chan deadline) { + go func(dc string, tsDeadlineCh <-chan *deadline) { for { select { case d := <-tsDeadlineCh: select { - case <-d.timer: + case <-d.timer.C: log.Error("[tso] tso request is canceled due to timeout", zap.String("dc-location", dc), errs.ZapError(errs.ErrClientGetTSOTimeout)) d.cancel() + timerpool.GlobalTimerPool.Put(d.timer) case <-d.done: - continue + timerpool.GlobalTimerPool.Put(d.timer) case <-ctx.Done(): + timerpool.GlobalTimerPool.Put(d.timer) return } case <-ctx.Done(): @@ -212,7 +236,7 @@ func (c *tsoClient) tsoDispatcherCheckLoop() { case <-ticker.C: case <-c.checkTSODispatcherCh: case <-loopCtx.Done(): - log.Info("exit tso dispacther loop") + log.Info("exit tso dispatcher loop") return } } @@ -230,6 +254,8 @@ func (c *tsoClient) checkAllocator( }() cc, u := c.GetTSOAllocatorClientConnByDCLocation(dc) healthCli := healthpb.NewHealthClient(cc) + ticker := time.NewTicker(time.Second) + defer ticker.Stop() for { // the pd/allocator leader change, we need to re-establish the stream if u != url { @@ -255,7 +281,7 @@ func (c *tsoClient) checkAllocator( select { case <-dispatcherCtx.Done(): return - case <-time.After(time.Second): + case <-ticker.C: // To ensure we can get the latest allocator leader // and once the leader is changed, we can exit this function. _, u = c.GetTSOAllocatorClientConnByDCLocation(dc) @@ -362,6 +388,7 @@ func (c *tsoClient) handleDispatcher( // Loop through each batch of TSO requests and send them for processing. streamLoopTimer := time.NewTimer(c.option.timeout) + defer streamLoopTimer.Stop() tsoBatchLoop: for { select { @@ -377,13 +404,23 @@ tsoBatchLoop: zap.String("dc-location", dc)) } else { log.Error("[tso] fetch pending tso requests error", - zap.String("dc-location", dc), errs.ZapError(errs.ErrClientGetTSO, err)) + zap.String("dc-location", dc), + errs.ZapError(errs.ErrClientGetTSO.FastGenByArgs("when fetch pending tso requests"), err)) } return } if maxBatchWaitInterval >= 0 { tbc.adjustBestBatchSize() } + // Stop the timer if it's not stopped. + if !streamLoopTimer.Stop() { + select { + case <-streamLoopTimer.C: // try to drain from the channel + default: + } + } + // We need be careful here, see more details in the comments of Timer.Reset. + // https://pkg.go.dev/time@master#Timer.Reset streamLoopTimer.Reset(c.option.timeout) // Choose a stream to send the TSO gRPC request. streamChoosingLoop: @@ -398,16 +435,20 @@ tsoBatchLoop: if c.updateTSOConnectionCtxs(dispatcherCtx, dc, &connectionCtxs) { continue streamChoosingLoop } + timer := time.NewTimer(retryInterval) select { case <-dispatcherCtx.Done(): + timer.Stop() return case <-streamLoopTimer.C: err = errs.ErrClientCreateTSOStream.FastGenByArgs(errs.RetryTimeoutErr) log.Error("[tso] create tso stream error", zap.String("dc-location", dc), errs.ZapError(err)) c.svcDiscovery.ScheduleCheckMemberChanged() - c.finishTSORequest(tbc.getCollectedRequests(), 0, 0, 0, errors.WithStack(err)) + c.finishRequest(tbc.getCollectedRequests(), 0, 0, 0, errors.WithStack(err)) + timer.Stop() continue tsoBatchLoop - case <-time.After(retryInterval): + case <-timer.C: + timer.Stop() continue streamChoosingLoop } } @@ -424,11 +465,7 @@ tsoBatchLoop: } } done := make(chan struct{}) - dl := deadline{ - timer: time.After(c.option.timeout), - done: done, - cancel: cancel, - } + dl := newTSDeadline(c.option.timeout, done, cancel) tsDeadlineCh, ok := c.tsDeadline.Load(dc) for !ok || tsDeadlineCh == nil { c.scheduleCheckTSDeadline() @@ -438,10 +475,10 @@ tsoBatchLoop: select { case <-dispatcherCtx.Done(): return - case tsDeadlineCh.(chan deadline) <- dl: + case tsDeadlineCh.(chan *deadline) <- dl: } opts = extractSpanReference(tbc, opts[:0]) - err = c.processTSORequests(stream, dc, tbc, opts) + err = c.processRequests(stream, dc, tbc, opts) close(done) // If error happens during tso stream handling, reset stream and run the next trial. if err != nil { @@ -451,7 +488,10 @@ tsoBatchLoop: default: } c.svcDiscovery.ScheduleCheckMemberChanged() - log.Error("[tso] getTS error", zap.String("dc-location", dc), zap.String("stream-addr", streamAddr), errs.ZapError(errs.ErrClientGetTSO, err)) + log.Error("[tso] getTS error", + zap.String("dc-location", dc), + zap.String("stream-addr", streamAddr), + errs.ZapError(errs.ErrClientGetTSO.FastGenByArgs("after processing requests"), err)) // Set `stream` to nil and remove this stream from the `connectionCtxs` due to error. connectionCtxs.Delete(streamAddr) cancel() @@ -498,8 +538,8 @@ func (c *tsoClient) chooseStream(connectionCtxs *sync.Map) (connectionCtx *tsoCo type tsoConnectionContext struct { streamAddr string - // Current stream to send gRPC requests, pdpb.PD_TsoClient for a leader/follower in the PD cluser, - // or tsopb.TSO_TsoClient for a primary/secondary in the TSO clusrer + // Current stream to send gRPC requests, pdpb.PD_TsoClient for a leader/follower in the PD cluster, + // or tsopb.TSO_TsoClient for a primary/secondary in the TSO cluster stream tsoStream ctx context.Context cancel context.CancelFunc @@ -550,6 +590,8 @@ func (c *tsoClient) tryConnectToTSO( } // retry several times before falling back to the follower when the network problem happens + ticker := time.NewTicker(retryInterval) + defer ticker.Stop() for i := 0; i < maxRetryTimes; i++ { c.svcDiscovery.ScheduleCheckMemberChanged() cc, url = c.GetTSOAllocatorClientConnByDCLocation(dc) @@ -579,7 +621,7 @@ func (c *tsoClient) tryConnectToTSO( select { case <-dispatcherCtx.Done(): return err - case <-time.After(retryInterval): + case <-ticker.C: } } @@ -616,7 +658,7 @@ func (c *tsoClient) tryConnectToTSO( // or of keyspace group primary/secondaries. func (c *tsoClient) getAllTSOStreamBuilders() map[string]tsoStreamBuilder { var ( - addrs = c.svcDiscovery.GetURLs() + addrs = c.svcDiscovery.GetServiceURLs() streamBuilders = make(map[string]tsoStreamBuilder, len(addrs)) cc *grpc.ClientConn err error @@ -663,7 +705,8 @@ func (c *tsoClient) tryConnectToTSOWithProxy(dispatcherCtx context.Context, dc s cctx, cancel := context.WithCancel(dispatcherCtx) // Do not proxy the leader client. if addr != leaderAddr { - log.Info("[tso] use follower to forward tso stream to do the proxy", zap.String("dc", dc), zap.String("addr", addr)) + log.Info("[tso] use follower to forward tso stream to do the proxy", + zap.String("dc", dc), zap.String("addr", addr)) cctx = grpcutil.BuildForwardContext(cctx, forwardedHost) } // Create the TSO stream. @@ -677,7 +720,8 @@ func (c *tsoClient) tryConnectToTSOWithProxy(dispatcherCtx context.Context, dc s connectionCtxs.Store(addr, &tsoConnectionContext{addr, stream, cctx, cancel}) continue } - log.Error("[tso] create the tso stream failed", zap.String("dc", dc), zap.String("addr", addr), errs.ZapError(err)) + log.Error("[tso] create the tso stream failed", + zap.String("dc", dc), zap.String("addr", addr), errs.ZapError(err)) cancel() } return nil @@ -692,68 +736,108 @@ func extractSpanReference(tbc *tsoBatchController, opts []opentracing.StartSpanO return opts } -func (c *tsoClient) processTSORequests(stream tsoStream, dcLocation string, tbc *tsoBatchController, opts []opentracing.StartSpanOption) error { +func (c *tsoClient) processRequests( + stream tsoStream, dcLocation string, tbc *tsoBatchController, opts []opentracing.StartSpanOption, +) error { if len(opts) > 0 { - span := opentracing.StartSpan("pdclient.processTSORequests", opts...) + span := opentracing.StartSpan("pdclient.processRequests", opts...) defer span.Finish() } requests := tbc.getCollectedRequests() + for _, req := range requests { + defer trace.StartRegion(req.requestCtx, "tsoReqSend").End() + } count := int64(len(requests)) - physical, logical, suffixBits, err := stream.processRequests(c.svcDiscovery.GetClusterID(), dcLocation, requests, tbc.batchStartTime) + reqKeyspaceGroupID := c.svcDiscovery.GetKeyspaceGroupID() + respKeyspaceGroupID, physical, logical, suffixBits, err := stream.processRequests( + c.svcDiscovery.GetClusterID(), c.svcDiscovery.GetKeyspaceID(), reqKeyspaceGroupID, + dcLocation, requests, tbc.batchStartTime) if err != nil { - c.finishTSORequest(requests, 0, 0, 0, err) + c.finishRequest(requests, 0, 0, 0, err) return err } // `logical` is the largest ts's logical part here, we need to do the subtracting before we finish each TSO request. - firstLogical := addLogical(logical, -count+1, suffixBits) - c.compareAndSwapTS(dcLocation, physical, firstLogical, suffixBits, count) - c.finishTSORequest(requests, physical, firstLogical, suffixBits, nil) + firstLogical := tsoutil.AddLogical(logical, -count+1, suffixBits) + curTSOInfo := &tsoInfo{ + tsoServer: stream.getServerAddr(), + reqKeyspaceGroupID: reqKeyspaceGroupID, + respKeyspaceGroupID: respKeyspaceGroupID, + respReceivedAt: time.Now(), + physical: physical, + logical: tsoutil.AddLogical(firstLogical, count-1, suffixBits), + } + c.compareAndSwapTS(dcLocation, curTSOInfo, physical, firstLogical) + c.finishRequest(requests, physical, firstLogical, suffixBits, nil) return nil } -// Because of the suffix, we need to shift the count before we add it to the logical part. -func addLogical(logical, count int64, suffixBits uint32) int64 { - return logical + count</tso//primary" in which - // is 5 digits integer with leading zeros. For now we use 0 as the default cluster id. - tsoPrimaryPrefix = "/ms/0/tso" + msServiceRootPath = "/ms" + tsoServiceName = "tso" + // tsoSvcDiscoveryFormat defines the key prefix for keyspace group primary election. + // The entire key is in the format of "/ms//tso//primary". + // The is 5 digits integer with leading zeros. + tsoSvcDiscoveryFormat = msServiceRootPath + "/%d/" + tsoServiceName + "/%05d/primary" + // initRetryInterval is the rpc retry interval during the initialization phase. + initRetryInterval = time.Second + // tsoQueryRetryMaxTimes is the max retry times for querying TSO. + tsoQueryRetryMaxTimes = 10 + // tsoQueryRetryInterval is the retry interval for querying TSO. + tsoQueryRetryInterval = 500 * time.Millisecond ) var _ ServiceDiscovery = (*tsoServiceDiscovery)(nil) var _ tsoAllocatorEventSource = (*tsoServiceDiscovery)(nil) +// keyspaceGroupSvcDiscovery is used for discovering the serving endpoints of the keyspace +// group to which the keyspace belongs +type keyspaceGroupSvcDiscovery struct { + sync.RWMutex + group *tsopb.KeyspaceGroup + // primaryAddr is the primary serving address + primaryAddr string + // secondaryAddrs are TSO secondary serving addresses + secondaryAddrs []string + // addrs are the primary/secondary serving addresses + addrs []string +} + +func (k *keyspaceGroupSvcDiscovery) update( + keyspaceGroup *tsopb.KeyspaceGroup, + newPrimaryAddr string, + secondaryAddrs, addrs []string, +) (oldPrimaryAddr string, primarySwitched, secondaryChanged bool) { + k.Lock() + defer k.Unlock() + + // If the new primary address is empty, we don't switch the primary address. + oldPrimaryAddr = k.primaryAddr + if len(newPrimaryAddr) > 0 { + primarySwitched = !strings.EqualFold(oldPrimaryAddr, newPrimaryAddr) + k.primaryAddr = newPrimaryAddr + } + + if !reflect.DeepEqual(k.secondaryAddrs, secondaryAddrs) { + k.secondaryAddrs = secondaryAddrs + secondaryChanged = true + } + + k.group = keyspaceGroup + k.addrs = addrs + return +} + +// tsoServerDiscovery is for discovering the serving endpoints of the TSO servers +// TODO: dynamically update the TSO server addresses in the case of TSO server failover +// and scale-out/in. +type tsoServerDiscovery struct { + sync.RWMutex + addrs []string + // used for round-robin load balancing + selectIdx int + // failureCount counts the consecutive failures for communicating with the tso servers + failureCount int +} + +func (t *tsoServerDiscovery) countFailure() bool { + t.Lock() + defer t.Unlock() + t.failureCount++ + return t.failureCount >= len(t.addrs) +} + +func (t *tsoServerDiscovery) resetFailure() { + t.Lock() + defer t.Unlock() + t.failureCount = 0 +} + // tsoServiceDiscovery is the service discovery client of the independent TSO service + type tsoServiceDiscovery struct { - clusterID uint64 - keyspaceID uint32 - urls atomic.Value // Store as []string - // primary key is the etcd path used for discoverying the serving endpoint of this keyspace - primaryKey string - // TSO Primary URL - primary atomic.Value // Store as string - // TSO Secondary URLs - secondaries atomic.Value // Store as []string - metacli MetaStorageClient + metacli MetaStorageClient + apiSvcDiscovery ServiceDiscovery + clusterID uint64 + keyspaceID atomic.Uint32 + + // defaultDiscoveryKey is the etcd path used for discovering the serving endpoints of + // the default keyspace group + defaultDiscoveryKey string + // tsoServersDiscovery is for discovering the serving endpoints of the TSO servers + *tsoServerDiscovery + + // keyspaceGroupSD is for discovering the serving endpoints of the keyspace group + keyspaceGroupSD *keyspaceGroupSvcDiscovery // addr -> a gRPC connection clientConns sync.Map // Store as map[string]*grpc.ClientConn - // primarySwitchedCbs will be called after the primary swichted - primarySwitchedCbs []func() - // membersChangedCbs will be called after there is any membership - // change in the primary and followers - membersChangedCbs []func() // localAllocPrimariesUpdatedCb will be called when the local tso allocator primary list is updated. - // The input is a map {DC Localtion -> Leader Addr} + // The input is a map {DC Location -> Leader Addr} localAllocPrimariesUpdatedCb tsoLocalServAddrsUpdatedFunc // globalAllocPrimariesUpdatedCb will be called when the local tso allocator primary list is updated. globalAllocPrimariesUpdatedCb tsoGlobalServAddrUpdatedFunc checkMembershipCh chan struct{} - wg *sync.WaitGroup - ctx context.Context - cancel context.CancelFunc + ctx context.Context + cancel context.CancelFunc + wg sync.WaitGroup + printFallbackLogOnce sync.Once tlsCfg *tlsutil.TLSConfig @@ -83,28 +156,47 @@ type tsoServiceDiscovery struct { } // newTSOServiceDiscovery returns a new client-side service discovery for the independent TSO service. -func newTSOServiceDiscovery(ctx context.Context, cancel context.CancelFunc, wg *sync.WaitGroup, metacli MetaStorageClient, - clusterID uint64, keyspaceID uint32, urls []string, tlsCfg *tlsutil.TLSConfig, option *option) ServiceDiscovery { - bc := &tsoServiceDiscovery{ +func newTSOServiceDiscovery( + ctx context.Context, metacli MetaStorageClient, apiSvcDiscovery ServiceDiscovery, + clusterID uint64, keyspaceID uint32, tlsCfg *tlsutil.TLSConfig, option *option, +) ServiceDiscovery { + ctx, cancel := context.WithCancel(ctx) + c := &tsoServiceDiscovery{ ctx: ctx, cancel: cancel, - wg: wg, metacli: metacli, - keyspaceID: keyspaceID, + apiSvcDiscovery: apiSvcDiscovery, clusterID: clusterID, - primaryKey: path.Join(tsoPrimaryPrefix, fmt.Sprintf("%05d", 0), "primary"), tlsCfg: tlsCfg, option: option, checkMembershipCh: make(chan struct{}, 1), } - bc.urls.Store(urls) + c.keyspaceID.Store(keyspaceID) + c.keyspaceGroupSD = &keyspaceGroupSvcDiscovery{ + primaryAddr: "", + secondaryAddrs: make([]string, 0), + addrs: make([]string, 0), + } + c.tsoServerDiscovery = &tsoServerDiscovery{addrs: make([]string, 0)} + // Start with the default keyspace group. The actual keyspace group, to which the keyspace belongs, + // will be discovered later. + c.defaultDiscoveryKey = fmt.Sprintf(tsoSvcDiscoveryFormat, clusterID, defaultKeySpaceGroupID) + + log.Info("created tso service discovery", + zap.Uint64("cluster-id", clusterID), + zap.Uint32("keyspace-id", keyspaceID), + zap.String("default-discovery-key", c.defaultDiscoveryKey)) - return bc + return c } // Init initialize the concrete client underlying func (c *tsoServiceDiscovery) Init() error { - if err := c.initRetry(c.updateMember); err != nil { + log.Info("initializing tso service discovery", + zap.Int("max-retry-times", c.option.maxRetryTimes), + zap.Duration("retry-interval", initRetryInterval)) + if err := c.retry(c.option.maxRetryTimes, initRetryInterval, c.updateMember); err != nil { + log.Error("failed to update member. initialization failed.", zap.Error(err)) c.cancel() return err } @@ -113,61 +205,110 @@ func (c *tsoServiceDiscovery) Init() error { return nil } -func (c *tsoServiceDiscovery) initRetry(f func() error) error { +func (c *tsoServiceDiscovery) retry( + maxRetryTimes int, retryInterval time.Duration, f func() error, +) error { var err error - for i := 0; i < c.option.maxRetryTimes; i++ { + ticker := time.NewTicker(retryInterval) + defer ticker.Stop() + for i := 0; i < maxRetryTimes; i++ { if err = f(); err == nil { return nil } select { case <-c.ctx.Done(): return err - case <-time.After(time.Second): + case <-ticker.C: } } return errors.WithStack(err) } +// Close releases all resources +func (c *tsoServiceDiscovery) Close() { + log.Info("closing tso service discovery") + + c.cancel() + c.wg.Wait() + + c.clientConns.Range(func(key, cc interface{}) bool { + if err := cc.(*grpc.ClientConn).Close(); err != nil { + log.Error("[tso] failed to close gRPC clientConn", errs.ZapError(errs.ErrCloseGRPCConn, err)) + } + c.clientConns.Delete(key) + return true + }) + + log.Info("tso service discovery is closed") +} + func (c *tsoServiceDiscovery) startCheckMemberLoop() { defer c.wg.Done() ctx, cancel := context.WithCancel(c.ctx) defer cancel() + ticker := time.NewTicker(memberUpdateInterval) + defer ticker.Stop() for { select { case <-c.checkMembershipCh: - case <-time.After(memberUpdateInterval): + case <-ticker.C: case <-ctx.Done(): + log.Info("[tso] exit check member loop") return } - if err := c.updateMember(); err != nil { + // Make sure tsoQueryRetryMaxTimes * tsoQueryRetryInterval is far less than memberUpdateInterval, + // so that we can speed up the process of tso service discovery when failover happens on the + // tso service side and also ensures it won't call updateMember too frequently during normal time. + if err := c.retry(tsoQueryRetryMaxTimes, tsoQueryRetryInterval, c.updateMember); err != nil { log.Error("[tso] failed to update member", errs.ZapError(err)) } } } -// Close releases all resources -func (c *tsoServiceDiscovery) Close() { - log.Info("close tso service discovery") - c.clientConns.Range(func(key, cc interface{}) bool { - if err := cc.(*grpc.ClientConn).Close(); err != nil { - log.Error("[tso] failed to close gRPC clientConn", errs.ZapError(errs.ErrCloseGRPCConn, err)) - } - c.clientConns.Delete(key) - return true - }) -} - // GetClusterID returns the ID of the cluster func (c *tsoServiceDiscovery) GetClusterID() uint64 { return c.clusterID } -// GetURLs returns the URLs of the servers. +// GetKeyspaceID returns the ID of the keyspace +func (c *tsoServiceDiscovery) GetKeyspaceID() uint32 { + return c.keyspaceID.Load() +} + +// GetKeyspaceGroupID returns the ID of the keyspace group. If the keyspace group is unknown, +// it returns the default keyspace group ID. +func (c *tsoServiceDiscovery) GetKeyspaceGroupID() uint32 { + c.keyspaceGroupSD.RLock() + defer c.keyspaceGroupSD.RUnlock() + if c.keyspaceGroupSD.group == nil { + return defaultKeySpaceGroupID + } + return c.keyspaceGroupSD.group.Id +} + +// DiscoverServiceURLs discovers the microservice with the specified type and returns the server urls. +func (c *tsoServiceDiscovery) DiscoverMicroservice(svcType serviceType) ([]string, error) { + var urls []string + + switch svcType { + case apiService: + case tsoService: + return c.apiSvcDiscovery.DiscoverMicroservice(tsoService) + default: + panic("invalid service type") + } + + return urls, nil +} + +// GetServiceURLs returns the URLs of the tso primary/secondary addresses of this keyspace group. // For testing use. It should only be called when the client is closed. -func (c *tsoServiceDiscovery) GetURLs() []string { - return c.urls.Load().([]string) +func (c *tsoServiceDiscovery) GetServiceURLs() []string { + c.keyspaceGroupSD.RLock() + defer c.keyspaceGroupSD.RUnlock() + return c.keyspaceGroupSD.addrs } // GetServingAddr returns the grpc client connection of the serving endpoint @@ -179,7 +320,7 @@ func (c *tsoServiceDiscovery) GetServingEndpointClientConn() *grpc.ClientConn { return nil } -// GetClientConns returns the mapping {addr -> a gRPC connectio} +// GetClientConns returns the mapping {addr -> a gRPC connection} func (c *tsoServiceDiscovery) GetClientConns() *sync.Map { return &c.clientConns } @@ -191,7 +332,7 @@ func (c *tsoServiceDiscovery) GetServingAddr() string { } // GetBackupAddrs gets the addresses of the current reachable and healthy -// backup service endpoints randomly. Backup service endpoints are secondaries in +// backup service endpoints. Backup service endpoints are secondaries in // a primary/secondary configured cluster. func (c *tsoServiceDiscovery) GetBackupAddrs() []string { return c.getSecondaryAddrs() @@ -202,7 +343,7 @@ func (c *tsoServiceDiscovery) GetOrCreateGRPCConn(addr string) (*grpc.ClientConn return grpcutil.GetOrCreateGRPCConn(c.ctx, &c.clientConns, addr, c.tlsCfg, c.option.gRPCDialOptions...) } -// ScheduleCheckMemberChanged is used to trigger a check to see if there is any change in ervice endpoints. +// ScheduleCheckMemberChanged is used to trigger a check to see if there is any change in service endpoints. func (c *tsoServiceDiscovery) ScheduleCheckMemberChanged() { select { case c.checkMembershipCh <- struct{}{}: @@ -213,19 +354,22 @@ func (c *tsoServiceDiscovery) ScheduleCheckMemberChanged() { // Immediately check if there is any membership change among the primary/secondaries in // a primary/secondary configured cluster. func (c *tsoServiceDiscovery) CheckMemberChanged() error { - return c.updateMember() + c.apiSvcDiscovery.CheckMemberChanged() + if err := c.retry(tsoQueryRetryMaxTimes, tsoQueryRetryInterval, c.updateMember); err != nil { + log.Error("[tso] failed to update member", errs.ZapError(err)) + return err + } + return nil } // AddServingAddrSwitchedCallback adds callbacks which will be called when the primary in // a primary/secondary configured cluster is switched. func (c *tsoServiceDiscovery) AddServingAddrSwitchedCallback(callbacks ...func()) { - c.primarySwitchedCbs = append(c.primarySwitchedCbs, callbacks...) } // AddServiceAddrsSwitchedCallback adds callbacks which will be called when any primary/secondary // in a primary/secondary configured cluster is changed. func (c *tsoServiceDiscovery) AddServiceAddrsSwitchedCallback(callbacks ...func()) { - c.membersChangedCbs = append(c.membersChangedCbs, callbacks...) } // SetTSOLocalServAddrsUpdatedCallback adds a callback which will be called when the local tso @@ -237,72 +381,263 @@ func (c *tsoServiceDiscovery) SetTSOLocalServAddrsUpdatedCallback(callback tsoLo // SetTSOGlobalServAddrUpdatedCallback adds a callback which will be called when the global tso // allocator leader is updated. func (c *tsoServiceDiscovery) SetTSOGlobalServAddrUpdatedCallback(callback tsoGlobalServAddrUpdatedFunc) { + addr := c.getPrimaryAddr() + if len(addr) > 0 { + callback(addr) + } c.globalAllocPrimariesUpdatedCb = callback } // getPrimaryAddr returns the primary address. func (c *tsoServiceDiscovery) getPrimaryAddr() string { - primaryAddr := c.primary.Load() - if primaryAddr == nil { - return "" - } - return primaryAddr.(string) + c.keyspaceGroupSD.RLock() + defer c.keyspaceGroupSD.RUnlock() + return c.keyspaceGroupSD.primaryAddr } // getSecondaryAddrs returns the secondary addresses. func (c *tsoServiceDiscovery) getSecondaryAddrs() []string { - secondaryAddrs := c.secondaries.Load() - if secondaryAddrs == nil { - return []string{} - } - return secondaryAddrs.([]string) + c.keyspaceGroupSD.RLock() + defer c.keyspaceGroupSD.RUnlock() + return c.keyspaceGroupSD.secondaryAddrs } -func (c *tsoServiceDiscovery) switchPrimary(addrs []string) error { - // FIXME: How to safely compare primary urls? For now, only allows one client url. - addr := addrs[0] - oldPrimary := c.getPrimaryAddr() - if addr == oldPrimary { - return nil +func (c *tsoServiceDiscovery) afterPrimarySwitched(oldPrimary, newPrimary string) error { + // Run callbacks + if c.globalAllocPrimariesUpdatedCb != nil { + if err := c.globalAllocPrimariesUpdatedCb(newPrimary); err != nil { + return err + } } + log.Info("[tso] switch primary", + zap.String("new-primary", newPrimary), + zap.String("old-primary", oldPrimary)) + return nil +} - if _, err := c.GetOrCreateGRPCConn(addr); err != nil { - log.Warn("[tso] failed to connect primary", zap.String("primary", addr), errs.ZapError(err)) +func (c *tsoServiceDiscovery) updateMember() error { + // The keyspace membership or the primary serving address of the keyspace group, to which this + // keyspace belongs, might have been changed. We need to query tso servers to get the latest info. + tsoServerAddr, err := c.getTSOServer(c.apiSvcDiscovery) + if err != nil { + log.Error("[tso] failed to get tso server", errs.ZapError(err)) return err } - // Set PD primary and Global TSO Allocator (which is also the PD primary) - c.primary.Store(addr) - // Run callbacks - if c.globalAllocPrimariesUpdatedCb != nil { - if err := c.globalAllocPrimariesUpdatedCb(addr); err != nil { + + keyspaceID := c.GetKeyspaceID() + var keyspaceGroup *tsopb.KeyspaceGroup + if len(tsoServerAddr) > 0 { + keyspaceGroup, err = c.findGroupByKeyspaceID(keyspaceID, tsoServerAddr, updateMemberTimeout) + if err != nil { + if c.tsoServerDiscovery.countFailure() { + log.Error("[tso] failed to find the keyspace group", + zap.Uint32("keyspace-id-in-request", keyspaceID), + zap.String("tso-server-addr", tsoServerAddr), + errs.ZapError(err)) + } + return err + } + c.tsoServerDiscovery.resetFailure() + } else { + // There is no error but no tso server address found, which means + // the server side hasn't been upgraded to the version that + // processes and returns GetClusterInfoResponse.TsoUrls. In this case, + // we fall back to the old way of discovering the tso primary addresses + // from etcd directly. + c.printFallbackLogOnce.Do(func() { + log.Warn("[tso] no tso server address found,"+ + " fallback to the legacy path to discover from etcd directly", + zap.Uint32("keyspace-id-in-request", keyspaceID), + zap.String("tso-server-addr", tsoServerAddr), + zap.String("discovery-key", c.defaultDiscoveryKey)) + }) + addrs, err := c.discoverWithLegacyPath() + if err != nil { + return err + } + if len(addrs) == 0 { + return errors.New("no tso server address found") + } + members := make([]*tsopb.KeyspaceGroupMember, 0, len(addrs)) + for _, addr := range addrs { + members = append(members, &tsopb.KeyspaceGroupMember{Address: addr}) + } + members[0].IsPrimary = true + keyspaceGroup = &tsopb.KeyspaceGroup{ + Id: defaultKeySpaceGroupID, + Members: members, + } + } + + oldGroupID := c.GetKeyspaceGroupID() + if oldGroupID != keyspaceGroup.Id { + log.Info("[tso] the keyspace group changed", + zap.Uint32("keyspace-id", keyspaceGroup.Id), + zap.Uint32("new-keyspace-group-id", keyspaceGroup.Id), + zap.Uint32("old-keyspace-group-id", oldGroupID)) + } + + // Initialize the serving addresses from the returned keyspace group info. + primaryAddr := "" + secondaryAddrs := make([]string, 0) + addrs := make([]string, 0, len(keyspaceGroup.Members)) + for _, m := range keyspaceGroup.Members { + addrs = append(addrs, m.Address) + if m.IsPrimary { + primaryAddr = m.Address + } else { + secondaryAddrs = append(secondaryAddrs, m.Address) + } + } + + // If the primary address is not empty, we need to create a grpc connection to it, and do it + // out of the critical section of the keyspace group service discovery. + if len(primaryAddr) > 0 { + if primarySwitched := !strings.EqualFold(primaryAddr, c.getPrimaryAddr()); primarySwitched { + if _, err := c.GetOrCreateGRPCConn(primaryAddr); err != nil { + log.Warn("[tso] failed to connect the next primary", + zap.Uint32("keyspace-id-in-request", keyspaceID), + zap.String("tso-server-addr", tsoServerAddr), + zap.String("next-primary", primaryAddr), errs.ZapError(err)) + return err + } + } + } + + oldPrimary, primarySwitched, _ := + c.keyspaceGroupSD.update(keyspaceGroup, primaryAddr, secondaryAddrs, addrs) + if primarySwitched { + log.Info("[tso] updated keyspace group service discovery info", + zap.Uint32("keyspace-id-in-request", keyspaceID), + zap.String("tso-server-addr", tsoServerAddr), + zap.String("keyspace-group-service", keyspaceGroup.String())) + if err := c.afterPrimarySwitched(oldPrimary, primaryAddr); err != nil { return err } } - for _, cb := range c.primarySwitchedCbs { - cb() + + // Even if the primary address is empty, we still updated other returned info above, including the + // keyspace group info and the secondary addresses. + if len(primaryAddr) == 0 { + return errors.New("no primary address found") } - log.Info("[tso] switch primary", zap.String("new-primary", addr), zap.String("old-primary", oldPrimary)) + return nil } -func (c *tsoServiceDiscovery) updateMember() error { - resp, err := c.metacli.Get(c.ctx, []byte(c.primaryKey)) +// Query the keyspace group info from the tso server by the keyspace ID. The server side will return +// the info of the keyspace group to which this keyspace belongs. +func (c *tsoServiceDiscovery) findGroupByKeyspaceID( + keyspaceID uint32, tsoSrvAddr string, timeout time.Duration, +) (*tsopb.KeyspaceGroup, error) { + failpoint.Inject("unexpectedCallOfFindGroupByKeyspaceID", func(val failpoint.Value) { + keyspaceToCheck, ok := val.(int) + if ok && keyspaceID == uint32(keyspaceToCheck) { + panic("findGroupByKeyspaceID is called unexpectedly") + } + }) + ctx, cancel := context.WithTimeout(c.ctx, timeout) + defer cancel() + + cc, err := c.GetOrCreateGRPCConn(tsoSrvAddr) if err != nil { - log.Error("[tso] failed to get the keyspace serving endpoint", errs.ZapError(err)) - return err + return nil, err + } + + resp, err := tsopb.NewTSOClient(cc).FindGroupByKeyspaceID( + ctx, &tsopb.FindGroupByKeyspaceIDRequest{ + Header: &tsopb.RequestHeader{ + ClusterId: c.clusterID, + KeyspaceId: keyspaceID, + KeyspaceGroupId: defaultKeySpaceGroupID, + }, + KeyspaceId: keyspaceID, + }) + if err != nil { + attachErr := errors.Errorf("error:%s target:%s status:%s", + err, cc.Target(), cc.GetState().String()) + return nil, errs.ErrClientFindGroupByKeyspaceID.Wrap(attachErr).GenWithStackByCause() + } + if resp.GetHeader().GetError() != nil { + attachErr := errors.Errorf("error:%s target:%s status:%s", + resp.GetHeader().GetError().String(), cc.Target(), cc.GetState().String()) + return nil, errs.ErrClientFindGroupByKeyspaceID.Wrap(attachErr).GenWithStackByCause() + } + if resp.KeyspaceGroup == nil { + attachErr := errors.Errorf("error:%s target:%s status:%s", + "no keyspace group found", cc.Target(), cc.GetState().String()) + return nil, errs.ErrClientFindGroupByKeyspaceID.Wrap(attachErr).GenWithStackByCause() } + return resp.KeyspaceGroup, nil +} + +func (c *tsoServiceDiscovery) getTSOServer(sd ServiceDiscovery) (string, error) { + c.Lock() + defer c.Unlock() + + var ( + addrs []string + err error + ) + t := c.tsoServerDiscovery + if len(t.addrs) == 0 || t.failureCount == len(t.addrs) { + addrs, err = sd.DiscoverMicroservice(tsoService) + if err != nil { + return "", err + } + failpoint.Inject("serverReturnsNoTSOAddrs", func() { + log.Info("[failpoint] injected error: server returns no tso addrs") + addrs = nil + }) + if len(addrs) == 0 { + // There is no error but no tso server address found, which means + // the server side hasn't been upgraded to the version that + // processes and returns GetClusterInfoResponse.TsoUrls. Return here + // and handle the fallback logic outside of this function. + return "", nil + } + + log.Info("update tso server addresses", zap.Strings("addrs", addrs)) + + t.addrs = addrs + t.selectIdx = 0 + t.failureCount = 0 + } + + // Pick a TSO server in a round-robin way. + tsoServerAddr := t.addrs[t.selectIdx] + t.selectIdx++ + t.selectIdx %= len(t.addrs) + + return tsoServerAddr, nil +} + +func (c *tsoServiceDiscovery) discoverWithLegacyPath() ([]string, error) { + resp, err := c.metacli.Get(c.ctx, []byte(c.defaultDiscoveryKey)) + if err != nil { + log.Error("[tso] failed to get the keyspace serving endpoint", + zap.String("discovery-key", c.defaultDiscoveryKey), errs.ZapError(err)) + return nil, err + } if resp == nil || len(resp.Kvs) == 0 { - log.Error("[tso] didn't find the keyspace serving endpoint") - return errs.ErrClientGetLeader + log.Error("[tso] didn't find the keyspace serving endpoint", + zap.String("primary-key", c.defaultDiscoveryKey)) + return nil, errs.ErrClientGetServingEndpoint } else if resp.Count > 1 { - return errs.ErrClientGetMultiResponse.FastGenByArgs(resp.Kvs) + return nil, errs.ErrClientGetMultiResponse.FastGenByArgs(resp.Kvs) } value := resp.Kvs[0].Value - member := &pdpb.Member{} - if err := proto.Unmarshal(value, member); err != nil { - return errs.ErrClientProtoUnmarshal.Wrap(err).GenWithStackByCause() + primary := &tsopb.Participant{} + if err := proto.Unmarshal(value, primary); err != nil { + return nil, errs.ErrClientProtoUnmarshal.Wrap(err).GenWithStackByCause() + } + listenUrls := primary.GetListenUrls() + if len(listenUrls) == 0 { + log.Error("[tso] the keyspace serving endpoint list is empty", + zap.String("discovery-key", c.defaultDiscoveryKey)) + return nil, errs.ErrClientGetServingEndpoint } - return c.switchPrimary(addrsToUrls([]string{member.Name})) + return listenUrls, nil } diff --git a/client/tso_stream.go b/client/tso_stream.go index f50865b9748..e3203818938 100644 --- a/client/tso_stream.go +++ b/client/tso_stream.go @@ -16,11 +16,13 @@ package pd import ( "context" + "io" "time" "github.com/pingcap/errors" "github.com/pingcap/kvproto/pkg/pdpb" "github.com/pingcap/kvproto/pkg/tsopb" + "github.com/tikv/pd/client/errs" "google.golang.org/grpc" ) @@ -33,13 +35,13 @@ type tsoStreamBuilderFactory interface { type pdTSOStreamBuilderFactory struct{} func (f *pdTSOStreamBuilderFactory) makeBuilder(cc *grpc.ClientConn) tsoStreamBuilder { - return &pdTSOStreamBuilder{client: pdpb.NewPDClient(cc)} + return &pdTSOStreamBuilder{client: pdpb.NewPDClient(cc), serverAddr: cc.Target()} } type tsoTSOStreamBuilderFactory struct{} func (f *tsoTSOStreamBuilderFactory) makeBuilder(cc *grpc.ClientConn) tsoStreamBuilder { - return &tsoTSOStreamBuilder{client: tsopb.NewTSOClient(cc)} + return &tsoTSOStreamBuilder{client: tsopb.NewTSOClient(cc), serverAddr: cc.Target()} } // TSO Stream Builder @@ -49,7 +51,8 @@ type tsoStreamBuilder interface { } type pdTSOStreamBuilder struct { - client pdpb.PDClient + serverAddr string + client pdpb.PDClient } func (b *pdTSOStreamBuilder) build(ctx context.Context, cancel context.CancelFunc, timeout time.Duration) (tsoStream, error) { @@ -59,32 +62,37 @@ func (b *pdTSOStreamBuilder) build(ctx context.Context, cancel context.CancelFun stream, err := b.client.Tso(ctx) done <- struct{}{} if err == nil { - return &pdTSOStream{stream: stream}, nil + return &pdTSOStream{stream: stream, serverAddr: b.serverAddr}, nil } return nil, err } type tsoTSOStreamBuilder struct { - client tsopb.TSOClient + serverAddr string + client tsopb.TSOClient } -func (b *tsoTSOStreamBuilder) build(ctx context.Context, cancel context.CancelFunc, timeout time.Duration) (tsoStream, error) { +func (b *tsoTSOStreamBuilder) build( + ctx context.Context, cancel context.CancelFunc, timeout time.Duration, +) (tsoStream, error) { done := make(chan struct{}) // TODO: we need to handle a conner case that this goroutine is timeout while the stream is successfully created. go checkStreamTimeout(ctx, cancel, done, timeout) stream, err := b.client.Tso(ctx) done <- struct{}{} if err == nil { - return &tsoTSOStream{stream: stream}, nil + return &tsoTSOStream{stream: stream, serverAddr: b.serverAddr}, nil } return nil, err } func checkStreamTimeout(ctx context.Context, cancel context.CancelFunc, done chan struct{}, timeout time.Duration) { + timer := time.NewTimer(timeout) + defer timer.Stop() select { case <-done: return - case <-time.After(timeout): + case <-timer.C: cancel() case <-ctx.Done(): } @@ -94,17 +102,26 @@ func checkStreamTimeout(ctx context.Context, cancel context.CancelFunc, done cha // TSO Stream type tsoStream interface { + getServerAddr() string // processRequests processes TSO requests in streaming mode to get timestamps - processRequests(clusterID uint64, dcLocation string, requests []*tsoRequest, - batchStartTime time.Time) (physical, logical int64, suffixBits uint32, err error) + processRequests( + clusterID uint64, keyspaceID, keyspaceGroupID uint32, dcLocation string, + requests []*tsoRequest, batchStartTime time.Time, + ) (respKeyspaceGroupID uint32, physical, logical int64, suffixBits uint32, err error) } type pdTSOStream struct { - stream pdpb.PD_TsoClient + serverAddr string + stream pdpb.PD_TsoClient } -func (s *pdTSOStream) processRequests(clusterID uint64, dcLocation string, requests []*tsoRequest, - batchStartTime time.Time) (physical, logical int64, suffixBits uint32, err error) { +func (s *pdTSOStream) getServerAddr() string { + return s.serverAddr +} + +func (s *pdTSOStream) processRequests( + clusterID uint64, _, _ uint32, dcLocation string, requests []*tsoRequest, batchStartTime time.Time, +) (respKeyspaceGroupID uint32, physical, logical int64, suffixBits uint32, err error) { start := time.Now() count := int64(len(requests)) req := &pdpb.TsoRequest{ @@ -116,13 +133,21 @@ func (s *pdTSOStream) processRequests(clusterID uint64, dcLocation string, reque } if err = s.stream.Send(req); err != nil { - err = errors.WithStack(err) + if err == io.EOF { + err = errs.ErrClientTSOStreamClosed + } else { + err = errors.WithStack(err) + } return } tsoBatchSendLatency.Observe(float64(time.Since(batchStartTime))) resp, err := s.stream.Recv() if err != nil { - err = errors.WithStack(err) + if err == io.EOF { + err = errs.ErrClientTSOStreamClosed + } else { + err = errors.WithStack(err) + } return } requestDurationTSO.Observe(time.Since(start).Seconds()) @@ -133,34 +158,53 @@ func (s *pdTSOStream) processRequests(clusterID uint64, dcLocation string, reque return } - physical, logical, suffixBits = resp.GetTimestamp().GetPhysical(), resp.GetTimestamp().GetLogical(), resp.GetTimestamp().GetSuffixBits() + ts := resp.GetTimestamp() + respKeyspaceGroupID = defaultKeySpaceGroupID + physical, logical, suffixBits = ts.GetPhysical(), ts.GetLogical(), ts.GetSuffixBits() return } type tsoTSOStream struct { - stream tsopb.TSO_TsoClient + serverAddr string + stream tsopb.TSO_TsoClient } -func (s *tsoTSOStream) processRequests(clusterID uint64, dcLocation string, requests []*tsoRequest, - batchStartTime time.Time) (physical, logical int64, suffixBits uint32, err error) { +func (s *tsoTSOStream) getServerAddr() string { + return s.serverAddr +} + +func (s *tsoTSOStream) processRequests( + clusterID uint64, keyspaceID, keyspaceGroupID uint32, dcLocation string, + requests []*tsoRequest, batchStartTime time.Time, +) (respKeyspaceGroupID uint32, physical, logical int64, suffixBits uint32, err error) { start := time.Now() count := int64(len(requests)) req := &tsopb.TsoRequest{ Header: &tsopb.RequestHeader{ - ClusterId: clusterID, + ClusterId: clusterID, + KeyspaceId: keyspaceID, + KeyspaceGroupId: keyspaceGroupID, }, Count: uint32(count), DcLocation: dcLocation, } if err = s.stream.Send(req); err != nil { - err = errors.WithStack(err) + if err == io.EOF { + err = errs.ErrClientTSOStreamClosed + } else { + err = errors.WithStack(err) + } return } tsoBatchSendLatency.Observe(float64(time.Since(batchStartTime))) resp, err := s.stream.Recv() if err != nil { - err = errors.WithStack(err) + if err == io.EOF { + err = errs.ErrClientTSOStreamClosed + } else { + err = errors.WithStack(err) + } return } requestDurationTSO.Observe(time.Since(start).Seconds()) @@ -171,6 +215,8 @@ func (s *tsoTSOStream) processRequests(clusterID uint64, dcLocation string, requ return } - physical, logical, suffixBits = resp.GetTimestamp().GetPhysical(), resp.GetTimestamp().GetLogical(), resp.GetTimestamp().GetSuffixBits() + ts := resp.GetTimestamp() + respKeyspaceGroupID = resp.GetHeader().GetKeyspaceGroupId() + physical, logical, suffixBits = ts.GetPhysical(), ts.GetLogical(), ts.GetSuffixBits() return } diff --git a/client/tsoutil/tsoutil.go b/client/tsoutil/tsoutil.go new file mode 100644 index 00000000000..ffc449640ac --- /dev/null +++ b/client/tsoutil/tsoutil.go @@ -0,0 +1,46 @@ +// Copyright 2023 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tsoutil + +import ( + "github.com/pingcap/kvproto/pkg/pdpb" +) + +// AddLogical shifts the count before we add it to the logical part. +func AddLogical(logical, count int64, suffixBits uint32) int64 { + return logical + count< tsoTwo, returns 1. +// If tsoOne = tsoTwo, returns 0. +// If tsoOne < tsoTwo, returns -1. +func CompareTimestamp(tsoOne, tsoTwo *pdpb.Timestamp) int { + if tsoOne.GetPhysical() > tsoTwo.GetPhysical() || (tsoOne.GetPhysical() == tsoTwo.GetPhysical() && tsoOne.GetLogical() > tsoTwo.GetLogical()) { + return 1 + } + if tsoOne.GetPhysical() == tsoTwo.GetPhysical() && tsoOne.GetLogical() == tsoTwo.GetLogical() { + return 0 + } + return -1 +} diff --git a/cmd/pd-server/main.go b/cmd/pd-server/main.go index 3f593f143b6..eb64e353daf 100644 --- a/cmd/pd-server/main.go +++ b/cmd/pd-server/main.go @@ -26,7 +26,8 @@ import ( "github.com/tikv/pd/pkg/autoscaling" "github.com/tikv/pd/pkg/dashboard" "github.com/tikv/pd/pkg/errs" - resource_manager "github.com/tikv/pd/pkg/mcs/resource_manager/server" + resource_manager "github.com/tikv/pd/pkg/mcs/resourcemanager/server" + scheduling "github.com/tikv/pd/pkg/mcs/scheduling/server" tso "github.com/tikv/pd/pkg/mcs/tso/server" "github.com/tikv/pd/pkg/schedule/schedulers" "github.com/tikv/pd/pkg/swaggerserver" @@ -67,6 +68,7 @@ func NewServiceCommand() *cobra.Command { } cmd.AddCommand(NewTSOServiceCommand()) cmd.AddCommand(NewResourceManagerServiceCommand()) + cmd.AddCommand(NewSchedulingServiceCommand()) cmd.AddCommand(NewAPIServiceCommand()) return cmd } @@ -82,9 +84,32 @@ func NewTSOServiceCommand() *cobra.Command { cmd.Flags().StringP("config", "", "", "config file") cmd.Flags().StringP("backend-endpoints", "", "", "url for etcd client") cmd.Flags().StringP("listen-addr", "", "", "listen address for tso service") + cmd.Flags().StringP("advertise-listen-addr", "", "", "advertise urls for listen address (default '${listen-addr}')") cmd.Flags().StringP("cacert", "", "", "path of file that contains list of trusted TLS CAs") cmd.Flags().StringP("cert", "", "", "path of file that contains X509 certificate in PEM format") cmd.Flags().StringP("key", "", "", "path of file that contains X509 key in PEM format") + cmd.Flags().StringP("log-level", "L", "info", "log level: debug, info, warn, error, fatal (default 'info')") + cmd.Flags().StringP("log-file", "", "", "log file path") + return cmd +} + +// NewSchedulingServiceCommand returns the scheduling service command. +func NewSchedulingServiceCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "scheduling", + Short: "Run the scheduling service", + Run: scheduling.CreateServerWrapper, + } + cmd.Flags().BoolP("version", "V", false, "print version information and exit") + cmd.Flags().StringP("config", "", "", "config file") + cmd.Flags().StringP("backend-endpoints", "", "", "url for etcd client") + cmd.Flags().StringP("listen-addr", "", "", "listen address for tso service") + cmd.Flags().StringP("advertise-listen-addr", "", "", "advertise urls for listen address (default '${listen-addr}')") + cmd.Flags().StringP("cacert", "", "", "path of file that contains list of trusted TLS CAs") + cmd.Flags().StringP("cert", "", "", "path of file that contains X509 certificate in PEM format") + cmd.Flags().StringP("key", "", "", "path of file that contains X509 key in PEM format") + cmd.Flags().StringP("log-level", "L", "info", "log level: debug, info, warn, error, fatal (default 'info')") + cmd.Flags().StringP("log-file", "", "", "log file path") return cmd } @@ -99,9 +124,12 @@ func NewResourceManagerServiceCommand() *cobra.Command { cmd.Flags().StringP("config", "", "", "config file") cmd.Flags().StringP("backend-endpoints", "", "", "url for etcd client") cmd.Flags().StringP("listen-addr", "", "", "listen address for resource management service") + cmd.Flags().StringP("advertise-listen-addr", "", "", "advertise urls for listen address (default '${listen-addr}')") cmd.Flags().StringP("cacert", "", "", "path of file that contains list of trusted TLS CAs") cmd.Flags().StringP("cert", "", "", "path of file that contains X509 certificate in PEM format") cmd.Flags().StringP("key", "", "", "path of file that contains X509 key in PEM format") + cmd.Flags().StringP("log-level", "L", "info", "log level: debug, info, warn, error, fatal (default 'info')") + cmd.Flags().StringP("log-file", "", "", "log file path") return cmd } @@ -122,10 +150,10 @@ func addFlags(cmd *cobra.Command) { cmd.Flags().BoolP("config-check", "", false, "check config file validity and exit") cmd.Flags().StringP("name", "", "", "human-readable name for this pd member") cmd.Flags().StringP("data-dir", "", "", "path to the data directory (default 'default.${name}')") - cmd.Flags().StringP("client-urls", "", "http://127.0.0.1:2379", "url for client traffic") - cmd.Flags().StringP("advertise-client-urls", "", "", "advertise url for client traffic (default '${client-urls}')") - cmd.Flags().StringP("peer-urls", "", "http://127.0.0.1:2380", "url for peer traffic") - cmd.Flags().StringP("advertise-peer-urls", "", "", "advertise url for peer traffic (default '${peer-urls}')") + cmd.Flags().StringP("client-urls", "", "", "urls for client traffic") + cmd.Flags().StringP("advertise-client-urls", "", "", "advertise urls for client traffic (default '${client-urls}')") + cmd.Flags().StringP("peer-urls", "", "", "urls for peer traffic") + cmd.Flags().StringP("advertise-peer-urls", "", "", "advertise urls for peer traffic (default '${peer-urls}')") cmd.Flags().StringP("initial-cluster", "", "", "initial cluster configuration for bootstrapping, e,g. pd=http://127.0.0.1:2380") cmd.Flags().StringP("join", "", "", "join to an existing cluster (usage: cluster's '${advertise-client-urls}'") cmd.Flags().StringP("metrics-addr", "", "", "prometheus pushgateway address, leaves it empty will disable prometheus push") @@ -206,7 +234,10 @@ func start(cmd *cobra.Command, args []string, services ...string) { // Creates server. ctx, cancel := context.WithCancel(context.Background()) - serviceBuilders := []server.HandlerBuilder{api.NewHandler, apiv2.NewV2Handler, swaggerserver.NewHandler, autoscaling.NewHandler} + serviceBuilders := []server.HandlerBuilder{api.NewHandler, apiv2.NewV2Handler, autoscaling.NewHandler} + if swaggerserver.Enabled() { + serviceBuilders = append(serviceBuilders, swaggerserver.NewHandler) + } serviceBuilders = append(serviceBuilders, dashboard.GetServiceBuilders()...) svr, err := server.CreateServer(ctx, cfg, services, serviceBuilders...) if err != nil { diff --git a/codecov.yml b/codecov.yml index dc58a648479..dc0f8a985ba 100644 --- a/codecov.yml +++ b/codecov.yml @@ -10,3 +10,20 @@ coverage: # basic target: auto threshold: 3% + +comment: + layout: "header, diff, flags" + behavior: default + require_changes: false + +flag_management: + default_rules: # the rules that will be followed for any flag added, generally + carryforward: true + statuses: + - type: project + target: 74% # increase it if you want to enforce higher coverage for project, current setting as 74% is for do not let the error be reported and lose the meaning of warning. + - type: patch + target: 74% # increase it if you want to enforce higher coverage for project, current setting as 74% is for do not let the error be reported and lose the meaning of warning. + +ignore: + - tests/** # integration test cases or tools. diff --git a/errors.toml b/errors.toml index ede86964087..43fc6a582aa 100644 --- a/errors.toml +++ b/errors.toml @@ -63,7 +63,7 @@ create TSO stream failed, %s ["PD:client:ErrClientGetLeader"] error = ''' -get leader from %v error +get leader failed, %v ''' ["PD:client:ErrClientGetMember"] @@ -71,6 +71,11 @@ error = ''' get member failed ''' +["PD:client:ErrClientGetMinTSO"] +error = ''' +get min TSO failed, %v +''' + ["PD:client:ErrClientGetTSO"] error = ''' get TSO failed, %v @@ -91,6 +96,11 @@ error = ''' TiKV cluster not bootstrapped, please start TiKV first ''' +["PD:cluster:ErrSchedulingIsHalted"] +error = ''' +scheduling is halted +''' + ["PD:cluster:ErrStoreIsUp"] error = ''' store is still up, please remove store gracefully @@ -481,6 +491,11 @@ error = ''' init file log error, %s ''' +["PD:member:ErrCheckCampaign"] +error = ''' +check campaign failed +''' + ["PD:member:ErrEtcdLeaderNotFound"] error = ''' etcd leader not found @@ -576,6 +591,21 @@ error = ''' region label rule not found for id %s ''' +["PD:resourcemanager:ErrDeleteReservedGroup"] +error = ''' +cannot delete reserved group +''' + +["PD:resourcemanager:ErrGroupNotExists"] +error = ''' +the %s resource group does not exist +''' + +["PD:resourcemanager:ErrInvalidGroup"] +error = ''' +invalid group settings, please check the group name, priority and the number of resources +''' + ["PD:schedule:ErrCreateOperator"] error = ''' unable to create operator, %s @@ -711,11 +741,56 @@ error = ''' get allocator failed, %s ''' +["PD:tso:ErrGetAllocatorManager"] +error = ''' +get allocator manager failed, %s +''' + ["PD:tso:ErrGetLocalAllocator"] error = ''' get local allocator failed, %s ''' +["PD:tso:ErrGetMinTS"] +error = ''' +get min ts failed, %s +''' + +["PD:tso:ErrKeyspaceGroupIDInvalid"] +error = ''' +the keyspace group id is invalid, %s +''' + +["PD:tso:ErrKeyspaceGroupIsMerging"] +error = ''' +the keyspace group %d is merging +''' + +["PD:tso:ErrKeyspaceGroupNotInitialized"] +error = ''' +the keyspace group %d isn't initialized +''' + +["PD:tso:ErrKeyspaceNotAssigned"] +error = ''' +the keyspace %d isn't assigned to any keyspace group +''' + +["PD:tso:ErrLoadKeyspaceGroupsRetryExhausted"] +error = ''' +load keyspace groups retry exhausted, %s +''' + +["PD:tso:ErrLoadKeyspaceGroupsTerminated"] +error = ''' +load keyspace groups terminated +''' + +["PD:tso:ErrLoadKeyspaceGroupsTimeout"] +error = ''' +load keyspace groups timeout +''' + ["PD:tso:ErrLogicOverflow"] error = ''' logic part overflow diff --git a/go.mod b/go.mod index b34f6c91997..da75ab13372 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,10 @@ go 1.20 require ( github.com/AlekSi/gocov-xml v1.0.0 github.com/BurntSushi/toml v0.3.1 - github.com/aws/aws-sdk-go v1.35.3 + github.com/aws/aws-sdk-go-v2/config v1.18.19 + github.com/aws/aws-sdk-go-v2/credentials v1.13.18 + github.com/aws/aws-sdk-go-v2/service/kms v1.20.8 + github.com/aws/aws-sdk-go-v2/service/sts v1.18.7 github.com/axw/gocov v1.0.0 github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e @@ -13,6 +16,7 @@ require ( github.com/docker/go-units v0.4.0 github.com/elliotchance/pie/v2 v2.1.0 github.com/gin-contrib/cors v1.4.0 + github.com/gin-contrib/pprof v1.4.0 github.com/gin-gonic/gin v1.8.1 github.com/go-echarts/go-echarts v1.0.0 github.com/gogo/protobuf v1.3.2 @@ -25,32 +29,32 @@ require ( github.com/phf/go-queue v0.0.0-20170504031614-9abe38d0371d github.com/pingcap/errcode v0.3.0 github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c - github.com/pingcap/failpoint v0.0.0-20200702092429-9f69995143ce - github.com/pingcap/kvproto v0.0.0-20230228041042-1e9aca94bab6 + github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00 + github.com/pingcap/kvproto v0.0.0-20230727073445-53e1f8730c30 github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 - github.com/pingcap/sysutil v0.0.0-20211208032423-041a72e5860d - github.com/pingcap/tidb-dashboard v0.0.0-20230209052558-a58fc2a7e924 + github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21 + github.com/pingcap/tidb-dashboard v0.0.0-20230705095454-5e220f970f27 github.com/prometheus/client_golang v1.11.1 github.com/prometheus/common v0.26.0 github.com/sasha-s/go-deadlock v0.2.0 - github.com/shirou/gopsutil/v3 v3.22.12 + github.com/shirou/gopsutil/v3 v3.23.3 github.com/smallnest/chanx v0.0.0-20221229104322-eb4c998d2072 github.com/spf13/cobra v1.0.0 github.com/spf13/pflag v1.0.5 - github.com/stretchr/testify v1.8.1 - github.com/swaggo/http-swagger v0.0.0-20200308142732-58ac5e232fba + github.com/stretchr/testify v1.8.2 + github.com/swaggo/http-swagger v1.2.6 github.com/swaggo/swag v1.8.3 github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965 github.com/unrolled/render v1.0.1 github.com/urfave/negroni v0.3.0 go.etcd.io/etcd v0.5.0-alpha.5.0.20220915004622-85b640cee793 go.uber.org/goleak v1.1.12 - go.uber.org/zap v1.19.1 + go.uber.org/zap v1.24.0 golang.org/x/exp v0.0.0-20230108222341-4b8118a2686a - golang.org/x/text v0.4.0 + golang.org/x/text v0.9.0 golang.org/x/time v0.1.0 - golang.org/x/tools v0.2.0 - google.golang.org/grpc v1.51.0 + golang.org/x/tools v0.6.0 + google.golang.org/grpc v1.54.0 gotest.tools/gotestsum v1.7.0 ) @@ -60,14 +64,23 @@ require ( github.com/PuerkitoBio/purell v1.1.1 // indirect github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect github.com/ReneKroon/ttlcache/v2 v2.3.0 // indirect - github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect github.com/VividCortex/mysqlerr v1.0.0 // indirect github.com/Xeoncross/go-aesctr-with-hmac v0.0.0-20200623134604-12b17a7ff502 // indirect + github.com/aws/aws-sdk-go-v2 v1.17.7 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.1 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.31 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.25 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.3.32 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.25 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.12.6 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.6 // indirect + github.com/aws/smithy-go v1.13.5 // indirect + github.com/benbjohnson/clock v1.3.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bitly/go-simplejson v0.5.0 // indirect github.com/breeswish/gin-jwt/v2 v2.6.4-jwt-patch // indirect github.com/cenkalti/backoff/v4 v4.0.2 // indirect - github.com/cespare/xxhash/v2 v2.1.2 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f // indirect github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f // indirect github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect @@ -95,11 +108,11 @@ require ( github.com/goccy/go-json v0.9.7 // indirect github.com/golang-jwt/jwt v3.2.1+incompatible // indirect github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 // indirect - github.com/golang/protobuf v1.5.2 // indirect + github.com/golang/protobuf v1.5.3 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/pprof v0.0.0-20211122183932-1daafda22083 // indirect github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect - github.com/google/uuid v1.3.0 // indirect + github.com/google/uuid v1.3.0 github.com/gorilla/websocket v1.4.2 // indirect github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4 // indirect github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect @@ -108,15 +121,14 @@ require ( github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect - github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/jonboulle/clockwork v0.2.2 // indirect github.com/joomcode/errorx v1.0.1 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/konsorten/go-windows-terminal-sequences v1.0.3 // indirect github.com/leodido/go-urn v1.2.1 // indirect - github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect - github.com/mailru/easyjson v0.7.6 // indirect + github.com/lufia/plan9stats v0.0.0-20230326075908-cb1d2100619a // indirect + github.com/mailru/easyjson v0.7.6 github.com/mattn/go-colorable v0.1.8 // indirect github.com/mattn/go-isatty v0.0.14 // indirect github.com/mattn/go-runewidth v0.0.8 // indirect @@ -135,20 +147,20 @@ require ( github.com/pingcap/tipb v0.0.0-20220718022156-3e2483c20a9e // indirect github.com/pkg/errors v0.9.1 github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect + github.com/power-devops/perfstat v0.0.0-20221212215047-62379fc7944b // indirect github.com/prometheus/client_model v0.2.0 // indirect github.com/prometheus/procfs v0.6.0 // indirect github.com/rs/cors v1.7.0 // indirect github.com/russross/blackfriday/v2 v2.0.1 // indirect github.com/samber/lo v1.37.0 // indirect - github.com/sergi/go-diff v1.0.1-0.20180205163309-da645544ed44 // indirect - github.com/shirou/gopsutil v3.21.3+incompatible // indirect + github.com/sergi/go-diff v1.1.0 // indirect + github.com/shoenig/go-m1cpu v0.1.5 // indirect github.com/shurcooL/httpgzip v0.0.0-20190720172056-320755c1c1b0 // indirect github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect github.com/sirupsen/logrus v1.6.0 // indirect github.com/soheilhy/cmux v0.1.4 github.com/stretchr/objx v0.5.0 // indirect - github.com/swaggo/files v0.0.0-20190704085106-630677cd5c14 // indirect + github.com/swaggo/files v0.0.0-20210815190702-a29dd2bc99b2 // indirect github.com/tidwall/gjson v1.9.3 // indirect github.com/tklauser/go-sysconf v0.3.11 // indirect github.com/tklauser/numcpus v0.6.0 // indirect @@ -164,19 +176,19 @@ require ( go.uber.org/atomic v1.10.0 go.uber.org/dig v1.9.0 // indirect go.uber.org/fx v1.12.0 // indirect - go.uber.org/multierr v1.7.0 // indirect + go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.1.0 // indirect - golang.org/x/image v0.0.0-20200119044424-58c23975cae1 // indirect - golang.org/x/mod v0.6.0 // indirect - golang.org/x/net v0.2.0 // indirect - golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783 // indirect + golang.org/x/image v0.5.0 // indirect + golang.org/x/mod v0.8.0 // indirect + golang.org/x/net v0.10.0 // indirect + golang.org/x/oauth2 v0.4.0 // indirect golang.org/x/sync v0.1.0 // indirect - golang.org/x/sys v0.3.0 // indirect - golang.org/x/term v0.2.0 // indirect + golang.org/x/sys v0.8.0 // indirect + golang.org/x/term v0.8.0 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20221202195650-67e5cbc046fd // indirect - google.golang.org/protobuf v1.28.1 // indirect - gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect + google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect + google.golang.org/protobuf v1.30.0 // indirect + gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect gorm.io/datatypes v1.1.0 // indirect @@ -187,7 +199,7 @@ require ( sigs.k8s.io/yaml v1.2.0 // indirect ) -replace google.golang.org/grpc v1.51.0 => google.golang.org/grpc v1.26.0 +replace google.golang.org/grpc v1.54.0 => google.golang.org/grpc v1.26.0 // When you modify PD cooperatively with kvproto, this will be useful to submit the PR to PD and the PR to // kvproto at the same time. You can run `go mod tidy` to make it replaced with go-mod style specification. diff --git a/go.sum b/go.sum index ba5a15f3909..74ef74bb07e 100644 --- a/go.sum +++ b/go.sum @@ -9,19 +9,17 @@ github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6Xge github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww= github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/ReneKroon/ttlcache/v2 v2.3.0 h1:qZnUjRKIrbKHH6vF5T7Y9Izn5ObfTZfyYpGhvz2BKPo= github.com/ReneKroon/ttlcache/v2 v2.3.0/go.mod h1:zbo6Pv/28e21Z8CzzqgYRArQYGYtjONRxaAKGxzQvG4= -github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUWq3EgK3CesDbo8upS2Vm9/P3FtgI+Jk= -github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/VividCortex/mysqlerr v1.0.0 h1:5pZ2TZA+YnzPgzBfiUWGqWmKDVNBdrkf9g+DNe1Tiq8= github.com/VividCortex/mysqlerr v1.0.0/go.mod h1:xERx8E4tBhLvpjzdUyQiSfUxeMcATEQrflDAfXsqcAE= github.com/Xeoncross/go-aesctr-with-hmac v0.0.0-20200623134604-12b17a7ff502 h1:L8IbaI/W6h5Cwgh0n4zGeZpVK78r/jBf9ASurHo9+/o= github.com/Xeoncross/go-aesctr-with-hmac v0.0.0-20200623134604-12b17a7ff502/go.mod h1:pmnBM9bxWSiHvC/gSWunUIyDvGn33EkP2CUjxFKtTTM= +github.com/agiledragon/gomonkey/v2 v2.3.1/go.mod h1:ap1AmDzcVOAz1YpeJ3TCzIgstoaWLA6jbbgxfB4w2iY= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -32,12 +30,37 @@ github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kd github.com/appleboy/gofight/v2 v2.1.2 h1:VOy3jow4vIK8BRQJoC/I9muxyYlJ2yb9ht2hZoS3rf4= github.com/appleboy/gofight/v2 v2.1.2/go.mod h1:frW+U1QZEdDgixycTj4CygQ48yLTUhplt43+Wczp3rw= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= -github.com/aws/aws-sdk-go v1.35.3 h1:r0puXncSaAfRt7Btml2swUo74Kao+vKhO3VLjwDjK54= -github.com/aws/aws-sdk-go v1.35.3/go.mod h1:H7NKnBqNVzoTJpGfLrQkkD+ytBA93eiDYi/+8rV9s48= +github.com/aws/aws-sdk-go-v2 v1.17.7 h1:CLSjnhJSTSogvqUGhIC6LqFKATMRexcxLZ0i/Nzk9Eg= +github.com/aws/aws-sdk-go-v2 v1.17.7/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= +github.com/aws/aws-sdk-go-v2/config v1.18.19 h1:AqFK6zFNtq4i1EYu+eC7lcKHYnZagMn6SW171la0bGw= +github.com/aws/aws-sdk-go-v2/config v1.18.19/go.mod h1:XvTmGMY8d52ougvakOv1RpiTLPz9dlG/OQHsKU/cMmY= +github.com/aws/aws-sdk-go-v2/credentials v1.13.18 h1:EQMdtHwz0ILTW1hoP+EwuWhwCG1hD6l3+RWFQABET4c= +github.com/aws/aws-sdk-go-v2/credentials v1.13.18/go.mod h1:vnwlwjIe+3XJPBYKu1et30ZPABG3VaXJYr8ryohpIyM= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.1 h1:gt57MN3liKiyGopcqgNzJb2+d9MJaKT/q1OksHNXVE4= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.1/go.mod h1:lfUx8puBRdM5lVVMQlwt2v+ofiG/X6Ms+dy0UkG/kXw= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.31 h1:sJLYcS+eZn5EeNINGHSCRAwUJMFVqklwkH36Vbyai7M= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.31/go.mod h1:QT0BqUvX1Bh2ABdTGnjqEjvjzrCfIniM9Sc8zn9Yndo= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.25 h1:1mnRASEKnkqsntcxHaysxwgVoUUp5dkiB+l3llKnqyg= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.25/go.mod h1:zBHOPwhBc3FlQjQJE/D3IfPWiWaQmT06Vq9aNukDo0k= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.32 h1:p5luUImdIqywn6JpQsW3tq5GNOxKmOnEpybzPx+d1lk= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.32/go.mod h1:XGhIBZDEgfqmFIugclZ6FU7v75nHhBDtzuB4xB/tEi4= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.25 h1:5LHn8JQ0qvjD9L9JhMtylnkcw7j05GDZqM9Oin6hpr0= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.25/go.mod h1:/95IA+0lMnzW6XzqYJRpjjsAbKEORVeO0anQqjd2CNU= +github.com/aws/aws-sdk-go-v2/service/kms v1.20.8 h1:R5f4VOFi3ScTe7TtePyxLqEhNqTJIAxL57MzrXFNs6I= +github.com/aws/aws-sdk-go-v2/service/kms v1.20.8/go.mod h1:OtP3pBOgmJM+acQyQcQXtQHets3yJoVuanCx2T5M7v4= +github.com/aws/aws-sdk-go-v2/service/sso v1.12.6 h1:5V7DWLBd7wTELVz5bPpwzYy/sikk0gsgZfj40X+l5OI= +github.com/aws/aws-sdk-go-v2/service/sso v1.12.6/go.mod h1:Y1VOmit/Fn6Tz1uFAeCO6Q7M2fmfXSCLeL5INVYsLuY= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.6 h1:B8cauxOH1W1v7rd8RdI/MWnoR4Ze0wIHWrb90qczxj4= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.6/go.mod h1:Lh/bc9XUf8CfOY6Jp5aIkQtN+j1mc+nExc+KXj9jx2s= +github.com/aws/aws-sdk-go-v2/service/sts v1.18.7 h1:bWNgNdRko2x6gqa0blfATqAZKZokPIeM1vfmQt2pnvM= +github.com/aws/aws-sdk-go-v2/service/sts v1.18.7/go.mod h1:JuTnSoeePXmMVe9G8NcjjwgOKEfZ4cOjMuT2IBT/2eI= +github.com/aws/smithy-go v1.13.5 h1:hgz0X/DX0dGqTYpGALqXJoRKRj5oQ7150i5FdTePzO8= +github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/axw/gocov v1.0.0 h1:YsqYR66hUmilVr23tu8USgnJIJvnwh3n7j5zRn7x4LU= github.com/axw/gocov v1.0.0/go.mod h1:LvQpEYiwwIb2nYkXY2fDWhg9/AsYqkhmrCshjlUJECE= -github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= +github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= +github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -55,8 +78,8 @@ github.com/cenkalti/backoff/v4 v4.0.2/go.mod h1:eEew/i+1Q6OrCDZh3WiXYv3+nJwBASZ8 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= -github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10 h1:Swpa1K6QvQznwJRcfTfQJmTE72DqScAa40E+fbHEXEE= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e h1:fY5BOSpyZCqRo5OhCuC+XN+r/bBCmeuuJtjz+bCNIf8= @@ -122,17 +145,15 @@ github.com/gin-contrib/cors v1.4.0 h1:oJ6gwtUl3lqV0WEIwM/LxPF1QZ5qe2lGWdY2+bz7y0 github.com/gin-contrib/cors v1.4.0/go.mod h1:bs9pNM0x/UsmHPBWT2xZz9ROh8xYjYkiURUfmBoMlcs= github.com/gin-contrib/gzip v0.0.1 h1:ezvKOL6jH+jlzdHNE4h9h8q8uMpDQjyl0NN0Jd7jozc= github.com/gin-contrib/gzip v0.0.1/go.mod h1:fGBJBCdt6qCZuCAOwWuFhBB4OOq9EFqlo5dEaFhhu5w= +github.com/gin-contrib/pprof v1.4.0 h1:XxiBSf5jWZ5i16lNOPbMTVdgHBdhfGRD5PZ1LWazzvg= +github.com/gin-contrib/pprof v1.4.0/go.mod h1:RrehPJasUVBPK6yTUwOl8/NP6i0vbUgmxtis+Z5KE90= github.com/gin-contrib/sse v0.0.0-20170109093832-22d885f9ecc7/go.mod h1:VJ0WA2NBN22VlZ2dKZQPAPnyWw5XTlK1KymzLKsr59s= -github.com/gin-contrib/sse v0.0.0-20190301062529-5545eab6dad3/go.mod h1:VJ0WA2NBN22VlZ2dKZQPAPnyWw5XTlK1KymzLKsr59s= github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= github.com/gin-gonic/gin v1.3.0/go.mod h1:7cKuhb5qV2ggCFctp2fJQ+ErvciLZrIeoOSOm6mUr7Y= -github.com/gin-gonic/gin v1.4.0/go.mod h1:OW2EZn3DO8Ln9oIKOvM++LBO+5UPHJJDH72/q/3rZdM= github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M= github.com/gin-gonic/gin v1.8.1 h1:4+fr/el88TOO3ewCmQr8cx/CtZ/umlIRIs5M4NTNjf8= github.com/gin-gonic/gin v1.8.1/go.mod h1:ji8BvRH1azfM+SYow9zQ6SZMvR8qOMZHmsCuWR9tTTk= -github.com/go-chi/chi v4.0.2+incompatible h1:maB6vn6FqCxrpz4FqWdh4+lwpyZIQS7YEAUcHlgXVRs= -github.com/go-chi/chi v4.0.2+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ= github.com/go-echarts/go-echarts v1.0.0 h1:n181E4iXwj4zrU9VYmdM2m8dyhERt2w9k9YhHqdp6A8= github.com/go-echarts/go-echarts v1.0.0/go.mod h1:qbmyAb/Rl1f2w7wKba1D4LoNq4U164yO4/wedFbcWyo= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= @@ -142,21 +163,15 @@ github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= -github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= -github.com/go-openapi/jsonpointer v0.17.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonreference v0.17.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= -github.com/go-openapi/jsonreference v0.19.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= github.com/go-openapi/jsonreference v0.19.6 h1:UBIxjkht+AWIgYzCDSv2GN+E/togfwXUJFRTWhl2Jjs= github.com/go-openapi/jsonreference v0.19.6/go.mod h1:diGHMEHg2IqXZGKxqyvWdfWU/aim5Dprw5bqpKkTvns= -github.com/go-openapi/spec v0.19.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= github.com/go-openapi/spec v0.20.4 h1:O8hJrt0UMnhHcluhIdUgCLRWyM2x7QkBXRvOs7m+O1M= github.com/go-openapi/spec v0.20.4/go.mod h1:faYFR1CvsJZ0mNsmsphTMSoRrNV3TEDoAM7FOEWeq8I= -github.com/go-openapi/swag v0.17.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.19.15 h1:D2NRCBzS9/pEY3gP9Nl8aDqGUcPFrwG2p+CNFrLyrCM= github.com/go-openapi/swag v0.19.15/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= @@ -173,7 +188,6 @@ github.com/go-playground/validator/v10 v10.10.0 h1:I7mrTYv78z8k8VXa/qJlOlEXn/nBh github.com/go-playground/validator/v10 v10.10.0/go.mod h1:74x4gJWsvQexRdW8Pn3dXSGrTK4nAUsbPlLADvpJkos= github.com/go-resty/resty/v2 v2.6.0 h1:joIR5PNLM2EFqqESUjCMGXrWmXNHEU9CEiK813oKYS4= github.com/go-resty/resty/v2 v2.6.0/go.mod h1:PwvJS6hvaPkjtjNg9ph+VrSD92bi5Zq73w/BIH7cC3Q= -github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-sql-driver/mysql v1.7.0 h1:ueSltNNllEqE3qcWBTD0iQd3IpL/6U+mJxLkazJ7YPc= github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= @@ -193,8 +207,8 @@ github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 h1:au07oEsX2xN0kt github.com/golang-sql/sqlexp v0.1.0 h1:ZCD6MBpcuOVfGVqsEmY5/4FtYiKz6tSyUv9LPEDei6A= github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 h1:DACJavvAHhabrF08vX0COfcOBJRhZ8lUbR+ZWIs0Y5g= github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/glog v1.1.0 h1:/d3pCKDPWNnvIWe0vVUpNP32qc8U3PDVxySP/y360qE= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef h1:veQD95Isof8w9/WXiA+pa3tz3fJXkt5B7QaRBrM62gk= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -213,8 +227,8 @@ github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvq github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= -github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= @@ -228,6 +242,7 @@ github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -240,7 +255,7 @@ github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/mux v1.7.4 h1:VuZ8uybHlWmqV03+zRzdwKL4tUnIp1MAQtp1mIFE1bc= github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= @@ -280,9 +295,7 @@ github.com/jinzhu/now v1.1.2/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/ github.com/jinzhu/now v1.1.4/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= -github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= -github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/joho/godotenv v1.4.0 h1:3l4+N6zfMWnkbPEXKng2o2/MR5mSwTrBih4ZEkkz1lg= github.com/joho/godotenv v1.4.0/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= @@ -303,6 +316,7 @@ github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/ github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= @@ -323,10 +337,10 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w= github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= -github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= +github.com/lufia/plan9stats v0.0.0-20230326075908-cb1d2100619a h1:N9zuLhTvBSRt0gWSiJswwQ2HqDmtX/ZCDJURnKUt1Ik= +github.com/lufia/plan9stats v0.0.0-20230326075908-cb1d2100619a/go.mod h1:JKx41uQRwqlTZabZc+kILPrO/3jlKnQ2Z8b7YiVw5cE= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA= @@ -336,7 +350,6 @@ github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVc github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8= github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= @@ -387,6 +400,11 @@ github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.20.1 h1:PA/3qinGoukvymdIDV8pii6tiZgC8kbmJO6Z5+b002Q= github.com/onsi/gomega v1.20.1/go.mod h1:DtrZpjmvpn2mPm4YWQa0/ALMDj9v4YxLgojwPeREyVo= +github.com/otiai10/copy v1.7.0/go.mod h1:rmRl6QPdJj6EiUqXQ/4Nn2lLXoNQjFCQbbNrxgc/t3U= +github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95/go.mod h1:9qAhocn7zKJG+0mI8eUu6xqkFDYS2kb2saOteoSB3cE= +github.com/otiai10/curr v1.0.0/go.mod h1:LskTG5wDwr8Rs+nNQ+1LlxRjAtTZZjtJW4rMXl6j4vs= +github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT91xUo= +github.com/otiai10/mint v1.3.3/go.mod h1:/yxELlJQ0ufhjUwhshSj+wFjZ78CnZ48/1wtmBH1OTc= github.com/pascaldekloe/name v0.0.0-20180628100202-0fd16699aae1/go.mod h1:eD5JxqMiuNYyFNmyY9rkJ/slN8y59oEu4Ei7F8OoKWQ= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml/v2 v2.0.1 h1:8e3L2cCQzLFi2CR4g7vGFuFxX7Jl1kKX8gW+iV0GUKU= @@ -395,8 +413,6 @@ github.com/petermattis/goid v0.0.0-20211229010228-4d14c490ee36 h1:64bxqeTEN0/xoE github.com/petermattis/goid v0.0.0-20211229010228-4d14c490ee36/go.mod h1:pxMtw7cyUw6B2bRH0ZBANSPg+AoSud1I1iyJHI69jH4= github.com/phf/go-queue v0.0.0-20170504031614-9abe38d0371d h1:U+PMnTlV2tu7RuMK5etusZG3Cf+rpow5hqQByeCzJ2g= github.com/phf/go-queue v0.0.0-20170504031614-9abe38d0371d/go.mod h1:lXfE4PvvTW5xOjO6Mba8zDPyw8M93B6AQ7frTGnMlA8= -github.com/pingcap/check v0.0.0-20190102082844-67f458068fc8/go.mod h1:B1+S9LNcuMyLH/4HMTViQOJevkGiik3wW2AN9zb2fNQ= -github.com/pingcap/check v0.0.0-20191107115940-caf2b9e6ccf4/go.mod h1:PYMCGwN0JHjoqGr3HrZoD+b8Tgx8bKnArhSq8YVzUMc= github.com/pingcap/check v0.0.0-20191216031241-8a5a85928f12 h1:rfD9v3+ppLPzoQBgZev0qYCpegrwyFx/BUpkApEiKdY= github.com/pingcap/errcode v0.3.0 h1:IF6LC/4+b1KNwrMlr2rBTUrojFPMexXBcDWZSpNwxjg= github.com/pingcap/errcode v0.3.0/go.mod h1:4b2X8xSqxIroj/IZ9MX/VGZhAwc11wB9wRIzHvz6SeM= @@ -405,19 +421,18 @@ github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTw github.com/pingcap/errors v0.11.5-0.20190809092503-95897b64e011/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c h1:xpW9bvK+HuuTmyFqUwr+jcCvpVkK7sumiz+ko5H9eq4= github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c/go.mod h1:X2r9ueLEUZgtx2cIogM0v4Zj5uvvzhuuiu7Pn8HzMPg= -github.com/pingcap/failpoint v0.0.0-20200702092429-9f69995143ce h1:Y1kCxlCtlPTMtVcOkjUcuQKh+YrluSo7+7YMCQSzy30= -github.com/pingcap/failpoint v0.0.0-20200702092429-9f69995143ce/go.mod h1:w4PEZ5y16LeofeeGwdgZB4ddv9bLyDuIX+ljstgKZyk= +github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00 h1:C3N3itkduZXDZFh4N3vQ5HEtld3S+Y+StULhWVvumU0= +github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00/go.mod h1:4qGtCB0QK0wBzKtFEGDhxXnSnbQApw1gc9siScUl8ew= github.com/pingcap/kvproto v0.0.0-20191211054548-3c6b38ea5107/go.mod h1:WWLmULLO7l8IOcQG+t+ItJ3fEcrL5FxF0Wu+HrMy26w= -github.com/pingcap/kvproto v0.0.0-20230228041042-1e9aca94bab6 h1:bgLRG7gPJCq6aduA65ZV7xWQBThTcuarBB9VdfAzV4g= -github.com/pingcap/kvproto v0.0.0-20230228041042-1e9aca94bab6/go.mod h1:KUrW1FGoznGMMTssYBu0czfAhn6vQcIrHyZoSC6T990= -github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8= +github.com/pingcap/kvproto v0.0.0-20230727073445-53e1f8730c30 h1:EvqKcDT7ceGLW0mXqM8Cp5Z8DfgQRnwj2YTnlCLj2QI= +github.com/pingcap/kvproto v0.0.0-20230727073445-53e1f8730c30/go.mod h1:r0q/CFcwvyeRhKtoqzmWMBebrtpIziQQ9vR+JKh1knc= github.com/pingcap/log v0.0.0-20210625125904-98ed8e2eb1c7/go.mod h1:8AanEdAHATuRurdGxZXBz0At+9avep+ub7U1AGYLIMM= github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 h1:HR/ylkkLmGdSSDaD8IDP+SZrdhV1Kibl9KrHxJ9eciw= github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4= -github.com/pingcap/sysutil v0.0.0-20211208032423-041a72e5860d h1:k3/APKZjXOyJrFy8VyYwRlZhMelpD3qBLJNsw3bPl/g= -github.com/pingcap/sysutil v0.0.0-20211208032423-041a72e5860d/go.mod h1:7j18ezaWTao2LHOyMlsc2Dg1vW+mDY9dEbPzVyOlaeM= -github.com/pingcap/tidb-dashboard v0.0.0-20230209052558-a58fc2a7e924 h1:49x3JR5zEYqjVqONKV9r/nrv0Rh5QU8ivIhktoLvP4g= -github.com/pingcap/tidb-dashboard v0.0.0-20230209052558-a58fc2a7e924/go.mod h1:OUzFMMVjR1GKlf4LWLqza9QNKjCrYJ7stVn/3PN0djM= +github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21 h1:QV6jqlfOkh8hqvEAgwBZa+4bSgO0EeKC7s5c6Luam2I= +github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21/go.mod h1:QYnjfA95ZaMefyl1NO8oPtKeb8pYUdnDVhQgf+qdpjM= +github.com/pingcap/tidb-dashboard v0.0.0-20230705095454-5e220f970f27 h1:K9lZMYuDuAiR5kOjFESwJ8KfSb4ui5zX6vZGbUp58uk= +github.com/pingcap/tidb-dashboard v0.0.0-20230705095454-5e220f970f27/go.mod h1:OUzFMMVjR1GKlf4LWLqza9QNKjCrYJ7stVn/3PN0djM= github.com/pingcap/tipb v0.0.0-20220718022156-3e2483c20a9e h1:FBaTXU8C3xgt/drM58VHxojHo/QoG1oPsgWTGvaSpO4= github.com/pingcap/tipb v0.0.0-20220718022156-3e2483c20a9e/go.mod h1:A7mrd7WHBl1o63LE2bIBGEJMTNWXqhgmYiOvMLxozfs= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= @@ -427,8 +442,9 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw= github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= +github.com/power-devops/perfstat v0.0.0-20221212215047-62379fc7944b h1:0LFwY6Q3gMACTjAbMZBjXAqTOzOwFaj2Ld6cjeQ7Rig= +github.com/power-devops/perfstat v0.0.0-20221212215047-62379fc7944b/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= @@ -467,12 +483,16 @@ github.com/samber/lo v1.37.0 h1:XjVcB8g6tgUp8rsPsJ2CvhClfImrpL04YpQHXeHPhRw= github.com/samber/lo v1.37.0/go.mod h1:9vaz2O4o8oOnK23pd2TrXufcbdbJIa3b6cstBWKpopA= github.com/sasha-s/go-deadlock v0.2.0 h1:lMqc+fUb7RrFS3gQLtoQsJ7/6TV/pAIFvBsqX73DK8Y= github.com/sasha-s/go-deadlock v0.2.0/go.mod h1:StQn567HiB1fF2yJ44N9au7wOhrPS3iZqiDbRupzT10= -github.com/sergi/go-diff v1.0.1-0.20180205163309-da645544ed44 h1:tB9NOR21++IjLyVx3/PCPhWMwqGNCMQEH96A6dMZ/gc= -github.com/sergi/go-diff v1.0.1-0.20180205163309-da645544ed44/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= -github.com/shirou/gopsutil v3.21.3+incompatible h1:uenXGGa8ESCQq+dbgtl916dmg6PSAz2cXov0uORQ9v8= -github.com/shirou/gopsutil v3.21.3+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= -github.com/shirou/gopsutil/v3 v3.22.12 h1:oG0ns6poeUSxf78JtOsfygNWuEHYYz8hnnNg7P04TJs= -github.com/shirou/gopsutil/v3 v3.22.12/go.mod h1:Xd7P1kwZcp5VW52+9XsirIKd/BROzbb2wdX3Kqlz9uI= +github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= +github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= +github.com/shirou/gopsutil/v3 v3.21.12/go.mod h1:BToYZVTlSVlfazpDDYFnsVZLaoRG+g8ufT6fPQLdJzA= +github.com/shirou/gopsutil/v3 v3.23.3 h1:Syt5vVZXUDXPEXpIBt5ziWsJ4LdSAAxF4l/xZeQgSEE= +github.com/shirou/gopsutil/v3 v3.23.3/go.mod h1:lSBNN6t3+D6W5e5nXTxc8KIMMVxAcS+6IJlffjRRlMU= +github.com/shoenig/go-m1cpu v0.1.4/go.mod h1:Wwvst4LR89UxjeFtLRMrpgRiyY4xPsejnVZym39dbAQ= +github.com/shoenig/go-m1cpu v0.1.5 h1:LF57Z/Fpb/WdGLjt2HZilNnmZOxg/q2bSKTQhgbrLrQ= +github.com/shoenig/go-m1cpu v0.1.5/go.mod h1:Wwvst4LR89UxjeFtLRMrpgRiyY4xPsejnVZym39dbAQ= +github.com/shoenig/test v0.6.3 h1:GVXWJFk9PiOjN0KoJ7VrJGH6uLPnqxR7/fe3HUPfE0c= +github.com/shoenig/test v0.6.3/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k= github.com/shurcooL/httpgzip v0.0.0-20190720172056-320755c1c1b0 h1:mj/nMDAwTBiaCqMEs4cYCqF7pO6Np7vhy1D1wcQGz+E= github.com/shurcooL/httpgzip v0.0.0-20190720172056-320755c1c1b0/go.mod h1:919LwcH0M7/W4fcZ0/jy0qGght1GIhqyS/EgWGH2j5Q= github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= @@ -483,6 +503,8 @@ github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/smallnest/chanx v0.0.0-20221229104322-eb4c998d2072 h1:Txo4SXVJq/OgEjwgkWoxkMoTjGlcrgsQE/XSghjmu0w= github.com/smallnest/chanx v0.0.0-20221229104322-eb4c998d2072/go.mod h1:+4nWMF0+CqEcU74SnX2NxaGqZ8zX4pcQ8Jcs77DbX5A= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.4 h1:0HKaf1o97UwFjHH9o5XsHUOF+tqmdA7KEzXLpiyaw0E= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= @@ -511,15 +533,13 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= -github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/swaggo/files v0.0.0-20190704085106-630677cd5c14 h1:PyYN9JH5jY9j6av01SpfRMb+1DWg/i3MbGOKPxJ2wjM= -github.com/swaggo/files v0.0.0-20190704085106-630677cd5c14/go.mod h1:gxQT6pBGRuIGunNf/+tSOB5OHvguWi8Tbt82WOkf35E= -github.com/swaggo/gin-swagger v1.2.0/go.mod h1:qlH2+W7zXGZkczuL+r2nEBR2JTT+/lX05Nn6vPhc7OI= -github.com/swaggo/http-swagger v0.0.0-20200308142732-58ac5e232fba h1:lUPlXKqgbqT2SVg2Y+eT9mu5wbqMnG+i/+Q9nK7C0Rs= -github.com/swaggo/http-swagger v0.0.0-20200308142732-58ac5e232fba/go.mod h1:O1lAbCgAAX/KZ80LM/OXwtWFI/5TvZlwxSg8Cq08PV0= -github.com/swaggo/swag v1.5.1/go.mod h1:1Bl9F/ZBpVWh22nY0zmYyASPO1lI/zIwRDrpZU+tv8Y= -github.com/swaggo/swag v1.6.3/go.mod h1:wcc83tB4Mb2aNiL/HP4MFeQdpHUrca+Rp/DRNgWAUio= +github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= +github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/swaggo/files v0.0.0-20210815190702-a29dd2bc99b2 h1:+iNTcqQJy0OZ5jk6a5NLib47eqXK8uYcPX+O4+cBpEM= +github.com/swaggo/files v0.0.0-20210815190702-a29dd2bc99b2/go.mod h1:lKJPbtWzJ9JhsTN1k1gZgleJWY/cqq0psdoMmaThG3w= +github.com/swaggo/http-swagger v1.2.6 h1:ihTjChUoSRMpFMjWw+0AkL1Ti4r6v8pCgVYLmQVRlRw= +github.com/swaggo/http-swagger v1.2.6/go.mod h1:CcoICgY3yVDk2u1LQUCMHbAj0fjlxIX+873psXlIKNA= +github.com/swaggo/swag v1.7.9/go.mod h1:gZ+TJ2w/Ve1RwQsA2IRoSOTidHz6DX+PIG8GWvbnoLU= github.com/swaggo/swag v1.8.3 h1:3pZSSCQ//gAH88lfmxM3Cd1+JCsxV8Md6f36b9hrZ5s= github.com/swaggo/swag v1.8.3/go.mod h1:jMLeXOOmYyjk8PvHTsXBdrubsNd9gUJTTCzL5iBnseg= github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965 h1:1oFLiOyVl+W7bnBzGhf7BbIv9loSFQcieWWYIjLqcAw= @@ -533,21 +553,19 @@ github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JT github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= -github.com/tklauser/go-sysconf v0.3.4/go.mod h1:Cl2c8ZRWfHD5IrfHo9VN+FX9kCFjIOyVklgXycLB6ek= +github.com/tklauser/go-sysconf v0.3.9/go.mod h1:11DU/5sG7UexIrp/O6g35hrWzu0JxlwQ3LSFUzyeuhs= github.com/tklauser/go-sysconf v0.3.11 h1:89WgdJhk5SNwJfu+GKyYveZ4IaJ7xAkecBo+KdJV0CM= github.com/tklauser/go-sysconf v0.3.11/go.mod h1:GqXfhXY3kiPa0nAXPDIQIWzJbMCB7AmcWpGR8lSZfqI= -github.com/tklauser/numcpus v0.2.1/go.mod h1:9aU+wOc6WjUIZEwWMP62PL/41d65P+iks1gBkr4QyP8= +github.com/tklauser/numcpus v0.3.0/go.mod h1:yFGUr7TUHQRAhyqBcEg0Ge34zDBAsIvJJcyE6boqnA8= github.com/tklauser/numcpus v0.6.0 h1:kebhY2Qt+3U6RNK7UqpYNA+tJ23IBEGKkB7JQBfDYms= github.com/tklauser/numcpus v0.6.0/go.mod h1:FEZLMke0lhOUG6w2JadTzp0a+Nl8PF/GFkQ5UVIcaL4= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20200427203606-3cfed13b9966 h1:j6JEOq5QWFker+d7mFQYOhjTZonQ7YkLTHm56dbn+yM= github.com/tmc/grpc-websocket-proxy v0.0.0-20200427203606-3cfed13b9966/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= -github.com/ugorji/go v1.1.5-pre/go.mod h1:FwP/aQVg39TXzItUBMwnWp9T9gPQnXw4Poh4/oBQZ/0= github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= github.com/ugorji/go v1.2.7/go.mod h1:nF9osbDWLy6bDVv/Rtoh6QgnvNDpmCalQV5urGCCS6M= github.com/ugorji/go/codec v0.0.0-20181022190402-e5e69e061d4f/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= -github.com/ugorji/go/codec v1.1.5-pre/go.mod h1:tULtS6Gy1AE1yCENaw4Vb//HLH5njI2tfCQDUqRd8fI= github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= github.com/ugorji/go/codec v1.2.7 h1:YPXUKf7fYbp/y8xloBqZOw2qaVggbfwMlI8WM3wZUJ0= github.com/ugorji/go/codec v1.2.7/go.mod h1:WGN1fab3R1fzQlVQTkfxVtIBhWDRqOviHU95kRgeqEY= @@ -568,6 +586,8 @@ github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1: github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg= github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= @@ -589,7 +609,6 @@ go.uber.org/fx v1.12.0 h1:+1+3Cz9M0dFMPy9SW9XUIUHye8bnPUm7q7DroNGWYG4= go.uber.org/fx v1.12.0/go.mod h1:egT3Kyg1JFYQkvKLZ3EsykxkNrZxgXS+gKoKo7abERY= go.uber.org/goleak v0.10.0/go.mod h1:VCZuO8V8mFPlL0F5J5GK1rtHV3DrFcQ1R8ryq7FK0aI= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= -go.uber.org/goleak v1.1.11-0.20210813005559-691160354723/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= @@ -597,16 +616,17 @@ go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+ go.uber.org/multierr v1.4.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= -go.uber.org/multierr v1.7.0 h1:zaiO/rmgFjbmCXdSYJWQcdvOCsthmdaHfr3Gm2Kx4Ec= go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.12.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= -go.uber.org/zap v1.19.1 h1:ue41HOKd1vGURxrmeKIgELGb3jPW9DMUDGtsinblHwI= -go.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI= +go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= +go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -615,14 +635,16 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200204104054-c9f3fb736b72/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.1.0 h1:MDRAIl0xIo9Io2xV565hzXHw3zVseKrJKodhohM5CjU= golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20230108222341-4b8118a2686a h1:tlXy25amD5A7gOfbXdqCGN5k8ESEed/Ee1E5RcrYnqU= golang.org/x/exp v0.0.0-20230108222341-4b8118a2686a/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= -golang.org/x/image v0.0.0-20200119044424-58c23975cae1 h1:5h3ngYt7+vXCDZCup/HkCQgW5XwmSvR/nA2JmJ0RErg= golang.org/x/image v0.0.0-20200119044424-58c23975cae1/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.5.0 h1:5JMiNunQeQw++mMOz48/ISeNu3Iweh/JaZU8ZLqHRrI= +golang.org/x/image v0.5.0/go.mod h1:FVC7BI/5Ym8R25iw5OLsgshdUBbT1h5jZTpA+mvAdZ4= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= @@ -634,8 +656,9 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.6.0 h1:b9gGHsz9/HhJ3HF5DHQytPpuwocVTChQJK3AvoLRD5I= -golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0 h1:LUYupSeNrTNCGzR/hVBk2NHZO4hXcVaW1k4Qx7rjPx8= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -646,13 +669,10 @@ golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= -golang.org/x/net v0.0.0-20190611141213-3f473d35a33a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= @@ -661,14 +681,16 @@ golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM= +golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.2.0 h1:sZfSu1wtKLGlWI4ZZayP0ck9Y73K1ynO6gqzTdBVdPU= -golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783 h1:nt+Q6cXKz4MosCSpnbMtqiQ8Oz0pxTef2B4Vca2lvfk= -golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= +golang.org/x/oauth2 v0.4.0 h1:NF0gk8LVPg1Ml7SSbGyySuoxdsXitj7TvgvuRxIMc/M= +golang.org/x/oauth2 v0.4.0/go.mod h1:RznEsdpjGAINPTOF0UH/t+xJ75L18YO3Ho6Pyn+uRec= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -677,6 +699,7 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -689,7 +712,6 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190610200419-93c9922d18ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -705,7 +727,6 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210217105451-b926d437f341/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -714,21 +735,29 @@ golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210816074244-15123e1e1f71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211013075003-97ac67df715c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ= -golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.2.0 h1:z85xZCsEl7bi/KwbNADeBYoOP0++7W1ipu+aGnpwzRM= -golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.8.0 h1:n5xxQn2i3PC0yLAbjTpNT85q/Kgzcr2gIoX9OrJUols= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.4.0 h1:BrVqGRd7+k1DiOgtnFvAkoQEWQvBc25ouMJM6429SFg= -golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.1.0 h1:xYY+Bajn2a7VBmTM5GikTmnK8ZuX8YgnQCqZpbBNtmA= @@ -738,17 +767,15 @@ golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524210228-3d17549cdc6b/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190606050223-4d9ae51c2468/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190611222205-d73e1c7e250b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190617190820-da514acc4774/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191030062658-86caa796c7ab/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191107010934-f79515f33823/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191114200427-caa0b0f7d508/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -760,8 +787,10 @@ golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20210112230658-8b4aab62c064/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.2.0 h1:G6AHpWxTMGY1KyEYoAQ5WTtIekUUvDNjan3ugu60JvE= -golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA= +golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -774,8 +803,8 @@ google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoA google.golang.org/genproto v0.0.0-20181004005441-af9cb2a35e7f/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20221202195650-67e5cbc046fd h1:OjndDrsik+Gt+e6fs45z9AxiewiKyLKYpA45W5Kpkks= -google.golang.org/genproto v0.0.0-20221202195650-67e5cbc046fd/go.mod h1:cTsE614GARnxrLsqKREzmNYJACSWWpAWdNMwnD7c2BE= +google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 h1:KpwkzHKEF7B9Zxg18WzOa7djJ+Ha5DzthMyZYQfEn2A= +google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU= google.golang.org/grpc v0.0.0-20180607172857-7a6a684ca69e/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -795,8 +824,8 @@ google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= -google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= +google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -810,8 +839,9 @@ gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= gopkg.in/go-playground/validator.v8 v8.18.2/go.mod h1:RX2a/7Ha8BgOhfk7j780h4/u/RRjR0eouCJSH80/M2Y= -gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= +gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= +gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= diff --git a/metrics/grafana/pd.json b/metrics/grafana/pd.json index 10316026349..6ca2829167c 100644 --- a/metrics/grafana/pd.json +++ b/metrics/grafana/pd.json @@ -796,13 +796,6 @@ "intervalFactor": 2, "legendFormat": "{{type}}", "refId": "B" - }, - { - "expr": "pd_regions_offline_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", type=\"offline-peer-region-count\", instance=\"$instance\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{type}}", - "refId": "C" } ], "thresholds": [ @@ -2340,6 +2333,113 @@ "alignLevel": null } }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "description": "The allowance status of the scheduling.", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 41 + }, + "hiddenSeries": false, + "id": 1464, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "paceLength": 10, + "percentage": false, + "pluginVersion": "7.5.10", + "pointradius": 1, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "pd_scheduling_allowance_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=\"$instance\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{kind}}", + "metric": "pd_scheduling_allowance_status", + "refId": "A", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Scheduling Allowance Status", + "tooltip": { + "shared": true, + "sort": 1, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:533", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:534", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, { "cacheTimeout": null, "colorBackground": false, @@ -2967,7 +3067,7 @@ "format": "time_series", "intervalFactor": 2, "legendFormat": "{{event}}", - "metric": "pd_scheduler_status", + "metric": "pd_schedule_operators_count", "refId": "A", "step": 4 } @@ -5029,6 +5129,111 @@ "alignLevel": null } }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "decimals": 2, + "description": "The total keys of hot write on leader Regions for each TiKV instance", + "fill": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 31 + }, + "id": 1465, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "pd_hotspot_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", store=~\"$store\", instance=\"$instance\", type=\"total_write_bytes_as_leader\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{address}}-store-{{store}}", + "metric": "pd_hotspot_status", + "refId": "A", + "step": 4 + }, + { + "exemplar": true, + "expr": "pd_scheduler_hot_peers_summary{type=\"exp-byte-rate-write-leader\"}", + "hide": true, + "interval": "", + "legendFormat": "exp-byte-rate-write-leader-{{store}}", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Total written bytes on hot leader Regions", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, { "aliasColors": {}, "bars": false, @@ -5153,7 +5358,7 @@ "h": 7, "w": 12, "x": 12, - "y": 31 + "y": 38 }, "id": 1445, "legend": { @@ -5347,12 +5552,13 @@ "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 0, + "description": "The select events of hot read scheduler", "fill": 0, "gridPos": { "h": 7, "w": 12, "x": 12, - "y": 38 + "y": 45 }, "id": 106, "legend": { @@ -5385,7 +5591,7 @@ "steppedLine": false, "targets": [ { - "expr": "rate(pd_scheduler_hot_region{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", store=~\"$store\", type=~\".*store.*\"}[1m])", + "expr": "rate(pd_scheduler_hot_region{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", store=~\"$store\", type=~\".*write.*\"}[1m])", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{type}}-{{store}}", @@ -5397,7 +5603,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Selector events", + "title": "Selector write events", "tooltip": { "shared": true, "sort": 0, @@ -5549,7 +5755,7 @@ "h": 7, "w": 12, "x": 12, - "y": 45 + "y": 52 }, "id": 148, "legend": { @@ -6630,6 +6836,102 @@ "align": false, "alignLevel": null } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "decimals": 0, + "description": "The select events of hot read scheduler", + "fill": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 81 + }, + "id": 1466, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(pd_scheduler_hot_region{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", store=~\"$store\", type=~\".*read.*\"}[1m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{type}}-{{store}}", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Selector read events", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } } ], "repeat": null, @@ -8718,12 +9020,20 @@ "steppedLine": false, "targets": [ { - "expr": "etcd_debugging_mvcc_db_total_size_in_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"pd\"}", + "expr": "etcd_mvcc_db_total_size_in_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"pd\"}", "format": "time_series", "hide": false, "intervalFactor": 1, - "legendFormat": "{{instance}}", + "legendFormat": "{{instance}}-physically-allocated", "refId": "A" + }, + { + "expr": "etcd_mvcc_db_total_size_in_use_in_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"pd\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{instance}}-logically-in-use", + "refId": "B" } ], "thresholds": [], @@ -10051,6 +10361,16 @@ "legendFormat": "tso request/secs", "refId": "A", "step": 2 + }, + { + "expr": "sum(rate(pd_tso_events{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (type,dc)", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{type}}-{{dc}}", + "hide": true, + "refId": "B", + "step": 2 } ], "thresholds": [], @@ -11839,7 +12159,7 @@ "x": 12, "y": 48 }, - "id": 1455, + "id": 1467, "interval": "", "options": { "displayMode": "lcd", diff --git a/pd.code-workspace b/pd.code-workspace index 7bda52958ba..c4603084a93 100644 --- a/pd.code-workspace +++ b/pd.code-workspace @@ -10,11 +10,15 @@ }, { "name": "pd-client-tests", - "path": "tests/client" + "path": "tests/integrations/client" }, { "name": "mcs-tests", - "path": "tests/mcs" + "path": "tests/integrations/mcs" + }, + { + "name": "tso-tests", + "path": "tests/integrations/tso" }, { "name": "pd-tso-bench", diff --git a/pkg/balancer/balancer.go b/pkg/balancer/balancer.go new file mode 100644 index 00000000000..e3105f4c0b8 --- /dev/null +++ b/pkg/balancer/balancer.go @@ -0,0 +1,61 @@ +// Copyright 2023 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package balancer + +// Policy is the policy of balancer. +type Policy int + +const ( + // PolicyRoundRobin is the round robin policy. + PolicyRoundRobin Policy = iota + // PolicyLeast is the policy to return the least used node. + // TODO: move indexed heap to pkg and use it. + PolicyLeast +) + +func (p Policy) String() string { + switch p { + case PolicyRoundRobin: + return "round-robin" + case PolicyLeast: + return "least" + default: + return "unknown" + } +} + +// Balancer is the interface for balancer. +type Balancer[T uint32 | string] interface { + // Next returns next one. + Next() T + // Put puts one into balancer. + Put(T) + // Delete deletes one from balancer. + Delete(T) + // GetAll returns all nodes. + GetAll() []T + // Len returns the length of nodes. + Len() int +} + +// GenByPolicy generates a balancer by policy. +func GenByPolicy[T uint32 | string](policy Policy) Balancer[T] { + switch policy { + case PolicyRoundRobin: + return NewRoundRobin[T]() + default: // only round-robin is supported now. + return NewRoundRobin[T]() + } +} diff --git a/pkg/balancer/balancer_test.go b/pkg/balancer/balancer_test.go new file mode 100644 index 00000000000..f95487a4cc7 --- /dev/null +++ b/pkg/balancer/balancer_test.go @@ -0,0 +1,102 @@ +// Copyright 2023 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package balancer + +import ( + "math/rand" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestBalancerPutAndDelete(t *testing.T) { + t.Parallel() + re := require.New(t) + balancers := []Balancer[uint32]{ + NewRoundRobin[uint32](), + } + for _, balancer := range balancers { + re.Equal(uint32(0), balancer.Next()) + // test put + exists := make(map[uint32]struct{}) + for i := 0; i < 100; i++ { + num := rand.Uint32() + balancer.Put(num) + exists[num] = struct{}{} + re.Equal(len(balancer.GetAll()), len(exists)) + t := balancer.Next() + re.Contains(exists, t) + } + // test delete + for num := range exists { + balancer.Delete(num) + delete(exists, num) + re.Equal(len(balancer.GetAll()), len(exists)) + if len(exists) == 0 { + break + } + t := balancer.Next() + re.NotEqual(t, num) + re.Contains(exists, t) + } + re.Equal(uint32(0), balancer.Next()) + } +} + +func TestBalancerDuplicate(t *testing.T) { + t.Parallel() + re := require.New(t) + balancers := []Balancer[uint32]{ + NewRoundRobin[uint32](), + } + for _, balancer := range balancers { + re.Len(balancer.GetAll(), 0) + // test duplicate put + balancer.Put(1) + re.Len(balancer.GetAll(), 1) + balancer.Put(1) + re.Len(balancer.GetAll(), 1) + // test duplicate delete + balancer.Delete(1) + re.Len(balancer.GetAll(), 0) + balancer.Delete(1) + re.Len(balancer.GetAll(), 0) + } +} + +func TestRoundRobin(t *testing.T) { + t.Parallel() + re := require.New(t) + balancer := NewRoundRobin[uint32]() + for i := 0; i < 100; i++ { + num := rand.Uint32() + balancer.Put(num) + } + statistics := make(map[uint32]int) + for i := 0; i < 1000; i++ { + statistics[balancer.Next()]++ + } + min := 1000 + max := 0 + for _, v := range statistics { + if v < min { + min = v + } + if v > max { + max = v + } + } + re.LessOrEqual(max-min, 10) +} diff --git a/pkg/balancer/round_robin.go b/pkg/balancer/round_robin.go new file mode 100644 index 00000000000..5013a447d3e --- /dev/null +++ b/pkg/balancer/round_robin.go @@ -0,0 +1,88 @@ +// Copyright 2023 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package balancer + +import ( + "sync" + "sync/atomic" +) + +// RoundRobin is a balancer that selects nodes in a round-robin fashion. +type RoundRobin[T uint32 | string] struct { + sync.RWMutex + nodes []T + exists map[T]struct{} + next uint32 +} + +// NewRoundRobin creates a balancer that selects nodes in a round-robin fashion. +func NewRoundRobin[T uint32 | string]() *RoundRobin[T] { + return &RoundRobin[T]{ + nodes: make([]T, 0), + exists: make(map[T]struct{}), + } +} + +// Next returns next address +func (r *RoundRobin[T]) Next() (t T) { + r.RLock() + defer r.RUnlock() + if len(r.nodes) == 0 { + return + } + next := atomic.AddUint32(&r.next, 1) + node := r.nodes[(int(next)-1)%len(r.nodes)] + return node +} + +// GetAll returns all nodes +func (r *RoundRobin[T]) GetAll() []T { + r.RLock() + defer r.RUnlock() + // return a copy to avoid data race + return append(r.nodes[:0:0], r.nodes...) +} + +// Put puts one into balancer. +func (r *RoundRobin[T]) Put(node T) { + r.Lock() + defer r.Unlock() + if _, ok := r.exists[node]; !ok { + r.nodes = append(r.nodes, node) + r.exists[node] = struct{}{} + } +} + +// Delete deletes one from balancer. +func (r *RoundRobin[T]) Delete(node T) { + r.Lock() + defer r.Unlock() + if _, ok := r.exists[node]; ok { + for i, n := range r.nodes { + if n == node { + r.nodes = append(r.nodes[:i], r.nodes[i+1:]...) + delete(r.exists, node) + break + } + } + } +} + +// Len returns the length of nodes. +func (r *RoundRobin[T]) Len() int { + r.RLock() + defer r.RUnlock() + return len(r.nodes) +} diff --git a/pkg/basicserver/basic_server.go b/pkg/basicserver/basic_server.go index def302051b9..3a21da0f9d9 100644 --- a/pkg/basicserver/basic_server.go +++ b/pkg/basicserver/basic_server.go @@ -21,24 +21,20 @@ import ( "go.etcd.io/etcd/clientv3" ) -// MemberProvider defines the common basic behaviors of a member -type MemberProvider interface { - GetName() string - GetClientUrls() []string -} - // Server defines the common basic behaviors of a server type Server interface { // Name returns the unique Name for this server in the cluster. Name() string + // GetAddr returns the address of the server. + GetAddr() string // Context returns the context of server. Context() context.Context // Run runs the server. Run() error // Close closes the server. Close() - // GetPrimary returns the primary of the server. - GetPrimary() MemberProvider + // GetLeaderListenUrls gets service endpoints from the leader in election group. + GetLeaderListenUrls() []string // GetClient returns builtin etcd client. GetClient() *clientv3.Client // GetHTTPClient returns builtin http client. diff --git a/pkg/cache/ttl.go b/pkg/cache/ttl.go index e1a91a15f72..14adf072dad 100644 --- a/pkg/cache/ttl.go +++ b/pkg/cache/ttl.go @@ -19,6 +19,7 @@ import ( "time" "github.com/pingcap/log" + "github.com/tikv/pd/pkg/utils/logutil" "github.com/tikv/pd/pkg/utils/syncutil" "go.uber.org/zap" ) @@ -142,6 +143,7 @@ func (c *ttlCache) Clear() { } func (c *ttlCache) doGC() { + defer logutil.LogPanic() ticker := time.NewTicker(c.gcInterval) defer ticker.Stop() diff --git a/pkg/cgroup/cgroup.go b/pkg/cgroup/cgroup.go index 316383c6ad5..2a99d2fcd3d 100644 --- a/pkg/cgroup/cgroup.go +++ b/pkg/cgroup/cgroup.go @@ -174,7 +174,7 @@ func detectControlPath(cgroupFilePath string, controller string) (string, error) var unifiedPathIfFound string for scanner.Scan() { fields := bytes.Split(scanner.Bytes(), []byte{':'}) - if len(fields) != 3 { + if len(fields) < 3 { // The lines should always have three fields, there's something fishy here. continue } @@ -186,7 +186,14 @@ func detectControlPath(cgroupFilePath string, controller string) (string, error) if f0 == "0" && f1 == "" { unifiedPathIfFound = string(fields[2]) } else if f1 == controller { - return string(fields[2]), nil + var result []byte + // In some case, the cgroup path contains `:`. We need to join them back. + if len(fields) > 3 { + result = bytes.Join(fields[2:], []byte(":")) + } else { + result = fields[2] + } + return string(result), nil } } @@ -194,11 +201,11 @@ func detectControlPath(cgroupFilePath string, controller string) (string, error) } // See http://man7.org/linux/man-pages/man5/proc.5.html for `mountinfo` format. -func getCgroupDetails(mountInfoPath string, cRoot string, controller string) (string, int, error) { +func getCgroupDetails(mountInfoPath string, cRoot string, controller string) (mount []string, version []int, err error) { //nolint:gosec info, err := os.Open(mountInfoPath) if err != nil { - return "", 0, errors.Wrapf(err, "failed to read mounts info from file: %s", mountInfoPath) + return nil, nil, errors.Wrapf(err, "failed to read mounts info from file: %s", mountInfoPath) } defer func() { err := info.Close() @@ -206,6 +213,8 @@ func getCgroupDetails(mountInfoPath string, cRoot string, controller string) (st log.Error("close mountInfoPath", zap.Error(err)) } }() + var foundVer1, foundVer2 = false, false + var mountPointVer1, mountPointVer2 string scanner := bufio.NewScanner(info) for scanner.Scan() { @@ -218,15 +227,17 @@ func getCgroupDetails(mountInfoPath string, cRoot string, controller string) (st if ok { mountPoint := string(fields[4]) if ver == 2 { - return mountPoint, ver, nil + foundVer2 = true + mountPointVer2 = mountPoint + continue } // It is possible that the controller mount and the cgroup path are not the same (both are relative to the NS root). // So start with the mount and construct the relative path of the cgroup. // To test: - // 1、start a docker to run unit test or tidb-server + // 1. start a docker to run unit test or tidb-server // > docker run -it --cpus=8 --memory=8g --name test --rm ubuntu:18.04 bash // - // 2、change the limit when the container is running + // 2. change the limit when the container is running // docker update --cpus=8 nsRelativePath := string(fields[3]) if !strings.Contains(nsRelativePath, "..") { @@ -234,13 +245,23 @@ func getCgroupDetails(mountInfoPath string, cRoot string, controller string) (st // the best action is to ignore the line and hope that the rest of the lines // will allow us to extract a valid path. if relPath, err := filepath.Rel(nsRelativePath, cRoot); err == nil { - return filepath.Join(mountPoint, relPath), ver, nil + mountPointVer1 = filepath.Join(mountPoint, relPath) + foundVer1 = true } } } } + if foundVer1 && foundVer2 { + return []string{mountPointVer1, mountPointVer2}, []int{1, 2}, nil + } + if foundVer1 { + return []string{mountPointVer1}, []int{1}, nil + } + if foundVer2 { + return []string{mountPointVer2}, []int{2}, nil + } - return "", 0, fmt.Errorf("failed to detect cgroup root mount and version") + return nil, nil, fmt.Errorf("failed to detect cgroup root mount and version") } func cgroupFileToUint64(filepath, desc string) (res uint64, err error) { diff --git a/pkg/cgroup/cgroup_cpu.go b/pkg/cgroup/cgroup_cpu.go index df1c104f77f..7063aa89bf9 100644 --- a/pkg/cgroup/cgroup_cpu.go +++ b/pkg/cgroup/cgroup_cpu.go @@ -41,30 +41,48 @@ func getCgroupCPUHelper(root string) (CPUUsage, error) { } var res CPUUsage - - switch ver { - case 1: - cgroupRoot := filepath.Join(root, mount) - res.Period, res.Quota, err = detectCPUQuotaInV1(cgroupRoot) + if len(mount) == 2 { + cgroupRootV1 := filepath.Join(root, mount[0]) + cgroupRootV2 := filepath.Join(root, mount[1], path) + res.Period, res.Quota, err = detectCPUQuotaInV2(cgroupRootV2) if err != nil { - return res, err + res.Period, res.Quota, err = detectCPUQuotaInV1(cgroupRootV1) } - res.Stime, res.Utime, err = detectCPUUsageInV1(cgroupRoot) if err != nil { return res, err } - case 2: - cgroupRoot := filepath.Join(root, mount, path) - res.Period, res.Quota, err = detectCPUQuotaInV2(cgroupRoot) + res.Stime, res.Utime, err = detectCPUUsageInV2(cgroupRootV2) if err != nil { - return res, err + res.Stime, res.Utime, err = detectCPUUsageInV1(cgroupRootV1) } - res.Stime, res.Utime, err = detectCPUUsageInV2(cgroupRoot) if err != nil { return res, err } - default: - return CPUUsage{}, fmt.Errorf("detected unknown cgroup version index: %d", ver) + } else { + switch ver[0] { + case 1: + cgroupRoot := filepath.Join(root, mount[0]) + res.Period, res.Quota, err = detectCPUQuotaInV1(cgroupRoot) + if err != nil { + return res, err + } + res.Stime, res.Utime, err = detectCPUUsageInV1(cgroupRoot) + if err != nil { + return res, err + } + case 2: + cgroupRoot := filepath.Join(root, mount[0], path) + res.Period, res.Quota, err = detectCPUQuotaInV2(cgroupRoot) + if err != nil { + return res, err + } + res.Stime, res.Utime, err = detectCPUUsageInV2(cgroupRoot) + if err != nil { + return res, err + } + default: + return CPUUsage{}, fmt.Errorf("detected unknown cgroup version index: %d", ver) + } } return res, nil diff --git a/pkg/cgroup/cgroup_memory.go b/pkg/cgroup/cgroup_memory.go index 2addce9de50..fb8e8f212dc 100644 --- a/pkg/cgroup/cgroup_memory.go +++ b/pkg/cgroup/cgroup_memory.go @@ -61,13 +61,20 @@ func getCgroupMemInactiveFileUsage(root string) (usage uint64, err error) { return 0, err } - switch ver { - case 1: - usage, err = detectMemInactiveFileUsageInV1(filepath.Join(root, mount)) - case 2: - usage, err = detectMemInactiveFileUsageInV2(filepath.Join(root, mount, path)) - default: - usage, err = 0, fmt.Errorf("detected unknown cgroup version index: %d", ver) + if len(mount) == 2 { + usage, err = detectMemInactiveFileUsageInV1(filepath.Join(root, mount[0])) + if err != nil { + usage, err = detectMemInactiveFileUsageInV2(filepath.Join(root, mount[1], path)) + } + } else { + switch ver[0] { + case 1: + usage, err = detectMemInactiveFileUsageInV1(filepath.Join(root, mount[0])) + case 2: + usage, err = detectMemInactiveFileUsageInV2(filepath.Join(root, mount[0], path)) + default: + usage, err = 0, fmt.Errorf("detected unknown cgroup version index: %d", ver) + } } return usage, err @@ -90,15 +97,22 @@ func getCgroupMemUsage(root string) (usage uint64, err error) { return 0, err } - switch ver { - case 1: - // cgroupv1 - usage, err = detectMemUsageInV1(filepath.Join(root, mount)) - case 2: - // cgroupv2 - usage, err = detectMemUsageInV2(filepath.Join(root, mount, path)) - default: - usage, err = 0, fmt.Errorf("detected unknown cgroup version index: %d", ver) + if len(ver) == 2 { + usage, err = detectMemUsageInV1(filepath.Join(root, mount[0])) + if err != nil { + usage, err = detectMemUsageInV2(filepath.Join(root, mount[0], path)) + } + } else { + switch ver[0] { + case 1: + // cgroupv1 + usage, err = detectMemUsageInV1(filepath.Join(root, mount[0])) + case 2: + // cgroupv2 + usage, err = detectMemUsageInV2(filepath.Join(root, mount[0], path)) + default: + usage, err = 0, fmt.Errorf("detected unknown cgroup version index: %d", ver) + } } return usage, err @@ -121,15 +135,22 @@ func getCgroupMemLimit(root string) (limit uint64, err error) { return 0, err } - switch ver { - case 1: - // cgroupv1 - limit, err = detectMemLimitInV1(filepath.Join(root, mount)) - case 2: - // cgroupv2 - limit, err = detectMemLimitInV2(filepath.Join(root, mount, path)) - default: - limit, err = 0, fmt.Errorf("detected unknown cgroup version index: %d", ver) + if len(ver) == 2 { + limit, err = detectMemLimitInV1(filepath.Join(root, mount[0])) + if err != nil { + limit, err = detectMemLimitInV2(filepath.Join(root, mount[1], path)) + } + } else { + switch ver[0] { + case 1: + // cgroupv1 + limit, err = detectMemLimitInV1(filepath.Join(root, mount[0])) + case 2: + // cgroupv2 + limit, err = detectMemLimitInV2(filepath.Join(root, mount[0], path)) + default: + limit, err = 0, fmt.Errorf("detected unknown cgroup version index: %d", ver) + } } return limit, err diff --git a/pkg/cgroup/cgroup_mock_test.go b/pkg/cgroup/cgroup_mock_test.go index becedaa9c96..949e93bb125 100644 --- a/pkg/cgroup/cgroup_mock_test.go +++ b/pkg/cgroup/cgroup_mock_test.go @@ -168,6 +168,14 @@ func TestCgroupsGetMemoryInactiveFileUsage(t *testing.T) { }, value: 1363746816, }, + { + paths: map[string]string{ + "/proc/self/cgroup": v1CgroupWithEccentricMemoryController, + "/proc/self/mountinfo": v1MountsWithEccentricMemController, + "/sys/fs/cgroup/memory/memory.stat": v1MemoryStat, + }, + value: 1363746816, + }, { paths: map[string]string{ "/proc/self/cgroup": v2CgroupWithMemoryController, @@ -280,6 +288,14 @@ func TestCgroupsGetMemoryLimit(t *testing.T) { }, limit: 9223372036854775807, }, + { + paths: map[string]string{ + "/proc/self/cgroup": v1CgroupWithEccentricMemoryController, + "/proc/self/mountinfo": v1MountsWithEccentricMemController, + "/sys/fs/cgroup/memory/memory.stat": v1MemoryStat, + }, + limit: 2936016896, + }, } { dir := createFiles(t, tc.paths) limit, err := getCgroupMemLimit(dir) @@ -289,6 +305,70 @@ func TestCgroupsGetMemoryLimit(t *testing.T) { } } +//nolint:gosec +const ( + v1CgroupWithEccentricMemoryController = ` +13:devices:/system.slice/containerd.service/kubepods-burstable-pod94598a35_ad1e_4a00_91b1_1db37e8f52f6.slice:cri-containerd:0ac322a00cf64a4d58144a1974b993d91537f3ceec12928b10d881af6be8bbb2 +12:freezer:/kubepods-burstable-pod94598a35_ad1e_4a00_91b1_1db37e8f52f6.slice:cri-containerd:0ac322a00cf64a4d58144a1974b993d91537f3ceec12928b10d881af6be8bbb2 +11:cpu,cpuacct:/system.slice/containerd.service/kubepods-burstable-pod94598a35_ad1e_4a00_91b1_1db37e8f52f6.slice:cri-containerd:0ac322a00cf64a4d58144a1974b993d91537f3ceec12928b10d881af6be8bbb2 +10:perf_event:/kubepods-burstable-pod94598a35_ad1e_4a00_91b1_1db37e8f52f6.slice:cri-containerd:0ac322a00cf64a4d58144a1974b993d91537f3ceec12928b10d881af6be8bbb2 +9:rdma:/ +8:pids:/system.slice/containerd.service/kubepods-burstable-pod94598a35_ad1e_4a00_91b1_1db37e8f52f6.slice:cri-containerd:0ac322a00cf64a4d58144a1974b993d91537f3ceec12928b10d881af6be8bbb2 +7:blkio:/system.slice/containerd.service/kubepods-burstable-pod94598a35_ad1e_4a00_91b1_1db37e8f52f6.slice:cri-containerd:0ac322a00cf64a4d58144a1974b993d91537f3ceec12928b10d881af6be8bbb2 +6:hugetlb:/kubepods-burstable-pod94598a35_ad1e_4a00_91b1_1db37e8f52f6.slice:cri-containerd:0ac322a00cf64a4d58144a1974b993d91537f3ceec12928b10d881af6be8bbb2 +5:memory:/system.slice/containerd.service/kubepods-burstable-pod94598a35_ad1e_4a00_91b1_1db37e8f52f6.slice:cri-containerd:0ac322a00cf64a4d58144a1974b993d91537f3ceec12928b10d881af6be8bbb2 +4:cpuset:/kubepods-burstable-pod94598a35_ad1e_4a00_91b1_1db37e8f52f6.slice:cri-containerd:0ac322a00cf64a4d58144a1974b993d91537f3ceec12928b10d881af6be8bbb2 +3:files:/ +2:net_cls,net_prio:/kubepods-burstable-pod94598a35_ad1e_4a00_91b1_1db37e8f52f6.slice:cri-containerd:0ac322a00cf64a4d58144a1974b993d91537f3ceec12928b10d881af6be8bbb2 +1:name=systemd:/system.slice/containerd.service/kubepods-burstable-pod94598a35_ad1e_4a00_91b1_1db37e8f52f6.slice:cri-containerd:0ac322a00cf64a4d58144a1974b993d91537f3ceec12928b10d881af6be8bbb2 +0::/ +` + v1MountsWithEccentricMemController = ` +1421 1021 0:133 / / rw,relatime master:412 - overlay overlay rw,lowerdir=/apps/data/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/1285288/fs:/apps/data/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/1285287/fs:/apps/data/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/1285286/fs:/apps/data/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/1285285/fs:/apps/data/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/1283928/fs,upperdir=/apps/data/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/1287880/fs,workdir=/apps/data/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/1287880/work +1442 1421 0:136 / /proc rw,nosuid,nodev,noexec,relatime - proc proc rw +1443 1421 0:137 / /dev rw,nosuid - tmpfs tmpfs rw,size=65536k,mode=755 +1444 1443 0:138 / /dev/pts rw,nosuid,noexec,relatime - devpts devpts rw,gid=5,mode=620,ptmxmode=666 +2303 1443 0:119 / /dev/mqueue rw,nosuid,nodev,noexec,relatime - mqueue mqueue rw +2304 1421 0:129 / /sys ro,nosuid,nodev,noexec,relatime - sysfs sysfs ro +2305 2304 0:139 / /sys/fs/cgroup rw,nosuid,nodev,noexec,relatime - tmpfs tmpfs rw,mode=755 +2306 2305 0:25 /system.slice/containerd.service/kubepods-burstable-pod94598a35_ad1e_4a00_91b1_1db37e8f52f6.slice:cri-containerd:0ac322a00cf64a4d58144a1974b993d91537f3ceec12928b10d881af6be8bbb2 /sys/fs/cgroup/systemd ro,nosuid,nodev,noexec,relatime master:5 - cgroup cgroup rw,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd +2307 2305 0:28 /kubepods-burstable-pod94598a35_ad1e_4a00_91b1_1db37e8f52f6.slice:cri-containerd:0ac322a00cf64a4d58144a1974b993d91537f3ceec12928b10d881af6be8bbb2 /sys/fs/cgroup/net_cls,net_prio ro,nosuid,nodev,noexec,relatime master:6 - cgroup cgroup rw,net_cls,net_prio +2308 2305 0:29 / /sys/fs/cgroup/files ro,nosuid,nodev,noexec,relatime master:7 - cgroup cgroup rw,files +2309 2305 0:30 /kubepods-burstable-pod94598a35_ad1e_4a00_91b1_1db37e8f52f6.slice:cri-containerd:0ac322a00cf64a4d58144a1974b993d91537f3ceec12928b10d881af6be8bbb2 /sys/fs/cgroup/cpuset ro,nosuid,nodev,noexec,relatime master:8 - cgroup cgroup rw,cpuset +2310 2305 0:31 /system.slice/containerd.service/kubepods-burstable-pod94598a35_ad1e_4a00_91b1_1db37e8f52f6.slice:cri-containerd:0ac322a00cf64a4d58144a1974b993d91537f3ceec12928b10d881af6be8bbb2 /sys/fs/cgroup/memory ro,nosuid,nodev,noexec,relatime master:9 - cgroup cgroup rw,memory +2311 2305 0:32 /kubepods-burstable-pod94598a35_ad1e_4a00_91b1_1db37e8f52f6.slice:cri-containerd:0ac322a00cf64a4d58144a1974b993d91537f3ceec12928b10d881af6be8bbb2 /sys/fs/cgroup/hugetlb ro,nosuid,nodev,noexec,relatime master:10 - cgroup cgroup rw,hugetlb +2312 2305 0:33 /system.slice/containerd.service/kubepods-burstable-pod94598a35_ad1e_4a00_91b1_1db37e8f52f6.slice:cri-containerd:0ac322a00cf64a4d58144a1974b993d91537f3ceec12928b10d881af6be8bbb2 /sys/fs/cgroup/blkio ro,nosuid,nodev,noexec,relatime master:11 - cgroup cgroup rw,blkio +2313 2305 0:34 /system.slice/containerd.service/kubepods-burstable-pod94598a35_ad1e_4a00_91b1_1db37e8f52f6.slice:cri-containerd:0ac322a00cf64a4d58144a1974b993d91537f3ceec12928b10d881af6be8bbb2 /sys/fs/cgroup/pids ro,nosuid,nodev,noexec,relatime master:12 - cgroup cgroup rw,pids +2314 2305 0:35 / /sys/fs/cgroup/rdma ro,nosuid,nodev,noexec,relatime master:13 - cgroup cgroup rw,rdma +2315 2305 0:36 /kubepods-burstable-pod94598a35_ad1e_4a00_91b1_1db37e8f52f6.slice:cri-containerd:0ac322a00cf64a4d58144a1974b993d91537f3ceec12928b10d881af6be8bbb2 /sys/fs/cgroup/perf_event ro,nosuid,nodev,noexec,relatime master:14 - cgroup cgroup rw,perf_event +2316 2305 0:37 /system.slice/containerd.service/kubepods-burstable-pod94598a35_ad1e_4a00_91b1_1db37e8f52f6.slice:cri-containerd:0ac322a00cf64a4d58144a1974b993d91537f3ceec12928b10d881af6be8bbb2 /sys/fs/cgroup/cpu,cpuacct ro,nosuid,nodev,noexec,relatime master:15 - cgroup cgroup rw,cpu,cpuacct +2317 2305 0:38 /kubepods-burstable-pod94598a35_ad1e_4a00_91b1_1db37e8f52f6.slice:cri-containerd:0ac322a00cf64a4d58144a1974b993d91537f3ceec12928b10d881af6be8bbb2 /sys/fs/cgroup/freezer ro,nosuid,nodev,noexec,relatime master:16 - cgroup cgroup rw,freezer +2318 2305 0:39 /system.slice/containerd.service/kubepods-burstable-pod94598a35_ad1e_4a00_91b1_1db37e8f52f6.slice:cri-containerd:0ac322a00cf64a4d58144a1974b993d91537f3ceec12928b10d881af6be8bbb2 /sys/fs/cgroup/devices ro,nosuid,nodev,noexec,relatime master:17 - cgroup cgroup rw,devices +2319 1421 0:101 / /etc/podinfo ro,relatime - tmpfs tmpfs rw +2320 1421 253:3 /data/containerd/io.containerd.grpc.v1.cri/sandboxes/22c18c845c47667097eb8973fd0ec05256be685cd1b1a8b0fe7c748a04401cdb/hostname /etc/hostname rw,relatime - xfs /dev/mapper/vg1-lvm1k8s rw,attr2,inode64,sunit=512,swidth=512,noquota +2321 1421 253:3 /data/kubelet/pods/94598a35-ad1e-4a00-91b1-1db37e8f52f6/volumes/kubernetes.io~configmap/config /etc/tikv ro,relatime - xfs /dev/mapper/vg1-lvm1k8s rw,attr2,inode64,sunit=512,swidth=512,noquota +2322 1443 0:104 / /dev/shm rw,nosuid,nodev,noexec,relatime - tmpfs shm rw,size=65536k +2323 1421 253:3 /data/kubelet/pods/94598a35-ad1e-4a00-91b1-1db37e8f52f6/etc-hosts /etc/hosts rw,relatime - xfs /dev/mapper/vg1-lvm1k8s rw,attr2,inode64,sunit=512,swidth=512,noquota +2324 1443 253:3 /data/kubelet/pods/94598a35-ad1e-4a00-91b1-1db37e8f52f6/containers/tikv/0981845c /dev/termination-log rw,relatime - xfs /dev/mapper/vg1-lvm1k8s rw,attr2,inode64,sunit=512,swidth=512,noquota +2325 1421 253:3 /data/containerd/io.containerd.grpc.v1.cri/sandboxes/22c18c845c47667097eb8973fd0ec05256be685cd1b1a8b0fe7c748a04401cdb/resolv.conf /etc/resolv.conf rw,relatime - xfs /dev/mapper/vg1-lvm1k8s rw,attr2,inode64,sunit=512,swidth=512,noquota +2326 1421 253:2 /pv03 /var/lib/tikv rw,relatime - xfs /dev/mapper/vg2-lvm2k8s rw,attr2,inode64,sunit=512,swidth=512,noquota +2327 1421 253:3 /data/kubelet/pods/94598a35-ad1e-4a00-91b1-1db37e8f52f6/volumes/kubernetes.io~configmap/startup-script /usr/local/bin ro,relatime - xfs /dev/mapper/vg1-lvm1k8s rw,attr2,inode64,sunit=512,swidth=512,noquota +2328 1421 0:102 / /run/secrets/kubernetes.io/serviceaccount ro,relatime - tmpfs tmpfs rw +1022 1442 0:136 /bus /proc/bus ro,nosuid,nodev,noexec,relatime - proc proc rw +1034 1442 0:136 /fs /proc/fs ro,nosuid,nodev,noexec,relatime - proc proc rw +1035 1442 0:136 /irq /proc/irq ro,nosuid,nodev,noexec,relatime - proc proc rw +1036 1442 0:136 /sys /proc/sys ro,nosuid,nodev,noexec,relatime - proc proc rw +1037 1442 0:136 /sysrq-trigger /proc/sysrq-trigger ro,nosuid,nodev,noexec,relatime - proc proc rw +1038 1442 0:161 / /proc/acpi ro,relatime - tmpfs tmpfs ro +1039 1442 0:137 /null /proc/kcore rw,nosuid - tmpfs tmpfs rw,size=65536k,mode=755 +1040 1442 0:137 /null /proc/keys rw,nosuid - tmpfs tmpfs rw,size=65536k,mode=755 +1041 1442 0:137 /null /proc/timer_list rw,nosuid - tmpfs tmpfs rw,size=65536k,mode=755 +1042 1442 0:137 /null /proc/sched_debug rw,nosuid - tmpfs tmpfs rw,size=65536k,mode=755 +1043 1442 0:162 / /proc/scsi ro,relatime - tmpfs tmpfs ro +1044 2304 0:163 / /sys/firmware ro,relatime - tmpfs tmpfs ro +` +) + func TestCgroupsGetCPU(t *testing.T) { for _, tc := range []struct { name string @@ -445,6 +525,20 @@ func TestCgroupsGetCPU(t *testing.T) { period: int64(1000), errMsg: "can't read cpu usage from cgroup v2", }, + { + paths: map[string]string{ + "/proc/self/cgroup": MixCgroup, + "/proc/self/mountinfo": MixMounts, + "/sys/fs/cgroup/cpu,cpuacct/user.slice/cpu.cfs_quota_us": "12345", + "/sys/fs/cgroup/cpu,cpuacct/user.slice/cpu.cfs_period_us": "67890", + "/sys/fs/cgroup/cpu,cpuacct/user.slice/cpuacct.usage_sys": "123", + "/sys/fs/cgroup/cpu,cpuacct/user.slice/cpuacct.usage_user": "456", + }, + quota: int64(12345), + period: int64(67890), + system: uint64(123), + user: uint64(456), + }, } { dir := createFiles(t, tc.paths) @@ -829,4 +923,178 @@ thp_collapse_alloc 0 //#nosec G101 v1CgroupWithCPUControllerNS2 = "5:cpu,cpuacct:/crdb_test" v1MountsWithCPUControllerNS2 = "161 43 0:37 /crdb_test /sys/fs/cgroup/cpu,cpuacct/crdb_test rw,relatime shared:95 - cgroup none rw,cpu,cpuacct" + + MixCgroup = `12:hugetlb:/ +11:memory:/user.slice/user-1006.slice/session-17838.scope +10:pids:/user.slice/user-1006.slice/session-17838.scope +9:devices:/user.slice +8:perf_event:/ +7:cpu,cpuacct:/user.slice +6:blkio:/user.slice +5:cpuset:/ +4:net_cls,net_prio:/ +3:freezer:/ +2:rdma:/ +1:name=systemd:/user.slice/user-1006.slice/session-17838.scope +0::/user.slice/user-1006.slice/session-17838.scope +` + + MixMounts = ` +25 30 0:23 / /sys rw,relatime shared:7 - sysfs sysfs rw +26 30 0:5 / /proc rw,relatime shared:14 - proc proc rw +27 30 0:6 / /dev rw,nosuid,noexec,relatime shared:2 - devtmpfs udev rw,size=197385544k,nr_inodes=49346386,mode=755 +28 27 0:24 / /dev/pts rw,relatime shared:3 - devpts devpts rw,gid=5,mode=620,ptmxmode=000 +29 30 0:25 / /run rw,nosuid,nodev,noexec,relatime shared:5 - tmpfs tmpfs rw,size=39486148k,mode=755 +30 1 8:3 / / rw,relatime shared:1 - ext4 /dev/sda3 rw,stripe=16 +31 25 0:7 / /sys/kernel/security rw,nosuid,nodev,noexec,relatime shared:8 - securityfs securityfs rw +32 27 0:26 / /dev/shm rw shared:4 - tmpfs tmpfs rw +33 29 0:27 / /run/lock rw,nosuid,nodev,noexec,relatime shared:6 - tmpfs tmpfs rw,size=5120k +34 25 0:28 / /sys/fs/cgroup ro,nosuid,nodev,noexec shared:9 - tmpfs tmpfs ro,mode=755 +35 34 0:29 / /sys/fs/cgroup/unified rw,nosuid,nodev,noexec,relatime shared:10 - cgroup2 cgroup2 rw +36 34 0:30 / /sys/fs/cgroup/systemd rw,nosuid,nodev,noexec,relatime shared:11 - cgroup cgroup rw,xattr,name=systemd +37 25 0:31 / /sys/fs/pstore rw,nosuid,nodev,noexec,relatime shared:12 - pstore pstore rw +39 34 0:33 / /sys/fs/cgroup/rdma rw,nosuid,nodev,noexec,relatime shared:15 - cgroup cgroup rw,rdma +40 34 0:34 / /sys/fs/cgroup/freezer rw,nosuid,nodev,noexec,relatime shared:16 - cgroup cgroup rw,freezer +41 34 0:35 / /sys/fs/cgroup/net_cls,net_prio rw,nosuid,nodev,noexec,relatime shared:17 - cgroup cgroup rw,net_cls,net_prio +42 34 0:36 / /sys/fs/cgroup/cpuset rw,nosuid,nodev,noexec,relatime shared:18 - cgroup cgroup rw,cpuset +43 34 0:37 / /sys/fs/cgroup/blkio rw,nosuid,nodev,noexec,relatime shared:19 - cgroup cgroup rw,blkio +44 34 0:38 / /sys/fs/cgroup/cpu,cpuacct rw,nosuid,nodev,noexec,relatime shared:20 - cgroup cgroup rw,cpu,cpuacct +45 34 0:39 / /sys/fs/cgroup/perf_event rw,nosuid,nodev,noexec,relatime shared:21 - cgroup cgroup rw,perf_event +46 34 0:40 / /sys/fs/cgroup/devices rw,nosuid,nodev,noexec,relatime shared:22 - cgroup cgroup rw,devices +47 34 0:41 / /sys/fs/cgroup/pids rw,nosuid,nodev,noexec,relatime shared:23 - cgroup cgroup rw,pids +48 34 0:42 / /sys/fs/cgroup/memory rw,nosuid,nodev,noexec,relatime shared:24 - cgroup cgroup rw,memory +49 34 0:43 / /sys/fs/cgroup/hugetlb rw,nosuid,nodev,noexec,relatime shared:25 - cgroup cgroup rw,hugetlb +50 26 0:44 / /proc/sys/fs/binfmt_misc rw,relatime shared:26 - autofs systemd-1 rw,fd=28,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=91621 +51 27 0:45 / /dev/hugepages rw,relatime shared:27 - hugetlbfs hugetlbfs rw,pagesize=2M +52 27 0:21 / /dev/mqueue rw,nosuid,nodev,noexec,relatime shared:28 - mqueue mqueue rw +53 25 0:8 / /sys/kernel/debug rw,nosuid,nodev,noexec,relatime shared:29 - debugfs debugfs rw +54 25 0:12 / /sys/kernel/tracing rw,nosuid,nodev,noexec,relatime shared:30 - tracefs tracefs rw +55 25 0:46 / /sys/fs/fuse/connections rw,nosuid,nodev,noexec,relatime shared:31 - fusectl fusectl rw +56 25 0:22 / /sys/kernel/config rw,nosuid,nodev,noexec,relatime shared:32 - configfs configfs rw +142 30 8:2 / /boot rw,relatime shared:79 - ext4 /dev/sda2 rw,stripe=16,data=ordered +145 30 259:1 / /data/nvme0n1 rw,relatime shared:81 - ext4 /dev/nvme0n1 rw +605 29 0:25 /snapd/ns /run/snapd/ns rw,nosuid,nodev,noexec,relatime - tmpfs tmpfs rw,size=39486148k,mode=755 +624 29 0:49 / /run/user/0 rw,nosuid,nodev,relatime shared:341 - tmpfs tmpfs rw,size=39486144k,mode=700 +642 30 259:3 / /mnt/c42ca499-9a7c-4d19-ae60-e8a46a6956ba rw,relatime shared:348 - ext4 /dev/nvme2n1 rw +798 30 259:2 / /mnt/a688878a-492b-4536-a03c-f50ce8a1f014 rw,relatime shared:386 - ext4 /dev/nvme3n1 rw +887 30 259:0 / /mnt/f97f162d-be90-4bfa-bae5-2698f5ce634d rw,relatime shared:424 - ext4 /dev/nvme1n1 rw +976 29 0:53 / /run/containerd/io.containerd.grpc.v1.cri/sandboxes/2191a8cf52bd8313c6abef93260d6964f6b7240117f7cc0723c9647caa78bb45/shm rw,nosuid,nodev,noexec,relatime shared:462 - tmpfs shm rw,size=65536k +993 29 0:54 / /run/containerd/io.containerd.runtime.v2.task/k8s.io/2191a8cf52bd8313c6abef93260d6964f6b7240117f7cc0723c9647caa78bb45/rootfs rw,relatime shared:469 - overlay overlay rw,lowerdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/1/fs,upperdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/8260/fs,workdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/8260/work,xino=off +1113 145 0:80 / /data/nvme0n1/kubelet/pods/428eee2f-da5c-44de-aae1-951b3746e3d8/volumes/kubernetes.io~secret/clustermesh-secrets rw,relatime shared:497 - tmpfs tmpfs rw +1185 145 0:89 / /data/nvme0n1/kubelet/pods/07d67bd3-f23b-4d2c-84ae-03a2df2f42a6/volumes/kubernetes.io~projected/kube-api-access-6c2d6 rw,relatime shared:518 - tmpfs tmpfs rw +1223 29 0:90 / /run/containerd/io.containerd.grpc.v1.cri/sandboxes/8231ae5d138d3e9da5a996bbc833175580fe7c201dfe66398df2cc0285e1bdfe/shm rw,nosuid,nodev,noexec,relatime shared:525 - tmpfs shm rw,size=65536k +1257 29 0:92 / /run/containerd/io.containerd.runtime.v2.task/k8s.io/8231ae5d138d3e9da5a996bbc833175580fe7c201dfe66398df2cc0285e1bdfe/rootfs rw,relatime shared:539 - overlay overlay rw,lowerdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/1/fs,upperdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/8264/fs,workdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/8264/work,xino=off +1521 145 0:112 / /data/nvme0n1/kubelet/pods/428eee2f-da5c-44de-aae1-951b3746e3d8/volumes/kubernetes.io~projected/kube-api-access-6kcqx rw,relatime shared:567 - tmpfs tmpfs rw +1519 29 0:121 / /run/containerd/io.containerd.grpc.v1.cri/sandboxes/11cbc306aeead0190cb68c6329bafaf979353cd193e8ecb47e61b693cfd2b0f5/shm rw,nosuid,nodev,noexec,relatime shared:574 - tmpfs shm rw,size=65536k +1556 29 0:122 / /run/containerd/io.containerd.runtime.v2.task/k8s.io/11cbc306aeead0190cb68c6329bafaf979353cd193e8ecb47e61b693cfd2b0f5/rootfs rw,relatime shared:581 - overlay overlay rw,lowerdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/1/fs,upperdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/8268/fs,workdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/8268/work,xino=off +5312 50 0:803 / /proc/sys/fs/binfmt_misc rw,nosuid,nodev,noexec,relatime shared:1817 - binfmt_misc binfmt_misc rw +2312 30 7:2 / /snap/lxd/23991 ro,nodev,relatime shared:742 - squashfs /dev/loop2 ro +458 605 0:4 mnt:[4026537320] /run/snapd/ns/lxd.mnt rw - nsfs nsfs rw +755 30 7:4 / /snap/lxd/24061 ro,nodev,relatime shared:511 - squashfs /dev/loop4 ro +1027 29 0:63 / /run/containerd/io.containerd.runtime.v2.task/k8s.io/a802a3a6b73af45d20aa3d7a07b6c280b618b91320b4378812a2552e413935c2/rootfs rw,relatime shared:476 - overlay overlay rw,lowerdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/13/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/12/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/11/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/10/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/6/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/4/fs,upperdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/26137/fs,workdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/26137/work,xino=off +4698 29 0:110 / /run/containerd/io.containerd.runtime.v2.task/k8s.io/8b6bac7cc0fd04bc2d1baf50f8f2fee0f81f36a13bf06f4b4ff195586a87588f/rootfs rw,relatime shared:2321 - overlay overlay rw,lowerdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/15/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/9/fs,upperdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/26141/fs,workdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/26141/work,xino=off +10124 29 0:176 / /run/containerd/io.containerd.runtime.v2.task/k8s.io/ed885e6b247e68336830e4120db20d7f43c7a996a82fa86a196e48743a52eff4/rootfs rw,relatime shared:2417 - overlay overlay rw,lowerdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/28/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/27/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/26/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/25/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/24/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/23/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/22/fs,upperdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/26143/fs,workdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/26143/work,xino=off +3432 30 7:3 / /snap/snapd/18357 ro,nodev,relatime shared:1001 - squashfs /dev/loop3 ro +2768 30 7:11 / /snap/core/14784 ro,nodev,relatime shared:938 - squashfs /dev/loop11 ro +2265 30 7:12 / /snap/k9s/151 ro,nodev,relatime shared:725 - squashfs /dev/loop12 ro +2297 30 7:13 / /snap/core20/1828 ro,nodev,relatime shared:765 - squashfs /dev/loop13 ro +1411 30 7:14 / /snap/go/10073 ro,nodev,relatime shared:550 - squashfs /dev/loop14 ro +115 145 0:103 / /data/nvme0n1/kubelet/pods/bb3ece51-01a1-4d9d-a48a-43093c72a3a2/volumes/kubernetes.io~projected/kube-api-access-5nvm9 rw,relatime shared:442 - tmpfs tmpfs rw +1454 29 0:108 / /run/containerd/io.containerd.grpc.v1.cri/sandboxes/282a3e51b05ed2b168285995fce71e9d882db9d4cb33e54a367791fe92fd8cd2/shm rw,nosuid,nodev,noexec,relatime shared:740 - tmpfs shm rw,size=65536k +1516 29 0:109 / /run/containerd/io.containerd.runtime.v2.task/k8s.io/282a3e51b05ed2b168285995fce71e9d882db9d4cb33e54a367791fe92fd8cd2/rootfs rw,relatime shared:788 - overlay overlay rw,lowerdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/1/fs,upperdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/26610/fs,workdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/26610/work,xino=off +2409 29 0:209 / /run/containerd/io.containerd.runtime.v2.task/k8s.io/29d6387bdeed4df9882df5f73645c071317999c6f913a739d42390507485e8c5/rootfs rw,relatime shared:869 - overlay overlay rw,lowerdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/30/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/9/fs,upperdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/26611/fs,workdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/26611/work,xino=off +2270 30 7:15 / /snap/core18/2714 ro,nodev,relatime shared:1068 - squashfs /dev/loop15 ro +137 30 0:71 / /var/lib/docker/overlay2/41ea64be1d943b66e7cce1d07ca48f1c6359dd7e983ffc4100b122289d4fc457/merged rw,relatime shared:49 - overlay overlay rw,lowerdir=/var/lib/docker/overlay2/l/MZDFUOLZV7OH5FLPS5ZWNO6JHC:/var/lib/docker/overlay2/l/SFPFSYXYX4E3ST5E7Y5BTG3RAG:/var/lib/docker/overlay2/l/NVHRNLY3A7KDIXEPAEZZUAKKAF:/var/lib/docker/overlay2/l/3NWZQJULTIAEMU33EBOV3DO4KK:/var/lib/docker/overlay2/l/24BFJIPXS2PJ4XI7A4OB4FCK3N,upperdir=/var/lib/docker/overlay2/41ea64be1d943b66e7cce1d07ca48f1c6359dd7e983ffc4100b122289d4fc457/diff,workdir=/var/lib/docker/overlay2/41ea64be1d943b66e7cce1d07ca48f1c6359dd7e983ffc4100b122289d4fc457/work,xino=off +138 30 0:72 / /var/lib/docker/overlay2/6ba83a6794d649bd38fb9698e067ce8fb22e6a976af0cd9d003b86847b54dde7/merged rw,relatime shared:73 - overlay overlay rw,lowerdir=/var/lib/docker/overlay2/l/DAWHHXAM2AYTI37UJOH6W453EZ:/var/lib/docker/overlay2/l/SFPFSYXYX4E3ST5E7Y5BTG3RAG:/var/lib/docker/overlay2/l/NVHRNLY3A7KDIXEPAEZZUAKKAF:/var/lib/docker/overlay2/l/3NWZQJULTIAEMU33EBOV3DO4KK:/var/lib/docker/overlay2/l/24BFJIPXS2PJ4XI7A4OB4FCK3N,upperdir=/var/lib/docker/overlay2/6ba83a6794d649bd38fb9698e067ce8fb22e6a976af0cd9d003b86847b54dde7/diff,workdir=/var/lib/docker/overlay2/6ba83a6794d649bd38fb9698e067ce8fb22e6a976af0cd9d003b86847b54dde7/work,xino=off +4039 29 0:4 net:[4026537384] /run/docker/netns/c8477f57c25f rw shared:123 - nsfs nsfs rw +4059 29 0:4 net:[4026537817] /run/docker/netns/64d7952bb68f rw shared:833 - nsfs nsfs rw +665 30 259:3 /vol1 /mnt/disks/c42ca499-9a7c-4d19-ae60-e8a46a6956ba_vol1 rw,relatime shared:348 - ext4 /dev/nvme2n1 rw +750 30 259:2 /vol1 /mnt/disks/a688878a-492b-4536-a03c-f50ce8a1f014_vol1 rw,relatime shared:386 - ext4 /dev/nvme3n1 rw +779 30 259:0 /vol1 /mnt/disks/f97f162d-be90-4bfa-bae5-2698f5ce634d_vol1 rw,relatime shared:424 - ext4 /dev/nvme1n1 rw +38 25 0:256 / /sys/fs/bpf rw,relatime shared:13 - bpf none rw +3174 30 7:16 / /snap/core/14946 ro,nodev,relatime shared:846 - squashfs /dev/loop16 ro +965 30 7:17 / /snap/core20/1852 ro,nodev,relatime shared:436 - squashfs /dev/loop17 ro +3663 30 7:5 / /snap/snapd/18596 ro,nodev,relatime shared:1170 - squashfs /dev/loop5 ro +2275 30 7:8 / /snap/core22/583 ro,nodev,relatime shared:449 - squashfs /dev/loop8 ro +4856 30 7:9 / /snap/core18/2721 ro,nodev,relatime shared:1229 - squashfs /dev/loop9 ro +1225 29 0:487 / /run/user/1003 rw,nosuid,nodev,relatime shared:987 - tmpfs tmpfs rw,size=39486144k,mode=700,uid=1003,gid=1003 +311 605 0:4 mnt:[4026537731] /run/snapd/ns/k9s.mnt rw - nsfs nsfs rw +80 29 0:32 / /run/containerd/io.containerd.grpc.v1.cri/sandboxes/c6189676909828b8d0cbce711e115fa3037e30122a4e359bfca0fc4f628d91da/shm rw,nosuid,nodev,noexec,relatime shared:55 - tmpfs shm rw,size=65536k +100 29 0:50 / /run/containerd/io.containerd.runtime.v2.task/k8s.io/c6189676909828b8d0cbce711e115fa3037e30122a4e359bfca0fc4f628d91da/rootfs rw,relatime shared:62 - overlay overlay rw,lowerdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/1/fs,upperdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/30973/fs,workdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/30973/work,xino=off +497 29 0:91 / /run/containerd/io.containerd.runtime.v2.task/k8s.io/19a62c19cdd472887f8b1510143d8a79151f70f24bef25a649b8c063a7fe0dff/rootfs rw,relatime shared:75 - overlay overlay rw,lowerdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/30976/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/30975/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/30974/fs,upperdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/30977/fs,workdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/30977/work,xino=off +643 145 0:113 / /data/nvme0n1/kubelet/pods/8c096a91-03cb-4948-b0b1-10bf212928e8/volumes/kubernetes.io~secret/kubevirt-virt-handler-certs rw,relatime shared:171 - tmpfs tmpfs rw +696 145 0:115 / /data/nvme0n1/kubelet/pods/8c096a91-03cb-4948-b0b1-10bf212928e8/volumes/kubernetes.io~secret/kubevirt-virt-handler-server-certs rw,relatime shared:223 - tmpfs tmpfs rw +699 145 0:116 / /data/nvme0n1/kubelet/pods/8c096a91-03cb-4948-b0b1-10bf212928e8/volumes/kubernetes.io~downward-api/podinfo rw,relatime shared:272 - tmpfs tmpfs rw +810 145 0:119 / /data/nvme0n1/kubelet/pods/8c096a91-03cb-4948-b0b1-10bf212928e8/volumes/kubernetes.io~projected/kube-api-access-b4ctw rw,relatime shared:323 - tmpfs tmpfs rw +848 29 0:4 net:[4026537303] /run/netns/cni-86b41b5e-f846-34a4-3d24-4bc6f7a2b70d rw shared:360 - nsfs nsfs rw +1191 29 0:133 / /run/containerd/io.containerd.grpc.v1.cri/sandboxes/e5326f2fb12efbadde2acc5095415191214862b730fb4abbe4acc4f0a982dfc0/shm rw,nosuid,nodev,noexec,relatime shared:382 - tmpfs shm rw,size=65536k +1226 29 0:134 / /run/containerd/io.containerd.runtime.v2.task/k8s.io/e5326f2fb12efbadde2acc5095415191214862b730fb4abbe4acc4f0a982dfc0/rootfs rw,relatime shared:396 - overlay overlay rw,lowerdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/1/fs,upperdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/30978/fs,workdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/30978/work,xino=off +3252 29 0:350 / /run/containerd/io.containerd.runtime.v2.task/k8s.io/0525c449b405367e89061da50eb3cd480692aa0c3f47e98944fd61a6d9d686e1/rootfs rw,relatime shared:1057 - overlay overlay rw,lowerdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/30998/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/30989/fs,upperdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/30999/fs,workdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/30999/work,xino=off +4090 145 0:422 / /data/nvme0n1/kubelet/pods/c7676e13-5326-4a20-9823-1709d2ec124f/volumes/kubernetes.io~secret/chaos-daemon-cert rw,relatime shared:1120 - tmpfs tmpfs rw +4149 145 0:423 / /data/nvme0n1/kubelet/pods/c7676e13-5326-4a20-9823-1709d2ec124f/volumes/kubernetes.io~projected/kube-api-access-jvl6g rw,relatime shared:1140 - tmpfs tmpfs rw +4166 29 0:4 net:[4026538463] /run/netns/cni-f4356533-4c6e-f374-942a-f8fb8aaef128 rw shared:1147 - nsfs nsfs rw +4218 29 0:424 / /run/containerd/io.containerd.grpc.v1.cri/sandboxes/38bf769b3260aac70efc295dbeeba2d9ae1e0de3df2347f1926e2344336ed61a/shm rw,nosuid,nodev,noexec,relatime shared:1154 - tmpfs shm rw,size=65536k +4238 29 0:425 / /run/containerd/io.containerd.runtime.v2.task/k8s.io/38bf769b3260aac70efc295dbeeba2d9ae1e0de3df2347f1926e2344336ed61a/rootfs rw,relatime shared:1161 - overlay overlay rw,lowerdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/1/fs,upperdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/31135/fs,workdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/31135/work,xino=off +3764 29 0:414 / /run/containerd/io.containerd.runtime.v2.task/k8s.io/debe913e95afafbe487c2ff40032ef88fedab297dc35f01873a17eb4d3911137/rootfs rw,relatime shared:1099 - overlay overlay rw,lowerdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/31149/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/31148/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/31147/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/31146/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/31145/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/31144/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/31143/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/31142/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/31141/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/31140/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/31139/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/31138/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/31137/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/31136/fs,upperdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/31150/fs,workdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/31150/work,xino=off +4325 145 0:447 / /data/nvme0n1/kubelet/pods/eaa8a805-56af-4054-a3d1-87d89b4427f7/volumes/kubernetes.io~projected/kube-api-access-kdkpp rw,relatime shared:1189 - tmpfs tmpfs rw +4354 29 0:4 net:[4026538607] /run/netns/cni-3767b6e7-dbc4-0ccd-d26c-b7a20f4db13a rw shared:1196 - nsfs nsfs rw +4374 29 0:448 / /run/containerd/io.containerd.grpc.v1.cri/sandboxes/ea5708a1a0574c874fa64e782929b1ce52a68561d6d4c57200dcaeae42a412a0/shm rw,nosuid,nodev,noexec,relatime shared:1203 - tmpfs shm rw,size=65536k +4402 29 0:449 / /run/containerd/io.containerd.runtime.v2.task/k8s.io/ea5708a1a0574c874fa64e782929b1ce52a68561d6d4c57200dcaeae42a412a0/rootfs rw,relatime shared:1210 - overlay overlay rw,lowerdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/1/fs,upperdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/31154/fs,workdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/31154/work,xino=off +4472 29 0:460 / /run/containerd/io.containerd.runtime.v2.task/k8s.io/3f73b6ed13e1e285c13f20c568e353945cc9705dd7b027985c281a0f03c514b4/rootfs rw,relatime shared:1217 - overlay overlay rw,lowerdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/31158/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/31157/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/31156/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/31155/fs,upperdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/31159/fs,workdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/31159/work,xino=off +4655 29 0:465 / /run/user/1006 rw,nosuid,nodev,relatime shared:1117 - tmpfs tmpfs rw,size=39486144k,mode=700,uid=1006,gid=1006 +5412 30 7:6 / /snap/core22/607 ro,nodev,relatime shared:1364 - squashfs /dev/loop6 ro +4865 30 7:1 / /snap/hello-world/29 ro,nodev,relatime shared:1292 - squashfs /dev/loop1 ro +4982 30 7:18 / /snap/go/10135 ro,nodev,relatime shared:1299 - squashfs /dev/loop18 ro +1136 145 0:149 / /data/nvme0n1/kubelet/pods/61b41e1e-6caf-4f4c-aa97-b8ba1b38168e/volumes/kubernetes.io~projected/kube-api-access-r9chm rw,relatime shared:404 - tmpfs tmpfs rw +1357 29 0:4 net:[4026537584] /run/netns/cni-b6788c05-b9f7-d2a8-1cca-729821632d90 rw shared:451 - nsfs nsfs rw +1497 29 0:167 / /run/containerd/io.containerd.grpc.v1.cri/sandboxes/af946ebe99e17756f7b3d7a97fcb290dcae7cc6b9f816f2d6942f15afba7c28b/shm rw,nosuid,nodev,noexec,relatime shared:477 - tmpfs shm rw,size=65536k +1657 29 0:168 / /run/containerd/io.containerd.runtime.v2.task/k8s.io/af946ebe99e17756f7b3d7a97fcb290dcae7cc6b9f816f2d6942f15afba7c28b/rootfs rw,relatime shared:510 - overlay overlay rw,lowerdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/1/fs,upperdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/33975/fs,workdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/33975/work,xino=off +1679 145 0:180 / /data/nvme0n1/kubelet/pods/7830474c-474d-4ea7-b568-bc70f0c804b2/volumes/kubernetes.io~projected/kube-api-access-lsctl rw,relatime shared:547 - tmpfs tmpfs rw +1740 29 0:4 net:[4026537658] /run/netns/cni-85144543-ac5f-d5cb-e113-613e2360d33e rw shared:559 - nsfs nsfs rw +1765 29 0:181 / /run/containerd/io.containerd.grpc.v1.cri/sandboxes/ecd66ebdf9e85e447076ab5f010a9a721102b34878b61a090537ffe4735d5cc8/shm rw,nosuid,nodev,noexec,relatime shared:596 - tmpfs shm rw,size=65536k +1785 29 0:182 / /run/containerd/io.containerd.runtime.v2.task/k8s.io/ecd66ebdf9e85e447076ab5f010a9a721102b34878b61a090537ffe4735d5cc8/rootfs rw,relatime shared:604 - overlay overlay rw,lowerdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/1/fs,upperdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/33979/fs,workdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/33979/work,xino=off +1827 29 0:192 / /run/containerd/io.containerd.runtime.v2.task/k8s.io/c78d2dc899501734043224ccae3b1844ca17bde418fa1a2e6b9230b3b8bb100c/rootfs rw,relatime shared:613 - overlay overlay rw,lowerdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/33982/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/33981/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/33980/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/33978/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/33977/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/33976/fs,upperdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/33983/fs,workdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/33983/work,xino=off +1860 145 259:2 /vol1 /data/nvme0n1/kubelet/pods/a954a19c-b1eb-49f4-903a-edc75070cb01/volumes/kubernetes.io~local-volume/local-pv-2ed50413 rw,relatime shared:386 - ext4 /dev/nvme3n1 rw +2017 145 0:194 / /data/nvme0n1/kubelet/pods/a954a19c-b1eb-49f4-903a-edc75070cb01/volumes/kubernetes.io~projected/kube-api-access-pfpxr rw,relatime shared:628 - tmpfs tmpfs rw +1897 29 0:201 / /run/containerd/io.containerd.runtime.v2.task/k8s.io/2d3a2171ef379ec2abb999beaf83e6a864352c1bfd12a3b7e02839842df996ee/rootfs rw,relatime shared:636 - overlay overlay rw,lowerdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/33296/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/33295/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/33294/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/31119/fs,upperdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/33984/fs,workdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/33984/work,xino=off +1916 29 0:4 net:[4026537735] /run/netns/cni-10de9502-de86-307b-30b6-4038e734172f rw shared:644 - nsfs nsfs rw +1963 29 0:212 / /run/containerd/io.containerd.grpc.v1.cri/sandboxes/8883ff252cae98cf882bd79b528a009c36f3618a6e1e2f0de5e8df85c90dca7f/shm rw,nosuid,nodev,noexec,relatime shared:652 - tmpfs shm rw,size=65536k +1983 29 0:213 / /run/containerd/io.containerd.runtime.v2.task/k8s.io/8883ff252cae98cf882bd79b528a009c36f3618a6e1e2f0de5e8df85c90dca7f/rootfs rw,relatime shared:660 - overlay overlay rw,lowerdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/1/fs,upperdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/33985/fs,workdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/33985/work,xino=off +2001 145 0:226 / /data/nvme0n1/kubelet/pods/8faa8aef-6440-4932-8281-5350b429bbf5/volumes/kubernetes.io~downward-api/annotations rw,relatime shared:668 - tmpfs tmpfs rw +2087 145 0:227 / /data/nvme0n1/kubelet/pods/8faa8aef-6440-4932-8281-5350b429bbf5/volumes/kubernetes.io~projected/kube-api-access-6c682 rw,relatime shared:677 - tmpfs tmpfs rw +2116 29 0:4 net:[4026537808] /run/netns/cni-6c186bcd-915d-ae9f-6240-f005d7558f3c rw shared:685 - nsfs nsfs rw +2164 29 0:228 / /run/containerd/io.containerd.grpc.v1.cri/sandboxes/98d1dc3da1f767caa508aeb7a13ba91eb29f93e6afcc6d9374c477956a587c9c/shm rw,nosuid,nodev,noexec,relatime shared:694 - tmpfs shm rw,size=65536k +2184 29 0:229 / /run/containerd/io.containerd.runtime.v2.task/k8s.io/98d1dc3da1f767caa508aeb7a13ba91eb29f93e6afcc6d9374c477956a587c9c/rootfs rw,relatime shared:703 - overlay overlay rw,lowerdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/1/fs,upperdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/33991/fs,workdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/33991/work,xino=off +2205 145 0:240 / /data/nvme0n1/kubelet/pods/661d95b2-0ff0-4ae4-b154-2a8cbc436538/volumes/kubernetes.io~secret/tls-assets rw,relatime shared:710 - tmpfs tmpfs rw +2259 145 0:241 / /data/nvme0n1/kubelet/pods/661d95b2-0ff0-4ae4-b154-2a8cbc436538/volumes/kubernetes.io~projected/kube-api-access-c494f rw,relatime shared:717 - tmpfs tmpfs rw +2280 29 0:4 net:[4026537950] /run/netns/cni-0a5d3c52-1cc9-1d7b-2189-30d737f65e6c rw shared:724 - nsfs nsfs rw +2358 29 0:242 / /run/containerd/io.containerd.grpc.v1.cri/sandboxes/d39f5fd65ac2df941fba6dd4e7cffb25a612111082711e6afb44aa16171c86be/shm rw,nosuid,nodev,noexec,relatime shared:755 - tmpfs shm rw,size=65536k +2395 29 0:243 / /run/containerd/io.containerd.runtime.v2.task/k8s.io/d39f5fd65ac2df941fba6dd4e7cffb25a612111082711e6afb44aa16171c86be/rootfs rw,relatime shared:766 - overlay overlay rw,lowerdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/1/fs,upperdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/33992/fs,workdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/33992/work,xino=off +2442 29 0:265 / /run/containerd/io.containerd.runtime.v2.task/k8s.io/9a265bdcf7c04c2d56cd82b5126f17a18dd34d557ffea27ba74f1006dcffd510/rootfs rw,relatime shared:775 - overlay overlay rw,lowerdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/33997/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/33996/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/33995/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/33994/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/33993/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/33990/fs,upperdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/33998/fs,workdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/33998/work,xino=off +2476 29 0:266 / /run/containerd/io.containerd.runtime.v2.task/k8s.io/f5bb1b65d7857de4c14b191dec87bb583ea42d0f7336ed9c622989446d52a7e3/rootfs rw,relatime shared:789 - overlay overlay rw,lowerdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/32889/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/32888/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/31105/fs,upperdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/33999/fs,workdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/33999/work,xino=off +2538 29 0:283 / /run/containerd/io.containerd.runtime.v2.task/k8s.io/9491e655ded1c73a4d985437535e88469725b90ad8e26d64cd854eead7ff5aa3/rootfs rw,relatime shared:796 - overlay overlay rw,lowerdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/31241/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/31240/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/31239/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/31238/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/31237/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/31236/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/31235/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/31234/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/31233/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/31232/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/30975/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/30974/fs,upperdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/34006/fs,workdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/34006/work,xino=off +2576 29 0:291 / /run/containerd/io.containerd.runtime.v2.task/k8s.io/cf0caadef274fa509bb964c214abc9329509c2372f39107d334a36312fd0a7e9/rootfs rw,relatime shared:804 - overlay overlay rw,lowerdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/31245/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/31244/fs,upperdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/34007/fs,workdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/34007/work,xino=off +2634 29 0:299 / /run/containerd/io.containerd.runtime.v2.task/k8s.io/e54b94b04ed316c75dea0c75e980aee377dc9d44f025017021c14dd147346c80/rootfs rw,relatime shared:811 - overlay overlay rw,lowerdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/31258/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/31257/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/31256/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/31255/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/31254/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/31253/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/31252/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/31251/fs,upperdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/34008/fs,workdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/34008/work,xino=off +2655 145 0:307 / /data/nvme0n1/kubelet/pods/54dd0c3f-051c-48f7-840c-1842661c69a3/volumes/kubernetes.io~downward-api/annotations rw,relatime shared:820 - tmpfs tmpfs rw +2725 145 259:3 /vol1 /data/nvme0n1/kubelet/pods/54dd0c3f-051c-48f7-840c-1842661c69a3/volumes/kubernetes.io~local-volume/local-pv-2928e757 rw,relatime shared:348 - ext4 /dev/nvme2n1 rw +2747 145 0:308 / /data/nvme0n1/kubelet/pods/54dd0c3f-051c-48f7-840c-1842661c69a3/volumes/kubernetes.io~projected/kube-api-access-clj59 rw,relatime shared:841 - tmpfs tmpfs rw +2764 29 0:4 net:[4026538031] /run/netns/cni-c14eeef7-4134-bb74-f1cc-007ecd2a97b5 rw shared:856 - nsfs nsfs rw +2876 29 0:309 / /run/containerd/io.containerd.grpc.v1.cri/sandboxes/4f0074f05034c726a79d9c779d6f1d0d1f209a6ece89524d25b2a0b416d87ee9/shm rw,nosuid,nodev,noexec,relatime shared:871 - tmpfs shm rw,size=65536k +2996 29 0:310 / /run/containerd/io.containerd.runtime.v2.task/k8s.io/4f0074f05034c726a79d9c779d6f1d0d1f209a6ece89524d25b2a0b416d87ee9/rootfs rw,relatime shared:878 - overlay overlay rw,lowerdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/1/fs,upperdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/34011/fs,workdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/34011/work,xino=off +3036 29 0:320 / /run/containerd/io.containerd.runtime.v2.task/k8s.io/d9bb40166d27212a2449a8cf85810a108c11aff3f9bde7d51b939ecd9acb4cf3/rootfs rw,relatime shared:885 - overlay overlay rw,lowerdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/33296/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/33295/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/33294/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/31119/fs,upperdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/34012/fs,workdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/34012/work,xino=off +3088 145 0:329 / /data/nvme0n1/kubelet/pods/e2df7121-3621-4707-b7e9-6b99c3ce58b8/volumes/kubernetes.io~downward-api/annotations rw,relatime shared:893 - tmpfs tmpfs rw +3120 145 259:0 /vol1 /data/nvme0n1/kubelet/pods/e2df7121-3621-4707-b7e9-6b99c3ce58b8/volumes/kubernetes.io~local-volume/local-pv-b1fc6e56 rw,relatime shared:424 - ext4 /dev/nvme1n1 rw +3141 145 0:330 / /data/nvme0n1/kubelet/pods/e2df7121-3621-4707-b7e9-6b99c3ce58b8/volumes/kubernetes.io~projected/kube-api-access-jpzvq rw,relatime shared:906 - tmpfs tmpfs rw +3164 29 0:4 net:[4026538106] /run/netns/cni-bebf9810-8b52-9b5d-42cf-8760067f6b0b rw shared:915 - nsfs nsfs rw +3196 29 0:331 / /run/containerd/io.containerd.grpc.v1.cri/sandboxes/d585588652781fc3d764c036575ca7680416cdd1c7725ec9819897ab9066ede9/shm rw,nosuid,nodev,noexec,relatime shared:922 - tmpfs shm rw,size=65536k +3231 29 0:332 / /run/containerd/io.containerd.runtime.v2.task/k8s.io/d585588652781fc3d764c036575ca7680416cdd1c7725ec9819897ab9066ede9/rootfs rw,relatime shared:930 - overlay overlay rw,lowerdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/1/fs,upperdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/34015/fs,workdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/34015/work,xino=off +3303 29 0:342 / /run/containerd/io.containerd.runtime.v2.task/k8s.io/a503134d5c6b832363be3926d09d6f641dfae06dd2108af08d80449368c4b04e/rootfs rw,relatime shared:937 - overlay overlay rw,lowerdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/33296/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/33295/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/33294/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/31119/fs,upperdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/34016/fs,workdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/34016/work,xino=off +1154 145 0:146 / /data/nvme0n1/kubelet/pods/4d283eb8-d2d7-490c-bb71-2be40dab9450/volumes/kubernetes.io~projected/kube-api-access-t8zfb rw,relatime shared:403 - tmpfs tmpfs rw +1299 29 0:4 net:[4026537510] /run/netns/cni-2f2e413a-7585-1a30-7cb1-ac92569fed15 rw shared:412 - nsfs nsfs rw +1329 29 0:147 / /run/containerd/io.containerd.grpc.v1.cri/sandboxes/3e4cd92bbd312e86fb59ebb9ce183220e0917434d86afb245c1715ffc5460bf8/shm rw,nosuid,nodev,noexec,relatime shared:421 - tmpfs shm rw,size=65536k +1355 29 0:148 / /run/containerd/io.containerd.runtime.v2.task/k8s.io/3e4cd92bbd312e86fb59ebb9ce183220e0917434d86afb245c1715ffc5460bf8/rootfs rw,relatime shared:437 - overlay overlay rw,lowerdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/1/fs,upperdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/34017/fs,workdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/34017/work,xino=off +1457 145 259:1 /kubelet/pods/4d283eb8-d2d7-490c-bb71-2be40dab9450/volumes/kubernetes.io~configmap/config/..2023_04_11_00_48_09.037538042/fluent-bit.conf /data/nvme0n1/kubelet/pods/4d283eb8-d2d7-490c-bb71-2be40dab9450/volume-subpaths/config/fluent-bit/0 rw,relatime shared:81 - ext4 /dev/nvme0n1 rw +1573 145 259:1 /kubelet/pods/4d283eb8-d2d7-490c-bb71-2be40dab9450/volumes/kubernetes.io~configmap/config/..2023_04_11_00_48_09.037538042/custom_parsers.conf /data/nvme0n1/kubelet/pods/4d283eb8-d2d7-490c-bb71-2be40dab9450/volume-subpaths/config/fluent-bit/1 rw,relatime shared:81 - ext4 /dev/nvme0n1 rw +1628 29 0:159 / /run/containerd/io.containerd.runtime.v2.task/k8s.io/5eec42b4a282163996409e4c7dbad906cb9649ef63f0e3cf16613c41e8c81909/rootfs rw,relatime shared:565 - overlay overlay rw,lowerdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/31166/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/31165/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/31164/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/31163/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/31162/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/31161/fs,upperdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/34018/fs,workdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/34018/work,xino=off +` ) diff --git a/pkg/core/basic_cluster.go b/pkg/core/basic_cluster.go index 1855239e990..1c8902ca8cb 100644 --- a/pkg/core/basic_cluster.go +++ b/pkg/core/basic_cluster.go @@ -175,10 +175,10 @@ func (bc *BasicCluster) ResetStoreLimit(storeID uint64, limitType storelimit.Typ // UpdateStoreStatus updates the information of the store. func (bc *BasicCluster) UpdateStoreStatus(storeID uint64) { - leaderCount, regionCount, witnessCount, pendingPeerCount, leaderRegionSize, regionSize := bc.RegionsInfo.GetStoreStats(storeID) + leaderCount, regionCount, witnessCount, learnerCount, pendingPeerCount, leaderRegionSize, regionSize := bc.RegionsInfo.GetStoreStats(storeID) bc.Stores.mu.Lock() defer bc.Stores.mu.Unlock() - bc.Stores.UpdateStoreStatus(storeID, leaderCount, regionCount, pendingPeerCount, leaderRegionSize, regionSize, witnessCount) + bc.Stores.UpdateStoreStatus(storeID, leaderCount, regionCount, witnessCount, learnerCount, pendingPeerCount, leaderRegionSize, regionSize) } // PutStore put a store. @@ -248,7 +248,7 @@ func (bc *BasicCluster) GetStoresWriteRate() (storeIDs []uint64, bytesRates, key // RegionSetInformer provides access to a shared informer of regions. type RegionSetInformer interface { - GetRegionCount() int + GetTotalRegionCount() int RandFollowerRegions(storeID uint64, ranges []KeyRange) []*RegionInfo RandLeaderRegions(storeID uint64, ranges []KeyRange) []*RegionInfo RandLearnerRegions(storeID uint64, ranges []KeyRange) []*RegionInfo diff --git a/pkg/core/constant/kind.go b/pkg/core/constant/kind.go index 20e9675775e..d8059a306e7 100644 --- a/pkg/core/constant/kind.go +++ b/pkg/core/constant/kind.go @@ -27,6 +27,21 @@ const ( PriorityLevelLen ) +func (p PriorityLevel) String() string { + switch p { + case Low: + return "low" + case Medium: + return "medium" + case High: + return "high" + case Urgent: + return "urgent" + default: + return "unknown" + } +} + // ScheduleKind distinguishes resources and schedule policy. type ScheduleKind struct { Resource ResourceKind @@ -51,6 +66,9 @@ const ( RegionKind // WitnessKind indicates the witness kind resource WitnessKind + + // ResourceKindLen represents the ResourceKind count + ResourceKindLen ) func (k ResourceKind) String() string { diff --git a/pkg/core/rangetree/range_tree_test.go b/pkg/core/rangetree/range_tree_test.go index 6bbd6860989..29845cf0bca 100644 --- a/pkg/core/rangetree/range_tree_test.go +++ b/pkg/core/rangetree/range_tree_test.go @@ -100,7 +100,7 @@ func TestRingPutItem(t *testing.T) { re.Len(bucketTree.GetOverlaps(newSimpleBucketItem([]byte("010"), []byte("110"))), 2) re.Empty(bucketTree.GetOverlaps(newSimpleBucketItem([]byte("200"), []byte("300")))) - // test1： insert one key range, the old overlaps will retain like split buckets. + // test1: insert one key range, the old overlaps will retain like split buckets. // key range: [002,010],[010,090],[090,100],[100,200] bucketTree.Update(newSimpleBucketItem([]byte("010"), []byte("090"))) re.Equal(4, bucketTree.Len()) diff --git a/pkg/core/region.go b/pkg/core/region.go index f661764e914..450fab499e6 100644 --- a/pkg/core/region.go +++ b/pkg/core/region.go @@ -64,6 +64,7 @@ type RegionInfo struct { readBytes uint64 readKeys uint64 approximateSize int64 + approximateKvSize int64 approximateKeys int64 interval *pdpb.TimeInterval replicationStatus *replication_modepb.RegionReplicationStatus @@ -151,6 +152,7 @@ func RegionFromHeartbeat(heartbeat *pdpb.RegionHeartbeatRequest, opts ...RegionC if heartbeat.GetApproximateSize() > 0 && regionSize < EmptyRegionApproximateSize { regionSize = EmptyRegionApproximateSize } + regionKvSize := heartbeat.GetApproximateKvSize() / units.MiB region := &RegionInfo{ term: heartbeat.GetTerm(), @@ -164,6 +166,7 @@ func RegionFromHeartbeat(heartbeat *pdpb.RegionHeartbeatRequest, opts ...RegionC readBytes: heartbeat.GetBytesRead(), readKeys: heartbeat.GetKeysRead(), approximateSize: int64(regionSize), + approximateKvSize: int64(regionKvSize), approximateKeys: int64(heartbeat.GetApproximateKeys()), interval: heartbeat.GetInterval(), replicationStatus: heartbeat.GetReplicationStatus(), @@ -230,6 +233,7 @@ func (r *RegionInfo) Clone(opts ...RegionCreateOption) *RegionInfo { readBytes: r.readBytes, readKeys: r.readKeys, approximateSize: r.approximateSize, + approximateKvSize: r.approximateKvSize, approximateKeys: r.approximateKeys, interval: typeutil.DeepClone(r.interval, TimeIntervalFactory), replicationStatus: r.replicationStatus, @@ -520,6 +524,11 @@ func (r *RegionInfo) GetStorePeerApproximateKeys(storeID uint64) int64 { return r.approximateKeys } +// GetApproximateKvSize returns the approximate kv size of the region. +func (r *RegionInfo) GetApproximateKvSize() int64 { + return r.approximateKvSize +} + // GetApproximateKeys returns the approximate keys of the region. func (r *RegionInfo) GetApproximateKeys() int64 { return r.approximateKeys @@ -1271,15 +1280,15 @@ func (r *RegionsInfo) GetMetaRegions() []*metapb.Region { } // GetStoreStats returns the store stats. -func (r *RegionsInfo) GetStoreStats(storeID uint64) (leader, region, witness, pending int, leaderSize, regionSize int64) { +func (r *RegionsInfo) GetStoreStats(storeID uint64) (leader, region, witness, learner, pending int, leaderSize, regionSize int64) { r.st.RLock() defer r.st.RUnlock() return r.leaders[storeID].length(), r.getStoreRegionCountLocked(storeID), r.witnesses[storeID].length(), - r.pendingPeers[storeID].length(), r.leaders[storeID].TotalSize(), r.getStoreRegionSizeLocked(storeID) + r.learners[storeID].length(), r.pendingPeers[storeID].length(), r.leaders[storeID].TotalSize(), r.getStoreRegionSizeLocked(storeID) } -// GetRegionCount gets the total count of RegionInfo of regionMap -func (r *RegionsInfo) GetRegionCount() int { +// GetTotalRegionCount gets the total count of RegionInfo of regionMap +func (r *RegionsInfo) GetTotalRegionCount() int { r.t.RLock() defer r.t.RUnlock() return len(r.regions) @@ -1473,8 +1482,8 @@ func (r *RegionInfo) GetWriteLoads() []float64 { } } -// GetRangeCount returns the number of regions that overlap with the range [startKey, endKey). -func (r *RegionsInfo) GetRangeCount(startKey, endKey []byte) int { +// GetRegionCount returns the number of regions that overlap with the range [startKey, endKey). +func (r *RegionsInfo) GetRegionCount(startKey, endKey []byte) int { r.t.RLock() defer r.t.RUnlock() start := ®ionItem{&RegionInfo{meta: &metapb.Region{StartKey: startKey}}} @@ -1496,9 +1505,9 @@ func (r *RegionsInfo) GetRangeCount(startKey, endKey []byte) int { return endIndex - startIndex + 1 } -// ScanRange scans regions intersecting [start key, end key), returns at most +// ScanRegions scans regions intersecting [start key, end key), returns at most // `limit` regions. limit <= 0 means no limit. -func (r *RegionsInfo) ScanRange(startKey, endKey []byte, limit int) []*RegionInfo { +func (r *RegionsInfo) ScanRegions(startKey, endKey []byte, limit int) []*RegionInfo { r.t.RLock() defer r.t.RUnlock() var res []*RegionInfo @@ -1515,9 +1524,9 @@ func (r *RegionsInfo) ScanRange(startKey, endKey []byte, limit int) []*RegionInf return res } -// ScanRangeWithIterator scans from the first region containing or behind start key, +// ScanRegionWithIterator scans from the first region containing or behind start key, // until iterator returns false. -func (r *RegionsInfo) ScanRangeWithIterator(startKey []byte, iterator func(region *RegionInfo) bool) { +func (r *RegionsInfo) ScanRegionWithIterator(startKey []byte, iterator func(region *RegionInfo) bool) { r.t.RLock() defer r.t.RUnlock() r.tree.scanRange(startKey, iterator) diff --git a/pkg/core/region_option.go b/pkg/core/region_option.go index 92820e3f549..ba46fab9420 100644 --- a/pkg/core/region_option.go +++ b/pkg/core/region_option.go @@ -274,6 +274,13 @@ func SetApproximateSize(v int64) RegionCreateOption { } } +// SetApproximateKvSize sets the approximate size for the region. +func SetApproximateKvSize(v int64) RegionCreateOption { + return func(region *RegionInfo) { + region.approximateKvSize = v + } +} + // SetApproximateKeys sets the approximate keys for the region. func SetApproximateKeys(v int64) RegionCreateOption { return func(region *RegionInfo) { diff --git a/pkg/core/region_tree.go b/pkg/core/region_tree.go index 7e84e5e07cc..db1c8c28fc7 100644 --- a/pkg/core/region_tree.go +++ b/pkg/core/region_tree.go @@ -16,7 +16,6 @@ package core import ( "bytes" "math/rand" - "time" "github.com/pingcap/kvproto/pkg/metapb" "github.com/pingcap/log" @@ -343,7 +342,3 @@ func (t *regionTree) TotalWriteRate() (bytesRate, keysRate float64) { } return t.totalWriteBytesRate, t.totalWriteKeysRate } - -func init() { - rand.New(rand.NewSource(time.Now().UnixNano())) -} diff --git a/pkg/core/store.go b/pkg/core/store.go index 18e857f5d08..1d3362cac0e 100644 --- a/pkg/core/store.go +++ b/pkg/core/store.go @@ -55,6 +55,7 @@ type StoreInfo struct { slowTrendEvicted bool // this store has been evicted as a slow store by trend, should not transfer leader to it leaderCount int regionCount int + learnerCount int witnessCount int leaderSize int64 regionSize int64 @@ -106,11 +107,27 @@ func (s *StoreInfo) Clone(opts ...StoreCreateOption) *StoreInfo { store := *s store.meta = typeutil.DeepClone(s.meta, StoreFactory) for _, opt := range opts { - opt(&store) + if opt != nil { + opt(&store) + } } return &store } +// LimitVersion returns the limit version of the store. +func (s *StoreInfo) LimitVersion() string { + s.mu.RLock() + defer s.mu.RUnlock() + return s.limiter.Version() +} + +// Feedback is used to update the store's limit. +func (s *StoreInfo) Feedback(e float64) { + if limit := s.limiter; limit != nil { + limit.Feedback(e) + } +} + // ShallowClone creates a copy of current StoreInfo, but not clone 'meta'. func (s *StoreInfo) ShallowClone(opts ...StoreCreateOption) *StoreInfo { store := *s @@ -137,10 +154,10 @@ func (s *StoreInfo) IsEvictedAsSlowTrend() bool { } // IsAvailable returns if the store bucket of limitation is available -func (s *StoreInfo) IsAvailable(limitType storelimit.Type) bool { +func (s *StoreInfo) IsAvailable(limitType storelimit.Type, level constant.PriorityLevel) bool { s.mu.RLock() defer s.mu.RUnlock() - return s.limiter.Available(storelimit.RegionInfluence[limitType], limitType, constant.Low) + return s.limiter.Available(storelimit.RegionInfluence[limitType], limitType, level) } // IsTiFlash returns true if the store is tiflash. @@ -259,6 +276,11 @@ func (s *StoreInfo) GetRegionCount() int { return s.regionCount } +// GetLearnerCount returns the learner count of the store. +func (s *StoreInfo) GetLearnerCount() int { + return s.learnerCount +} + // GetWitnessCount returns the witness count of the store. func (s *StoreInfo) GetWitnessCount() int { return s.witnessCount @@ -441,7 +463,8 @@ func (s *StoreInfo) IsLowSpace(lowSpaceRatio float64) bool { } // See https://github.com/tikv/pd/issues/3444 and https://github.com/tikv/pd/issues/5391 // TODO: we need find a better way to get the init region number when starting a new cluster. - if s.regionCount < InitClusterRegionThreshold && s.GetAvailable() > initialMinSpace { + // We don't need to consider the store as low space when the capacity is 0. + if s.regionCount < InitClusterRegionThreshold && s.GetAvailable() > initialMinSpace || s.GetCapacity() == 0 { return false } return s.AvailableRatio() < 1-lowSpaceRatio @@ -772,11 +795,12 @@ func (s *StoresInfo) SetRegionSize(storeID uint64, regionSize int64) { } // UpdateStoreStatus updates the information of the store. -func (s *StoresInfo) UpdateStoreStatus(storeID uint64, leaderCount int, regionCount int, pendingPeerCount int, leaderSize int64, regionSize int64, witnessCount int) { +func (s *StoresInfo) UpdateStoreStatus(storeID uint64, leaderCount, regionCount, witnessCount, learnerCount, pendingPeerCount int, leaderSize int64, regionSize int64) { if store, ok := s.stores[storeID]; ok { newStore := store.ShallowClone(SetLeaderCount(leaderCount), SetRegionCount(regionCount), SetWitnessCount(witnessCount), + SetLearnerCount(learnerCount), SetPendingPeerCount(pendingPeerCount), SetLeaderSize(leaderSize), SetRegionSize(regionSize)) diff --git a/pkg/core/store_option.go b/pkg/core/store_option.go index e663fb5d37e..a1e15fd056d 100644 --- a/pkg/core/store_option.go +++ b/pkg/core/store_option.go @@ -165,6 +165,13 @@ func SetRegionCount(regionCount int) StoreCreateOption { } } +// SetLearnerCount sets the learner count for the store. +func SetLearnerCount(learnerCount int) StoreCreateOption { + return func(store *StoreInfo) { + store.learnerCount = learnerCount + } +} + // SetWitnessCount sets the witness count for the store. func SetWitnessCount(witnessCount int) StoreCreateOption { return func(store *StoreInfo) { @@ -259,6 +266,15 @@ func ResetStoreLimit(limitType storelimit.Type, ratePerSec ...float64) StoreCrea } } +// SetStoreLimit set the store for a store, it may switch the store limit mode. +func SetStoreLimit(limit storelimit.StoreLimit) StoreCreateOption { + return func(store *StoreInfo) { + store.mu.Lock() + defer store.mu.Unlock() + store.limiter = limit + } +} + // SetLastAwakenTime sets last awaken time for the store. func SetLastAwakenTime(lastAwaken time.Time) StoreCreateOption { return func(store *StoreInfo) { diff --git a/pkg/core/store_test.go b/pkg/core/store_test.go index 33451a22935..be0fd0f9418 100644 --- a/pkg/core/store_test.go +++ b/pkg/core/store_test.go @@ -141,6 +141,8 @@ func TestLowSpaceRatio(t *testing.T) { re.True(store.IsLowSpace(0.8)) store.rawStats.Available = store.rawStats.Capacity >> 2 re.False(store.IsLowSpace(0.8)) + store.rawStats.Capacity = 0 + re.False(store.IsLowSpace(0.8)) } func TestLowSpaceScoreV2(t *testing.T) { @@ -172,7 +174,7 @@ func TestLowSpaceScoreV2(t *testing.T) { bigger: newStoreInfoWithAvailable(1, 10*units.GiB, 100*units.GiB, 1.5), small: newStoreInfoWithAvailable(2, 10*units.GiB, 100*units.GiB, 1.4), }, { - // store1 and store2 has same capacity and regionSize（40g) + // store1 and store2 has same capacity and regionSize (40g) // but store1 has less available space size bigger: newStoreInfoWithAvailable(1, 60*units.GiB, 100*units.GiB, 1), small: newStoreInfoWithAvailable(2, 80*units.GiB, 100*units.GiB, 2), diff --git a/pkg/core/storelimit/limit.go b/pkg/core/storelimit/limit.go index b5a3312ff56..b598daf6db7 100644 --- a/pkg/core/storelimit/limit.go +++ b/pkg/core/storelimit/limit.go @@ -32,7 +32,22 @@ const ( storeLimitTypeLen ) +const ( + // VersionV1 represents the rate limit version of the store limit + VersionV1 = "v1" + // VersionV2 represents the sliding window version of the store limit + VersionV2 = "v2" +) + // StoreLimit is an interface to control the operator rate of store +// TODO: add a method to control the rate of store +// the normal control flow is: +// 1. check the store limit with Available in checker or scheduler. +// 2. check the store limit with Available in operator controller again. +// the different between 1 and 2 is that 1 maybe not use the operator level. +// 3. take the cost of operator with Take in operator controller. +// 4. ack will put back the cost into the limit for the next waiting operator after the operator is finished. +// the cost is the operator influence, so the influence should be same in the life of the operator. type StoreLimit interface { // Available returns true if the store can accept the operator Available(cost int64, typ Type, level constant.PriorityLevel) bool @@ -40,4 +55,11 @@ type StoreLimit interface { Take(count int64, typ Type, level constant.PriorityLevel) bool // Reset resets the store limit Reset(rate float64, typ Type) + // Feedback update limit capacity by auto-tuning. + Feedback(e float64) + // Ack put back the cost into the limit for the next waiting operator after the operator is finished. + // only snapshot type can use this method. + Ack(cost int64, typ Type) + // Version returns the version of the store limit + Version() string } diff --git a/pkg/core/storelimit/limit_test.go b/pkg/core/storelimit/limit_test.go index 7ff321776ed..6f57c01eccb 100644 --- a/pkg/core/storelimit/limit_test.go +++ b/pkg/core/storelimit/limit_test.go @@ -15,7 +15,12 @@ package storelimit import ( + "container/list" + "context" + "math/rand" + "sync/atomic" "testing" + "time" "github.com/stretchr/testify/require" "github.com/tikv/pd/pkg/core/constant" @@ -24,7 +29,8 @@ import ( func TestStoreLimit(t *testing.T) { re := require.New(t) rate := int64(15) - limit := NewStoreRateLimit(float64(rate)) + limit := NewStoreRateLimit(float64(rate)).(*StoreRateLimit) + re.Equal(limit.Rate(AddPeer), float64(15)) re.True(limit.Available(influence*rate, AddPeer, constant.Low)) re.True(limit.Take(influence*rate, AddPeer, constant.Low)) re.False(limit.Take(influence, AddPeer, constant.Low)) @@ -41,8 +47,8 @@ func TestStoreLimit(t *testing.T) { func TestSlidingWindow(t *testing.T) { t.Parallel() re := require.New(t) - capacity := int64(10) - s := NewSlidingWindows(float64(capacity)) + capacity := int64(defaultWindowSize) + s := NewSlidingWindows() re.Len(s.windows, int(constant.PriorityLevelLen)) // capacity:[10, 10, 10, 10] for i, v := range s.windows { @@ -56,28 +62,28 @@ func TestSlidingWindow(t *testing.T) { re.True(s.Available(capacity, SendSnapshot, constant.Low)) re.True(s.Take(capacity, SendSnapshot, constant.Low)) re.False(s.Available(capacity, SendSnapshot, constant.Low)) - s.Ack(capacity) + s.Ack(capacity, SendSnapshot) re.True(s.Available(capacity, SendSnapshot, constant.Low)) // case 1: it will occupy the normal window size not the core.High window. re.True(s.Take(capacity, SendSnapshot, constant.High)) - re.EqualValues(capacity, s.GetUsed()) + re.EqualValues([]int64{capacity, 0, 0, 0}, s.GetUsed()) re.EqualValues(0, s.windows[constant.High].getUsed()) - s.Ack(capacity) - re.EqualValues(s.GetUsed(), 0) + s.Ack(capacity, SendSnapshot) + re.EqualValues([]int64{0, 0, 0, 0}, s.GetUsed()) // case 2: it will occupy the core.High window size if the normal window is full. capacity = 2000 - s.Reset(float64(capacity), SendSnapshot) + s.set(float64(capacity), SendSnapshot) re.True(s.Take(capacity-minSnapSize, SendSnapshot, constant.Low)) re.True(s.Take(capacity-minSnapSize, SendSnapshot, constant.Low)) re.False(s.Take(capacity, SendSnapshot, constant.Low)) re.True(s.Take(capacity-minSnapSize, SendSnapshot, constant.Medium)) re.False(s.Take(capacity-minSnapSize, SendSnapshot, constant.Medium)) - re.EqualValues(s.GetUsed(), capacity+capacity+capacity-minSnapSize*3) - s.Ack(capacity - minSnapSize) - s.Ack(capacity - minSnapSize) - re.Equal(s.GetUsed(), capacity-minSnapSize) + re.EqualValues([]int64{capacity + capacity - minSnapSize*2, capacity - minSnapSize, 0, 0}, s.GetUsed()) + s.Ack(capacity-minSnapSize, SendSnapshot) + s.Ack(capacity-minSnapSize, SendSnapshot) + re.Equal([]int64{capacity - minSnapSize, 0, 0, 0}, s.GetUsed()) // case 3: skip the type is not the SendSnapshot for i := 0; i < 10; i++ { @@ -108,3 +114,84 @@ func TestWindow(t *testing.T) { re.EqualValues(s.ack(minSnapSize*2), minSnapSize) re.EqualValues(s.getUsed(), 0) } + +func TestFeedback(t *testing.T) { + s := NewSlidingWindows() + re := require.New(t) + type SnapshotStats struct { + total int64 + remaining int64 + size int64 + start int64 + } + // region size is 10GB, snapshot write limit is 100MB/s and the snapshot concurrency is 3. + // the best strategy is that the tikv executing queue equals the wait. + const regionSize, limit, wait = int64(10000), int64(100), int64(4) + var iter atomic.Int32 + iter.Store(100) + ops := make(chan int64, 10) + ctx, cancel := context.WithCancel(context.Background()) + + // generate the operator + go func() { + for { + if s.Available(regionSize, SendSnapshot, constant.Low) && iter.Load() > 0 { + iter.Add(-1) + size := regionSize - rand.Int63n(regionSize/10) + s.Take(size, SendSnapshot, constant.Low) + ops <- size + } + if iter.Load() == 0 { + cancel() + return + } + } + }() + + // receive the operator + queue := list.List{} + interval := time.Microsecond * 100 + ticker := time.NewTicker(interval) + defer ticker.Stop() + + // tick is the time that the snapshot has been executed. + tick := int64(0) + for { + select { + case op := <-ops: + stats := &SnapshotStats{ + total: op / limit, + remaining: op, + size: op, + start: tick, + } + queue.PushBack(stats) + case <-ctx.Done(): + return + case <-ticker.C: + tick++ + first := queue.Front() + if first == nil { + continue + } + stats := first.Value.(*SnapshotStats) + if stats.remaining > 0 { + stats.remaining -= limit + continue + } + cost := tick - stats.start + exec := stats.total + if exec < 5 { + exec = 5 + } + err := exec*wait - cost + queue.Remove(first) + s.Feedback(float64(err)) + if iter.Load() < 5 { + re.Greater(float64(s.GetCap()), float64(regionSize*(wait-2))) + re.Less(float64(s.GetCap()), float64(regionSize*wait)) + } + s.Ack(stats.size, SendSnapshot) + } + } +} diff --git a/pkg/core/storelimit/sliding_window.go b/pkg/core/storelimit/sliding_window.go index 34b39c36253..0a70eb548d0 100644 --- a/pkg/core/storelimit/sliding_window.go +++ b/pkg/core/storelimit/sliding_window.go @@ -22,36 +22,66 @@ import ( const ( // minSnapSize is the min value to check the windows has enough size. minSnapSize = 10 + // defaultWindowSize is the default window size. + defaultWindowSize = 100 + + defaultProportion = 20 + defaultIntegral = 10 ) +var _ StoreLimit = &SlidingWindows{} + // SlidingWindows is a multi sliding windows type SlidingWindows struct { mu syncutil.RWMutex windows []*window + lastSum float64 } // NewSlidingWindows is the construct of SlidingWindows. -func NewSlidingWindows(cap float64) *SlidingWindows { - if cap < 0 { - cap = minSnapSize - } +func NewSlidingWindows() *SlidingWindows { windows := make([]*window, constant.PriorityLevelLen) for i := 0; i < int(constant.PriorityLevelLen); i++ { - windows[i] = newWindow(int64(cap) >> i) + windows[i] = newWindow(int64(defaultWindowSize) >> i) } return &SlidingWindows{ windows: windows, } } -// Reset resets the capacity of the sliding windows. -// It doesn't clear all the used, only set the capacity. -func (s *SlidingWindows) Reset(cap float64, typ Type) { +// Version returns v2 +func (s *SlidingWindows) Version() string { + return VersionV2 +} + +// Feedback is used to update the capacity of the sliding windows. +func (s *SlidingWindows) Feedback(e float64) { + s.mu.Lock() + defer s.mu.Unlock() + // If the limiter is available, we don't need to update the capacity. + if s.windows[constant.Low].available() { + return + } + s.lastSum += e + // There are two constants to control the proportion of the sum and the current error. + // The sum of the error is used to ensure the capacity is more stable even if the error is zero. + // In the final scene, the sum of the error should be stable and the current error should be zero. + cap := defaultProportion*e + defaultIntegral*s.lastSum + // The capacity should be at least the default window size. + if cap < defaultWindowSize { + cap = defaultWindowSize + } + s.set(cap, SendSnapshot) +} + +// Reset does nothing because the capacity depends on the feedback. +func (s *SlidingWindows) Reset(_ float64, _ Type) { +} + +func (s *SlidingWindows) set(cap float64, typ Type) { if typ != SendSnapshot { return } - s.mu.Lock() - defer s.mu.Unlock() if cap < 0 { cap = minSnapSize } @@ -60,13 +90,20 @@ func (s *SlidingWindows) Reset(cap float64, typ Type) { } } +// GetCap returns the capacity of the sliding windows. +func (s *SlidingWindows) GetCap() int64 { + s.mu.RLock() + defer s.mu.RUnlock() + return s.windows[0].capacity +} + // GetUsed returns the used size in the sliding windows. -func (s *SlidingWindows) GetUsed() int64 { +func (s *SlidingWindows) GetUsed() []int64 { s.mu.RLock() defer s.mu.RUnlock() - used := int64(0) - for _, v := range s.windows { - used += v.getUsed() + used := make([]int64, len(s.windows)) + for i, v := range s.windows { + used[i] = v.getUsed() } return used } @@ -107,7 +144,10 @@ func (s *SlidingWindows) Take(token int64, typ Type, level constant.PriorityLeve // Ack indicates that some executing operator has been finished. // The order of refilling windows is from high to low. // It will refill the highest window first. -func (s *SlidingWindows) Ack(token int64) { +func (s *SlidingWindows) Ack(token int64, typ Type) { + if typ != SendSnapshot { + return + } s.mu.Lock() defer s.mu.Unlock() for i := constant.PriorityLevelLen - 1; i >= 0; i-- { diff --git a/pkg/core/storelimit/store_limit.go b/pkg/core/storelimit/store_limit.go index 824431bf54e..dc1de88e09f 100644 --- a/pkg/core/storelimit/store_limit.go +++ b/pkg/core/storelimit/store_limit.go @@ -30,14 +30,14 @@ const ( smallInfluence = 200 ) -// RegionInfluence represents the influence of a operator step, which is used by store limit. +// RegionInfluence represents the influence of an operator step, which is used by store limit. var RegionInfluence = []int64{ AddPeer: influence, RemovePeer: influence, SendSnapshot: influence, } -// SmallRegionInfluence represents the influence of a operator step +// SmallRegionInfluence represents the influence of an operator step // when the region size is smaller than smallRegionThreshold, which is used by store limit. var SmallRegionInfluence = []int64{ AddPeer: smallInfluence, @@ -81,6 +81,17 @@ func NewStoreRateLimit(ratePerSec float64) StoreLimit { } } +// Ack does nothing. +func (l *StoreRateLimit) Ack(_ int64, _ Type) {} + +// Version returns v1 +func (l *StoreRateLimit) Version() string { + return VersionV1 +} + +// Feedback does nothing. +func (l *StoreRateLimit) Feedback(_ float64) {} + // Available returns the number of available tokens. // notice that the priority level is not used. func (l *StoreRateLimit) Available(cost int64, typ Type, _ constant.PriorityLevel) bool { @@ -90,6 +101,14 @@ func (l *StoreRateLimit) Available(cost int64, typ Type, _ constant.PriorityLeve return l.limits[typ].Available(cost) } +// Rate returns the capacity of the store limit. +func (l *StoreRateLimit) Rate(typ Type) float64 { + if l.limits[typ] == nil { + return 0.0 + } + return l.limits[typ].ratePerSec +} + // Take takes count tokens from the bucket without blocking. // notice that the priority level is not used. func (l *StoreRateLimit) Take(cost int64, typ Type, _ constant.PriorityLevel) bool { diff --git a/pkg/dashboard/keyvisual/input/core.go b/pkg/dashboard/keyvisual/input/core.go index 5b273841534..3ca5f96cd81 100644 --- a/pkg/dashboard/keyvisual/input/core.go +++ b/pkg/dashboard/keyvisual/input/core.go @@ -94,7 +94,7 @@ func clusterScan(rc *core.BasicCluster) RegionsInfo { regions := make([]*core.RegionInfo, 0, limit) for { - rs := rc.ScanRange(startKey, endKey, limit) + rs := rc.ScanRegions(startKey, endKey, limit) length := len(rs) if length == 0 { break diff --git a/pkg/election/leadership.go b/pkg/election/leadership.go index 30485a863da..e6c75a01674 100644 --- a/pkg/election/leadership.go +++ b/pkg/election/leadership.go @@ -16,7 +16,9 @@ package election import ( "context" + "sync" "sync/atomic" + "time" "github.com/pingcap/failpoint" "github.com/pingcap/kvproto/pkg/pdpb" @@ -29,6 +31,11 @@ import ( "go.uber.org/zap" ) +const ( + watchLoopUnhealthyTimeout = 60 * time.Second + detectHealthyInterval = 10 * time.Second +) + // GetLeader gets the corresponding leader from etcd by given leaderPath (as the key). func GetLeader(c *clientv3.Client, leaderPath string) (*pdpb.Member, int64, error) { leader := &pdpb.Member{} @@ -54,8 +61,9 @@ type Leadership struct { leaderKey string leaderValue string - keepAliveCtx context.Context - keepAliveCancelFunc context.CancelFunc + keepAliveCtx context.Context + keepAliveCancelFunc context.CancelFunc + keepAliveCancelFuncLock sync.Mutex } // NewLeadership creates a new Leadership. @@ -117,7 +125,7 @@ func (ls *Leadership) Campaign(leaseTimeout int64, leaderData string, cmps ...cl finalCmps = append(finalCmps, clientv3.Compare(clientv3.CreateRevision(ls.leaderKey), "=", 0)) resp, err := kv.NewSlowLogTxn(ls.client). If(finalCmps...). - Then(clientv3.OpPut(ls.leaderKey, leaderData, clientv3.WithLease(newLease.ID))). + Then(clientv3.OpPut(ls.leaderKey, leaderData, clientv3.WithLease(newLease.ID.Load().(clientv3.LeaseID)))). Commit() log.Info("check campaign resp", zap.Any("resp", resp)) if err != nil { @@ -137,7 +145,9 @@ func (ls *Leadership) Keep(ctx context.Context) { if ls == nil { return } + ls.keepAliveCancelFuncLock.Lock() ls.keepAliveCtx, ls.keepAliveCancelFunc = context.WithCancel(ctx) + ls.keepAliveCancelFuncLock.Unlock() go ls.getLease().KeepAlive(ls.keepAliveCtx) } @@ -178,26 +188,86 @@ func (ls *Leadership) Watch(serverCtx context.Context, revision int64) { if ls == nil { return } + + interval := detectHealthyInterval + unhealthyTimeout := watchLoopUnhealthyTimeout + failpoint.Inject("fastTick", func() { + unhealthyTimeout = 5 * time.Second + interval = 1 * time.Second + }) + ticker := time.NewTicker(interval) + defer ticker.Stop() + lastHealthyTime := time.Now() + watcher := clientv3.NewWatcher(ls.client) defer watcher.Close() - ctx, cancel := context.WithCancel(serverCtx) - defer cancel() - // The revision is the revision of last modification on this key. - // If the revision is compacted, will meet required revision has been compacted error. - // In this case, use the compact revision to re-watch the key. + var watchChanCancel context.CancelFunc + defer func() { + if watchChanCancel != nil { + watchChanCancel() + } + }() for { failpoint.Inject("delayWatcher", nil) - rch := watcher.Watch(ctx, ls.leaderKey, clientv3.WithRev(revision)) - for wresp := range rch { + if watchChanCancel != nil { + watchChanCancel() + } + // In order to prevent a watch stream being stuck in a partitioned node, + // make sure to wrap context with "WithRequireLeader". + watchChanCtx, cancel := context.WithCancel(clientv3.WithRequireLeader(serverCtx)) + watchChanCancel = cancel + + // When etcd is not available, the watcher.Watch will block, + // so we check the etcd availability first. + if !etcdutil.IsHealthy(serverCtx, ls.client) { + if time.Since(lastHealthyTime) > unhealthyTimeout { + log.Error("the connect of leadership watcher is unhealthy", + zap.Int64("revision", revision), + zap.String("leader-key", ls.leaderKey), + zap.String("purpose", ls.purpose)) + return + } + select { + case <-serverCtx.Done(): + log.Info("server is closed, exit leader watch loop", + zap.String("leader-key", ls.leaderKey), + zap.String("purpose", ls.purpose)) + return + case <-ticker.C: + // continue to check the etcd availability + continue + } + } + + watchChan := watcher.Watch(watchChanCtx, ls.leaderKey, clientv3.WithRev(revision)) + WatchChanLoop: + select { + case <-serverCtx.Done(): + log.Info("server is closed, exit leader watch loop", + zap.String("leader-key", ls.leaderKey), + zap.String("purpose", ls.purpose)) + return + case <-ticker.C: + if !etcdutil.IsHealthy(serverCtx, ls.client) { + if time.Since(lastHealthyTime) > unhealthyTimeout { + log.Error("the connect of leadership watcher is unhealthy", + zap.Int64("revision", revision), + zap.String("leader-key", ls.leaderKey), + zap.String("purpose", ls.purpose)) + return + } + goto WatchChanLoop + } + case wresp := <-watchChan: // meet compacted error, use the compact revision. if wresp.CompactRevision != 0 { log.Warn("required revision has been compacted, use the compact revision", zap.Int64("required-revision", revision), zap.Int64("compact-revision", wresp.CompactRevision)) revision = wresp.CompactRevision - break - } - if wresp.Canceled { + lastHealthyTime = time.Now() + continue + } else if wresp.Err() != nil { // wresp.Err() contains CompactRevision not equal to 0 log.Error("leadership watcher is canceled with", zap.Int64("revision", revision), zap.String("leader-key", ls.leaderKey), @@ -209,19 +279,16 @@ func (ls *Leadership) Watch(serverCtx context.Context, revision int64) { for _, ev := range wresp.Events { if ev.Type == mvccpb.DELETE { log.Info("current leadership is deleted", + zap.Int64("revision", wresp.Header.Revision), zap.String("leader-key", ls.leaderKey), zap.String("purpose", ls.purpose)) return } } + revision = wresp.Header.Revision + 1 } - - select { - case <-ctx.Done(): - // server closed, return - return - default: - } + lastHealthyTime = time.Now() + goto WatchChanLoop // use goto to avoid to create a new watchChan } } @@ -230,8 +297,10 @@ func (ls *Leadership) Reset() { if ls == nil || ls.getLease() == nil { return } + ls.keepAliveCancelFuncLock.Lock() if ls.keepAliveCancelFunc != nil { ls.keepAliveCancelFunc() } + ls.keepAliveCancelFuncLock.Unlock() ls.getLease().Close() } diff --git a/pkg/election/leadership_test.go b/pkg/election/leadership_test.go index 2857c89a881..1021462a902 100644 --- a/pkg/election/leadership_test.go +++ b/pkg/election/leadership_test.go @@ -19,8 +19,10 @@ import ( "testing" "time" + "github.com/pingcap/failpoint" "github.com/stretchr/testify/require" "github.com/tikv/pd/pkg/utils/etcdutil" + "github.com/tikv/pd/pkg/utils/testutil" "go.etcd.io/etcd/clientv3" "go.etcd.io/etcd/embed" ) @@ -110,7 +112,7 @@ func TestLeadership(t *testing.T) { // Check the lease. lease1 := leadership1.getLease() re.NotNil(lease1) - lease2 := leadership1.getLease() + lease2 := leadership2.getLease() re.NotNil(lease2) re.True(lease1.IsExpired()) @@ -118,3 +120,87 @@ func TestLeadership(t *testing.T) { re.NoError(lease1.Close()) re.NoError(lease2.Close()) } + +func TestExitWatch(t *testing.T) { + re := require.New(t) + leaderKey := "/test_leader" + re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/election/fastTick", "return(true)")) + re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/utils/etcdutil/fastTick", "return(true)")) + // Case1: close the client before the watch loop starts + checkExitWatch(t, leaderKey, func(server *embed.Etcd, client *clientv3.Client) { + re.NoError(failpoint.Enable("github.com/tikv/pd/server/delayWatcher", `pause`)) + client.Close() + re.NoError(failpoint.Disable("github.com/tikv/pd/server/delayWatcher")) + }) + // Case2: close the client when the watch loop is running + checkExitWatch(t, leaderKey, func(server *embed.Etcd, client *clientv3.Client) { + // Wait for the watch loop to start + time.Sleep(500 * time.Millisecond) + client.Close() + }) + // Case3: delete the leader key + checkExitWatch(t, leaderKey, func(server *embed.Etcd, client *clientv3.Client) { + leaderKey := leaderKey + _, err := client.Delete(context.Background(), leaderKey) + re.NoError(err) + }) + // Case4: close the server before the watch loop starts + checkExitWatch(t, leaderKey, func(server *embed.Etcd, client *clientv3.Client) { + re.NoError(failpoint.Enable("github.com/tikv/pd/server/delayWatcher", `pause`)) + server.Close() + re.NoError(failpoint.Disable("github.com/tikv/pd/server/delayWatcher")) + }) + // Case5: close the server when the watch loop is running + checkExitWatch(t, leaderKey, func(server *embed.Etcd, client *clientv3.Client) { + // Wait for the watch loop to start + time.Sleep(500 * time.Millisecond) + server.Close() + }) + re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/election/fastTick")) + re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/utils/etcdutil/fastTick")) +} + +func checkExitWatch(t *testing.T, leaderKey string, injectFunc func(server *embed.Etcd, client *clientv3.Client)) { + re := require.New(t) + cfg := etcdutil.NewTestSingleConfig(t) + etcd, err := embed.StartEtcd(cfg) + defer func() { + etcd.Close() + }() + re.NoError(err) + + ep := cfg.LCUrls[0].String() + client1, err := clientv3.New(clientv3.Config{ + Endpoints: []string{ep}, + }) + re.NoError(err) + client2, err := clientv3.New(clientv3.Config{ + Endpoints: []string{ep}, + }) + re.NoError(err) + + <-etcd.Server.ReadyNotify() + + leadership1 := NewLeadership(client1, leaderKey, "test_leader_1") + leadership2 := NewLeadership(client2, leaderKey, "test_leader_2") + err = leadership1.Campaign(defaultLeaseTimeout, "test_leader_1") + re.NoError(err) + resp, err := client2.Get(context.Background(), leaderKey) + re.NoError(err) + done := make(chan struct{}) + go func() { + leadership2.Watch(context.Background(), resp.Header.Revision) + done <- struct{}{} + }() + + injectFunc(etcd, client2) + + testutil.Eventually(re, func() bool { + select { + case <-done: + return true + default: + return false + } + }) +} diff --git a/pkg/election/lease.go b/pkg/election/lease.go index 6df418fd697..1e3e66ddcce 100644 --- a/pkg/election/lease.go +++ b/pkg/election/lease.go @@ -22,6 +22,7 @@ import ( "github.com/pingcap/log" "github.com/tikv/pd/pkg/errs" "github.com/tikv/pd/pkg/utils/etcdutil" + "github.com/tikv/pd/pkg/utils/logutil" "github.com/tikv/pd/pkg/utils/typeutil" "go.etcd.io/etcd/clientv3" "go.uber.org/zap" @@ -41,7 +42,7 @@ type lease struct { // etcd client and lease client *clientv3.Client lease clientv3.Lease - ID clientv3.LeaseID + ID atomic.Value // store as clientv3.LeaseID // leaseTimeout and expireTime are used to control the lease's lifetime leaseTimeout time.Duration expireTime atomic.Value @@ -63,7 +64,7 @@ func (l *lease) Grant(leaseTimeout int64) error { log.Warn("lease grants too slow", zap.Duration("cost", cost), zap.String("purpose", l.Purpose)) } log.Info("lease granted", zap.Int64("lease-id", int64(leaseResp.ID)), zap.Int64("lease-timeout", leaseTimeout), zap.String("purpose", l.Purpose)) - l.ID = leaseResp.ID + l.ID.Store(leaseResp.ID) l.leaseTimeout = time.Duration(leaseTimeout) * time.Second l.expireTime.Store(start.Add(time.Duration(leaseResp.TTL) * time.Second)) return nil @@ -79,7 +80,11 @@ func (l *lease) Close() error { // Try to revoke lease to make subsequent elections faster. ctx, cancel := context.WithTimeout(l.client.Ctx(), revokeLeaseTimeout) defer cancel() - l.lease.Revoke(ctx, l.ID) + var leaseID clientv3.LeaseID + if l.ID.Load() != nil { + leaseID = l.ID.Load().(clientv3.LeaseID) + } + l.lease.Revoke(ctx, leaseID) return l.lease.Close() } @@ -94,6 +99,8 @@ func (l *lease) IsExpired() bool { // KeepAlive auto renews the lease and update expireTime. func (l *lease) KeepAlive(ctx context.Context) { + defer logutil.LogPanic() + if l == nil { return } @@ -102,6 +109,8 @@ func (l *lease) KeepAlive(ctx context.Context) { timeCh := l.keepAliveWorker(ctx, l.leaseTimeout/3) var maxExpire time.Time + timer := time.NewTimer(l.leaseTimeout) + defer timer.Stop() for { select { case t := <-timeCh: @@ -115,7 +124,17 @@ func (l *lease) KeepAlive(ctx context.Context) { l.expireTime.Store(t) } } - case <-time.After(l.leaseTimeout): + // Stop the timer if it's not stopped. + if !timer.Stop() { + select { + case <-timer.C: // try to drain from the channel + default: + } + } + // We need be careful here, see more details in the comments of Timer.Reset. + // https://pkg.go.dev/time@master#Timer.Reset + timer.Reset(l.leaseTimeout) + case <-timer.C: log.Info("lease timeout", zap.Time("expire", l.expireTime.Load().(time.Time)), zap.String("purpose", l.Purpose)) return case <-ctx.Done(): @@ -129,6 +148,7 @@ func (l *lease) keepAliveWorker(ctx context.Context, interval time.Duration) <-c ch := make(chan time.Time) go func() { + defer logutil.LogPanic() ticker := time.NewTicker(interval) defer ticker.Stop() @@ -137,10 +157,15 @@ func (l *lease) keepAliveWorker(ctx context.Context, interval time.Duration) <-c for { go func() { + defer logutil.LogPanic() start := time.Now() ctx1, cancel := context.WithTimeout(ctx, l.leaseTimeout) defer cancel() - res, err := l.lease.KeepAliveOnce(ctx1, l.ID) + var leaseID clientv3.LeaseID + if l.ID.Load() != nil { + leaseID = l.ID.Load().(clientv3.LeaseID) + } + res, err := l.lease.KeepAliveOnce(ctx1, leaseID) if err != nil { log.Warn("lease keep alive failed", zap.String("purpose", l.Purpose), errs.ZapError(err)) return diff --git a/pkg/encryption/kms.go b/pkg/encryption/kms.go index 95df9543c19..3e70b2deeb5 100644 --- a/pkg/encryption/kms.go +++ b/pkg/encryption/kms.go @@ -15,14 +15,13 @@ package encryption import ( + "context" "os" - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/credentials" - "github.com/aws/aws-sdk-go/aws/credentials/stscreds" - "github.com/aws/aws-sdk-go/aws/session" - "github.com/aws/aws-sdk-go/service/kms" - "github.com/aws/aws-sdk-go/service/sts" + sdkconfig "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials/stscreds" + "github.com/aws/aws-sdk-go-v2/service/kms" + "github.com/aws/aws-sdk-go-v2/service/sts" "github.com/pingcap/kvproto/pkg/encryptionpb" "github.com/tikv/pd/pkg/errs" ) @@ -48,24 +47,40 @@ func newMasterKeyFromKMS( if config.Vendor != kmsVendorAWS { return nil, errs.ErrEncryptionKMS.GenWithStack("unsupported KMS vendor: %s", config.Vendor) } - credentials, err := newAwsCredentials() - if err != nil { - return nil, err - } - session, err := session.NewSession(&aws.Config{ - Credentials: credentials, - Region: &config.Region, - Endpoint: &config.Endpoint, - }) + + cfg, err := sdkconfig.LoadDefaultConfig(context.TODO(), + sdkconfig.WithRegion(config.Region), + ) if err != nil { return nil, errs.ErrEncryptionKMS.Wrap(err).GenWithStack( - "fail to create AWS session to access KMS CMK") + "fail to load default config") } - client := kms.New(session) + + // Credentials from K8S IAM role. + roleArn := os.Getenv(envAwsRoleArn) + tokenFile := os.Getenv(envAwsWebIdentityTokenFile) + sessionName := os.Getenv(envAwsRoleSessionName) + optFn := func(options *kms.Options) {} + // Session name is optional. + if roleArn != "" && tokenFile != "" { + client := sts.NewFromConfig(cfg) + webIdentityRoleProvider := stscreds.NewWebIdentityRoleProvider( + client, + roleArn, + stscreds.IdentityTokenFile(tokenFile), + func(o *stscreds.WebIdentityRoleOptions) { + o.RoleSessionName = sessionName + }, + ) + optFn = func(options *kms.Options) { + options.Credentials = webIdentityRoleProvider + } + } + client := kms.NewFromConfig(cfg, optFn) if len(ciphertextKey) == 0 { - numberOfBytes := int64(masterKeyLength) + numberOfBytes := int32(masterKeyLength) // Create a new data key. - output, err := client.GenerateDataKey(&kms.GenerateDataKeyInput{ + output, err := client.GenerateDataKey(context.Background(), &kms.GenerateDataKeyInput{ KeyId: &config.KeyId, NumberOfBytes: &numberOfBytes, }) @@ -84,7 +99,7 @@ func newMasterKeyFromKMS( } } else { // Decrypt existing data key. - output, err := client.Decrypt(&kms.DecryptInput{ + output, err := client.Decrypt(context.Background(), &kms.DecryptInput{ KeyId: &config.KeyId, CiphertextBlob: ciphertextKey, }) @@ -104,36 +119,3 @@ func newMasterKeyFromKMS( } return } - -func newAwsCredentials() (*credentials.Credentials, error) { - var providers []credentials.Provider - - // Credentials from K8S IAM role. - roleArn := os.Getenv(envAwsRoleArn) - tokenFile := os.Getenv(envAwsWebIdentityTokenFile) - sessionName := os.Getenv(envAwsRoleSessionName) - // Session name is optional. - if roleArn != "" && tokenFile != "" { - session, err := session.NewSession() - if err != nil { - return nil, errs.ErrEncryptionKMS.Wrap(err).GenWithStack( - "fail to create AWS session to create a WebIdentityRoleProvider") - } - webIdentityProvider := stscreds.NewWebIdentityRoleProvider( - sts.New(session), roleArn, sessionName, tokenFile) - providers = append(providers, webIdentityProvider) - } - - providers = append(providers, - // Credentials from AWS environment variables. - &credentials.EnvProvider{}, - // Credentials from default AWS credentials file. - &credentials.SharedCredentialsProvider{ - Filename: "", - Profile: "", - }, - ) - - credentials := credentials.NewChainCredentials(providers) - return credentials, nil -} diff --git a/pkg/errs/errno.go b/pkg/errs/errno.go index 155d7ef45a4..0bd2a57dba5 100644 --- a/pkg/errs/errno.go +++ b/pkg/errs/errno.go @@ -17,10 +17,18 @@ package errs import "github.com/pingcap/errors" const ( - // NotLeaderErr indicates the the non-leader member received the requests which should be received by leader. + // NotLeaderErr indicates the non-leader member received the requests which should be received by leader. + // Note: keep the same as the ones defined on the client side, because the client side checks if an error message + // contains this string to judge whether the leader is changed. NotLeaderErr = "is not leader" - // MismatchLeaderErr indicates the the non-leader member received the requests which should be received by leader. + // MismatchLeaderErr indicates the non-leader member received the requests which should be received by leader. + // Note: keep the same as the ones defined on the client side, because the client side checks if an error message + // contains this string to judge whether the leader is changed. MismatchLeaderErr = "mismatch leader id" + // NotServedErr indicates an tso node/pod received the requests for the keyspace groups which are not served by it. + // Note: keep the same as the ones defined on the client side, because the client side checks if an error message + // contains this string to judge whether the leader is changed. + NotServedErr = "is not served" ) // common error in multiple packages @@ -31,20 +39,30 @@ var ( // tso errors var ( - ErrSetLocalTSOConfig = errors.Normalize("set local tso config failed, %s", errors.RFCCodeText("PD:tso:ErrSetLocalTSOConfig")) - ErrGetAllocator = errors.Normalize("get allocator failed, %s", errors.RFCCodeText("PD:tso:ErrGetAllocator")) - ErrGetLocalAllocator = errors.Normalize("get local allocator failed, %s", errors.RFCCodeText("PD:tso:ErrGetLocalAllocator")) - ErrSyncMaxTS = errors.Normalize("sync max ts failed, %s", errors.RFCCodeText("PD:tso:ErrSyncMaxTS")) - ErrResetUserTimestamp = errors.Normalize("reset user timestamp failed, %s", errors.RFCCodeText("PD:tso:ErrResetUserTimestamp")) - ErrGenerateTimestamp = errors.Normalize("generate timestamp failed, %s", errors.RFCCodeText("PD:tso:ErrGenerateTimestamp")) - ErrLogicOverflow = errors.Normalize("logic part overflow", errors.RFCCodeText("PD:tso:ErrLogicOverflow")) - ErrProxyTSOTimeout = errors.Normalize("proxy tso timeout", errors.RFCCodeText("PD:tso:ErrProxyTSOTimeout")) + ErrSetLocalTSOConfig = errors.Normalize("set local tso config failed, %s", errors.RFCCodeText("PD:tso:ErrSetLocalTSOConfig")) + ErrGetAllocator = errors.Normalize("get allocator failed, %s", errors.RFCCodeText("PD:tso:ErrGetAllocator")) + ErrGetLocalAllocator = errors.Normalize("get local allocator failed, %s", errors.RFCCodeText("PD:tso:ErrGetLocalAllocator")) + ErrSyncMaxTS = errors.Normalize("sync max ts failed, %s", errors.RFCCodeText("PD:tso:ErrSyncMaxTS")) + ErrResetUserTimestamp = errors.Normalize("reset user timestamp failed, %s", errors.RFCCodeText("PD:tso:ErrResetUserTimestamp")) + ErrGenerateTimestamp = errors.Normalize("generate timestamp failed, %s", errors.RFCCodeText("PD:tso:ErrGenerateTimestamp")) + ErrLogicOverflow = errors.Normalize("logic part overflow", errors.RFCCodeText("PD:tso:ErrLogicOverflow")) + ErrProxyTSOTimeout = errors.Normalize("proxy tso timeout", errors.RFCCodeText("PD:tso:ErrProxyTSOTimeout")) + ErrKeyspaceGroupIDInvalid = errors.Normalize("the keyspace group id is invalid, %s", errors.RFCCodeText("PD:tso:ErrKeyspaceGroupIDInvalid")) + ErrGetAllocatorManager = errors.Normalize("get allocator manager failed, %s", errors.RFCCodeText("PD:tso:ErrGetAllocatorManager")) + ErrLoadKeyspaceGroupsTimeout = errors.Normalize("load keyspace groups timeout", errors.RFCCodeText("PD:tso:ErrLoadKeyspaceGroupsTimeout")) + ErrLoadKeyspaceGroupsTerminated = errors.Normalize("load keyspace groups terminated", errors.RFCCodeText("PD:tso:ErrLoadKeyspaceGroupsTerminated")) + ErrLoadKeyspaceGroupsRetryExhausted = errors.Normalize("load keyspace groups retry exhausted, %s", errors.RFCCodeText("PD:tso:ErrLoadKeyspaceGroupsRetryExhausted")) + ErrKeyspaceGroupNotInitialized = errors.Normalize("the keyspace group %d isn't initialized", errors.RFCCodeText("PD:tso:ErrKeyspaceGroupNotInitialized")) + ErrKeyspaceNotAssigned = errors.Normalize("the keyspace %d isn't assigned to any keyspace group", errors.RFCCodeText("PD:tso:ErrKeyspaceNotAssigned")) + ErrGetMinTS = errors.Normalize("get min ts failed, %s", errors.RFCCodeText("PD:tso:ErrGetMinTS")) + ErrKeyspaceGroupIsMerging = errors.Normalize("the keyspace group %d is merging", errors.RFCCodeText("PD:tso:ErrKeyspaceGroupIsMerging")) ) // member errors var ( ErrEtcdLeaderNotFound = errors.Normalize("etcd leader not found", errors.RFCCodeText("PD:member:ErrEtcdLeaderNotFound")) ErrMarshalLeader = errors.Normalize("marshal leader failed", errors.RFCCodeText("PD:member:ErrMarshalLeader")) + ErrCheckCampaign = errors.Normalize("check campaign failed", errors.RFCCodeText("PD:member:ErrCheckCampaign")) ) // core errors @@ -67,8 +85,9 @@ var ( ErrClientCreateTSOStream = errors.Normalize("create TSO stream failed, %s", errors.RFCCodeText("PD:client:ErrClientCreateTSOStream")) ErrClientGetTSOTimeout = errors.Normalize("get TSO timeout", errors.RFCCodeText("PD:client:ErrClientGetTSOTimeout")) ErrClientGetTSO = errors.Normalize("get TSO failed, %v", errors.RFCCodeText("PD:client:ErrClientGetTSO")) - ErrClientGetLeader = errors.Normalize("get leader from %v error", errors.RFCCodeText("PD:client:ErrClientGetLeader")) + ErrClientGetLeader = errors.Normalize("get leader failed, %v", errors.RFCCodeText("PD:client:ErrClientGetLeader")) ErrClientGetMember = errors.Normalize("get member failed", errors.RFCCodeText("PD:client:ErrClientGetMember")) + ErrClientGetMinTSO = errors.Normalize("get min TSO failed, %v", errors.RFCCodeText("PD:client:ErrClientGetMinTSO")) ) // schedule errors @@ -122,9 +141,10 @@ var ( // cluster errors var ( - ErrNotBootstrapped = errors.Normalize("TiKV cluster not bootstrapped, please start TiKV first", errors.RFCCodeText("PD:cluster:ErrNotBootstrapped")) - ErrStoreIsUp = errors.Normalize("store is still up, please remove store gracefully", errors.RFCCodeText("PD:cluster:ErrStoreIsUp")) - ErrInvalidStoreID = errors.Normalize("invalid store id %d, not found", errors.RFCCodeText("PD:cluster:ErrInvalidStoreID")) + ErrNotBootstrapped = errors.Normalize("TiKV cluster not bootstrapped, please start TiKV first", errors.RFCCodeText("PD:cluster:ErrNotBootstrapped")) + ErrStoreIsUp = errors.Normalize("store is still up, please remove store gracefully", errors.RFCCodeText("PD:cluster:ErrStoreIsUp")) + ErrInvalidStoreID = errors.Normalize("invalid store id %d, not found", errors.RFCCodeText("PD:cluster:ErrInvalidStoreID")) + ErrSchedulingIsHalted = errors.Normalize("scheduling is halted", errors.RFCCodeText("PD:cluster:ErrSchedulingIsHalted")) ) // versioninfo errors @@ -349,3 +369,10 @@ var ( ErrProgressWrongStatus = errors.Normalize("progress status is wrong", errors.RFCCodeText("PD:progress:ErrProgressWrongStatus")) ErrProgressNotFound = errors.Normalize("no progress found for %s", errors.RFCCodeText("PD:progress:ErrProgressNotFound")) ) + +// Resource Manager errors +var ( + ErrResourceGroupNotExists = errors.Normalize("the %s resource group does not exist", errors.RFCCodeText("PD:resourcemanager:ErrGroupNotExists")) + ErrDeleteReservedGroup = errors.Normalize("cannot delete reserved group", errors.RFCCodeText("PD:resourcemanager:ErrDeleteReservedGroup")) + ErrInvalidGroup = errors.Normalize("invalid group settings, please check the group name, priority and the number of resources", errors.RFCCodeText("PD:resourcemanager:ErrInvalidGroup")) +) diff --git a/pkg/gc/metrics.go b/pkg/gc/metrics.go new file mode 100644 index 00000000000..e70ecbf2209 --- /dev/null +++ b/pkg/gc/metrics.go @@ -0,0 +1,31 @@ +// Copyright 2023 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gc + +import "github.com/prometheus/client_golang/prometheus" + +var ( + gcSafePointGauge = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: "pd", + Subsystem: "gc", + Name: "gc_safepoint", + Help: "The ts of gc safepoint", + }, []string{"type"}) +) + +func init() { + prometheus.MustRegister(gcSafePointGauge) +} diff --git a/server/gc/safepoint.go b/pkg/gc/safepoint.go similarity index 79% rename from server/gc/safepoint.go rename to pkg/gc/safepoint.go index 167f0319daa..be12cfb78a5 100644 --- a/server/gc/safepoint.go +++ b/pkg/gc/safepoint.go @@ -18,20 +18,26 @@ import ( "math" "time" + "github.com/pingcap/errors" "github.com/tikv/pd/pkg/storage/endpoint" "github.com/tikv/pd/pkg/utils/syncutil" + "github.com/tikv/pd/server/config" ) +var blockGCSafePointErrmsg = "don't allow update gc safe point v1." +var blockServiceSafepointErrmsg = "don't allow update service safe point v1." + // SafePointManager is the manager for safePoint of GC and services. type SafePointManager struct { gcLock syncutil.Mutex serviceGCLock syncutil.Mutex store endpoint.GCSafePointStorage + cfg config.PDServerConfig } // NewSafePointManager creates a SafePointManager of GC and services. -func NewSafePointManager(store endpoint.GCSafePointStorage) *SafePointManager { - return &SafePointManager{store: store} +func NewSafePointManager(store endpoint.GCSafePointStorage, cfg config.PDServerConfig) *SafePointManager { + return &SafePointManager{store: store, cfg: cfg} } // LoadGCSafePoint loads current GC safe point from storage. @@ -49,15 +55,26 @@ func (manager *SafePointManager) UpdateGCSafePoint(newSafePoint uint64) (oldSafe if err != nil { return } + if manager.cfg.BlockSafePointV1 { + err = errors.Errorf(blockGCSafePointErrmsg) + return + } + if oldSafePoint >= newSafePoint { return } err = manager.store.SaveGCSafePoint(newSafePoint) + if err == nil { + gcSafePointGauge.WithLabelValues("gc_safepoint").Set(float64(newSafePoint)) + } return } // UpdateServiceGCSafePoint update the safepoint for a specific service. func (manager *SafePointManager) UpdateServiceGCSafePoint(serviceID string, newSafePoint uint64, ttl int64, now time.Time) (minServiceSafePoint *endpoint.ServiceSafePoint, updated bool, err error) { + if manager.cfg.BlockSafePointV1 { + return nil, false, errors.Errorf(blockServiceSafepointErrmsg) + } manager.serviceGCLock.Lock() defer manager.serviceGCLock.Unlock() minServiceSafePoint, err = manager.store.LoadMinServiceGCSafePoint(now) diff --git a/server/gc/safepoint_test.go b/pkg/gc/safepoint_test.go similarity index 84% rename from server/gc/safepoint_test.go rename to pkg/gc/safepoint_test.go index 41ce02639fd..39cd3660b2b 100644 --- a/server/gc/safepoint_test.go +++ b/pkg/gc/safepoint_test.go @@ -23,6 +23,7 @@ import ( "github.com/stretchr/testify/require" "github.com/tikv/pd/pkg/storage/endpoint" "github.com/tikv/pd/pkg/storage/kv" + "github.com/tikv/pd/server/config" ) func newGCStorage() endpoint.GCSafePointStorage { @@ -30,7 +31,7 @@ func newGCStorage() endpoint.GCSafePointStorage { } func TestGCSafePointUpdateSequentially(t *testing.T) { - gcSafePointManager := NewSafePointManager(newGCStorage()) + gcSafePointManager := NewSafePointManager(newGCStorage(), config.PDServerConfig{}) re := require.New(t) curSafePoint := uint64(0) // update gc safePoint with asc value. @@ -59,7 +60,7 @@ func TestGCSafePointUpdateSequentially(t *testing.T) { } func TestGCSafePointUpdateCurrently(t *testing.T) { - gcSafePointManager := NewSafePointManager(newGCStorage()) + gcSafePointManager := NewSafePointManager(newGCStorage(), config.PDServerConfig{}) maxSafePoint := uint64(1000) wg := sync.WaitGroup{} re := require.New(t) @@ -83,7 +84,7 @@ func TestGCSafePointUpdateCurrently(t *testing.T) { func TestServiceGCSafePointUpdate(t *testing.T) { re := require.New(t) - manager := NewSafePointManager(newGCStorage()) + manager := NewSafePointManager(newGCStorage(), config.PDServerConfig{}) gcworkerServiceID := "gc_worker" cdcServiceID := "cdc" brServiceID := "br" @@ -162,3 +163,22 @@ func TestServiceGCSafePointUpdate(t *testing.T) { re.NoError(err) re.True(updated) } + +func TestBlockUpdateSafePointV1(t *testing.T) { + re := require.New(t) + manager := NewSafePointManager(newGCStorage(), config.PDServerConfig{BlockSafePointV1: true}) + gcworkerServiceID := "gc_worker" + gcWorkerSafePoint := uint64(8) + + min, updated, err := manager.UpdateServiceGCSafePoint(gcworkerServiceID, gcWorkerSafePoint, math.MaxInt64, time.Now()) + re.Error(err, blockServiceSafepointErrmsg) + re.Equal(err.Error(), blockServiceSafepointErrmsg) + re.False(updated) + re.Nil(min) + + oldSafePoint, err := manager.UpdateGCSafePoint(gcWorkerSafePoint) + re.Error(err) + re.Equal(err.Error(), blockGCSafePointErrmsg) + + re.Equal(uint64(0), oldSafePoint) +} diff --git a/pkg/gc/safepoint_v2.go b/pkg/gc/safepoint_v2.go new file mode 100644 index 00000000000..f936601d2d9 --- /dev/null +++ b/pkg/gc/safepoint_v2.go @@ -0,0 +1,214 @@ +// Copyright 2023 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gc + +import ( + "context" + "time" + + "github.com/pingcap/failpoint" + "github.com/pingcap/kvproto/pkg/keyspacepb" + "github.com/pingcap/log" + "github.com/pkg/errors" + "github.com/tikv/pd/pkg/keyspace" + "github.com/tikv/pd/pkg/slice" + "github.com/tikv/pd/pkg/storage/endpoint" + "github.com/tikv/pd/pkg/storage/kv" + "github.com/tikv/pd/pkg/utils/syncutil" + "go.uber.org/zap" +) + +var ( + // allowUpdateSafePoint specifies under which states is a keyspace allowed to update it's gc & service safe points. + allowUpdateSafePoint = []keyspacepb.KeyspaceState{ + keyspacepb.KeyspaceState_ENABLED, + keyspacepb.KeyspaceState_DISABLED, + } +) + +// SafePointV2Manager is the manager for GCSafePointV2 and ServiceSafePointV2. +type SafePointV2Manager struct { + *syncutil.LockGroup + ctx context.Context + // keyspaceStorage stores keyspace meta. + keyspaceStorage endpoint.KeyspaceStorage + // v2Storage is the storage GCSafePointV2 and ServiceSafePointV2. + v2Storage endpoint.SafePointV2Storage + // v1Storage is the storage for v1 format GCSafePoint and ServiceGCSafePoint, it's used during pd update. + v1Storage endpoint.GCSafePointStorage +} + +// NewSafePointManagerV2 returns a new SafePointV2Manager. +func NewSafePointManagerV2( + ctx context.Context, + keyspaceStore endpoint.KeyspaceStorage, + v2Storage endpoint.SafePointV2Storage, + v1Storage endpoint.GCSafePointStorage, +) *SafePointV2Manager { + return &SafePointV2Manager{ + ctx: ctx, + LockGroup: syncutil.NewLockGroup(syncutil.WithHash(keyspace.MaskKeyspaceID)), + keyspaceStorage: keyspaceStore, + v2Storage: v2Storage, + v1Storage: v1Storage, + } +} + +// LoadGCSafePoint returns GCSafePointV2 of keyspaceID. +func (manager *SafePointV2Manager) LoadGCSafePoint(keyspaceID uint32) (*endpoint.GCSafePointV2, error) { + manager.Lock(keyspaceID) + defer manager.Unlock(keyspaceID) + // Check if keyspace is valid to load. + if err := manager.checkKeyspace(keyspaceID, false); err != nil { + return nil, err + } + gcSafePoint, err := manager.getGCSafePoint(keyspaceID) + if err != nil { + log.Warn("failed to load gc safe point", + zap.Uint32("keyspace-id", keyspaceID), + zap.Error(err), + ) + return nil, err + } + return gcSafePoint, nil +} + +// checkKeyspace check if target keyspace exists, and if request is a update request, +// also check if keyspace state allows for update. +func (manager *SafePointV2Manager) checkKeyspace(keyspaceID uint32, updateRequest bool) error { + failpoint.Inject("checkKeyspace", func() { + failpoint.Return(nil) + }) + + err := manager.keyspaceStorage.RunInTxn(manager.ctx, func(txn kv.Txn) error { + meta, err := manager.keyspaceStorage.LoadKeyspaceMeta(txn, keyspaceID) + if err != nil { + return err + } + // If a keyspace does not exist, then loading its gc safe point is prohibited. + if meta == nil { + return keyspace.ErrKeyspaceNotFound + } + // If keyspace's state does not permit updating safe point, we return error. + if updateRequest && !slice.Contains(allowUpdateSafePoint, meta.GetState()) { + return errors.Errorf("cannot update keyspace that's %s", meta.GetState().String()) + } + return nil + }) + if err != nil { + log.Warn("check keyspace failed", + zap.Uint32("keyspace-id", keyspaceID), + zap.Error(err), + ) + } + return err +} + +// getGCSafePoint first try to load gc safepoint from v2 storage, if failed, load from v1 storage instead. +func (manager *SafePointV2Manager) getGCSafePoint(keyspaceID uint32) (*endpoint.GCSafePointV2, error) { + v2SafePoint, err := manager.v2Storage.LoadGCSafePointV2(keyspaceID) + if err != nil { + return nil, err + } + // If failed to find a valid safe point, check if a safe point exist in v1 storage, and use it. + if v2SafePoint.SafePoint == 0 { + v1SafePoint, err := manager.v1Storage.LoadGCSafePoint() + if err != nil { + return nil, err + } + log.Info("keyspace does not have a gc safe point, using v1 gc safe point instead", + zap.Uint32("keyspace-id", keyspaceID), + zap.Uint64("gc-safe-point-v1", v1SafePoint)) + v2SafePoint.SafePoint = v1SafePoint + } + return v2SafePoint, nil +} + +// UpdateGCSafePoint is used to update gc safe point for given keyspace. +func (manager *SafePointV2Manager) UpdateGCSafePoint(gcSafePoint *endpoint.GCSafePointV2) (oldGCSafePoint *endpoint.GCSafePointV2, err error) { + manager.Lock(gcSafePoint.KeyspaceID) + defer manager.Unlock(gcSafePoint.KeyspaceID) + // Check if keyspace is valid to load. + if err = manager.checkKeyspace(gcSafePoint.KeyspaceID, true); err != nil { + return + } + oldGCSafePoint, err = manager.getGCSafePoint(gcSafePoint.KeyspaceID) + if err != nil { + return + } + if oldGCSafePoint.SafePoint >= gcSafePoint.SafePoint { + return + } + err = manager.v2Storage.SaveGCSafePointV2(gcSafePoint) + return +} + +// UpdateServiceSafePoint update keyspace service safe point with the given serviceSafePoint. +func (manager *SafePointV2Manager) UpdateServiceSafePoint(serviceSafePoint *endpoint.ServiceSafePointV2, now time.Time) (*endpoint.ServiceSafePointV2, error) { + manager.Lock(serviceSafePoint.KeyspaceID) + defer manager.Unlock(serviceSafePoint.KeyspaceID) + // Check if keyspace is valid to update. + if err := manager.checkKeyspace(serviceSafePoint.KeyspaceID, true); err != nil { + return nil, err + } + minServiceSafePoint, err := manager.v2Storage.LoadMinServiceSafePointV2(serviceSafePoint.KeyspaceID, now) + if err != nil { + return nil, err + } + if serviceSafePoint.SafePoint < minServiceSafePoint.SafePoint { + log.Warn("failed to update service safe point, proposed safe point smaller than current min", + zap.Error(err), + zap.Uint32("keyspace-id", serviceSafePoint.KeyspaceID), + zap.Uint64("request-service-safe-point", serviceSafePoint.SafePoint), + zap.Uint64("min-service-safe-point", minServiceSafePoint.SafePoint), + ) + return minServiceSafePoint, nil + } + if err = manager.v2Storage.SaveServiceSafePointV2(serviceSafePoint); err != nil { + return nil, err + } + // If the updated safe point is the original min safe point, reload min safe point. + if serviceSafePoint.ServiceID == minServiceSafePoint.ServiceID { + minServiceSafePoint, err = manager.v2Storage.LoadMinServiceSafePointV2(serviceSafePoint.KeyspaceID, now) + } + if err != nil { + log.Info("update service safe point", + zap.String("service-id", serviceSafePoint.ServiceID), + zap.Int64("expire-at", serviceSafePoint.ExpiredAt), + zap.Uint64("safepoint", serviceSafePoint.SafePoint), + ) + } + return minServiceSafePoint, err +} + +// RemoveServiceSafePoint remove keyspace service safe point with the given keyspaceID and serviceID. +func (manager *SafePointV2Manager) RemoveServiceSafePoint(keyspaceID uint32, serviceID string, now time.Time) (*endpoint.ServiceSafePointV2, error) { + manager.Lock(keyspaceID) + defer manager.Unlock(keyspaceID) + // Check if keyspace is valid to update. + if err := manager.checkKeyspace(keyspaceID, true); err != nil { + return nil, err + } + // Remove target safe point. + if err := manager.v2Storage.RemoveServiceSafePointV2(keyspaceID, serviceID); err != nil { + return nil, err + } + // Load min safe point. + minServiceSafePoint, err := manager.v2Storage.LoadMinServiceSafePointV2(keyspaceID, now) + if err != nil { + return nil, err + } + return minServiceSafePoint, nil +} diff --git a/pkg/gctuner/memory_limit_tuner.go b/pkg/gctuner/memory_limit_tuner.go index 59200368c63..77a8f9db4c6 100644 --- a/pkg/gctuner/memory_limit_tuner.go +++ b/pkg/gctuner/memory_limit_tuner.go @@ -23,6 +23,7 @@ import ( "github.com/pingcap/log" util "github.com/tikv/pd/pkg/gogc" "github.com/tikv/pd/pkg/memory" + "github.com/tikv/pd/pkg/utils/logutil" atomicutil "go.uber.org/atomic" "go.uber.org/zap" ) @@ -72,6 +73,7 @@ func (t *memoryLimitTuner) tuning() { if float64(r.HeapInuse)*ratio > float64(setMemoryLimit(-1)) { if t.nextGCTriggeredByMemoryLimit.Load() && t.waitingReset.CompareAndSwap(false, true) { go func() { + defer logutil.LogPanic() memory.MemoryLimitGCLast.Store(time.Now()) memory.MemoryLimitGCTotal.Add(1) setMemoryLimit(t.calcMemoryLimit(fallbackPercentage)) diff --git a/pkg/keyspace/keyspace.go b/pkg/keyspace/keyspace.go new file mode 100644 index 00000000000..1ce599b5f81 --- /dev/null +++ b/pkg/keyspace/keyspace.go @@ -0,0 +1,822 @@ +// Copyright 2022 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package keyspace + +import ( + "bytes" + "context" + "strconv" + "time" + + "github.com/pingcap/errors" + "github.com/pingcap/failpoint" + "github.com/pingcap/kvproto/pkg/keyspacepb" + "github.com/pingcap/log" + "github.com/tikv/pd/pkg/id" + "github.com/tikv/pd/pkg/mcs/utils" + "github.com/tikv/pd/pkg/schedule/core" + "github.com/tikv/pd/pkg/schedule/labeler" + "github.com/tikv/pd/pkg/slice" + "github.com/tikv/pd/pkg/storage/endpoint" + "github.com/tikv/pd/pkg/storage/kv" + "github.com/tikv/pd/pkg/utils/syncutil" + "go.uber.org/zap" +) + +const ( + // AllocStep set idAllocator's step when write persistent window boundary. + // Use a lower value for denser idAllocation in the event of frequent pd leader change. + AllocStep = uint64(100) + // AllocLabel is used to label keyspace idAllocator's metrics. + AllocLabel = "keyspace-idAlloc" + // regionLabelIDPrefix is used to prefix the keyspace region label. + regionLabelIDPrefix = "keyspaces/" + // regionLabelKey is the key for keyspace id in keyspace region label. + regionLabelKey = "id" + // UserKindKey is the key for user kind in keyspace config. + UserKindKey = "user_kind" + // TSOKeyspaceGroupIDKey is the key for tso keyspace group id in keyspace config. + // Note: Config[TSOKeyspaceGroupIDKey] is only used to judge whether there is keyspace group id. + // It will not update the keyspace group id when merging or splitting. + TSOKeyspaceGroupIDKey = "tso_keyspace_group_id" + // MaxEtcdTxnOps is the max value of operations in an etcd txn. The default limit of etcd txn op is 128. + // We use 120 here to leave some space for other operations. + // See: https://github.com/etcd-io/etcd/blob/d3e43d4de6f6d9575b489dd7850a85e37e0f6b6c/server/embed/config.go#L61 + MaxEtcdTxnOps = 120 +) + +// Config is the interface for keyspace config. +type Config interface { + GetPreAlloc() []string + ToWaitRegionSplit() bool + GetWaitRegionSplitTimeout() time.Duration + GetCheckRegionSplitInterval() time.Duration +} + +// Manager manages keyspace related data. +// It validates requests and provides concurrency control. +type Manager struct { + // ctx is the context of the manager, to be used in transaction. + ctx context.Context + // metaLock guards keyspace meta. + metaLock *syncutil.LockGroup + // idAllocator allocates keyspace id. + idAllocator id.Allocator + // store is the storage for keyspace related information. + store endpoint.KeyspaceStorage + // rc is the raft cluster of the server. + cluster core.ClusterInformer + // config is the configurations of the manager. + config Config + // kgm is the keyspace group manager of the server. + kgm *GroupManager + // nextPatrolStartID is the next start id of keyspace assignment patrol. + nextPatrolStartID uint32 +} + +// CreateKeyspaceRequest represents necessary arguments to create a keyspace. +type CreateKeyspaceRequest struct { + // Name of the keyspace to be created. + // Using an existing name will result in error. + Name string + Config map[string]string + // CreateTime is the timestamp used to record creation time. + CreateTime int64 + // IsPreAlloc indicates whether the keyspace is pre-allocated when the cluster starts. + IsPreAlloc bool +} + +// NewKeyspaceManager creates a Manager of keyspace related data. +func NewKeyspaceManager( + ctx context.Context, + store endpoint.KeyspaceStorage, + cluster core.ClusterInformer, + idAllocator id.Allocator, + config Config, + kgm *GroupManager, +) *Manager { + return &Manager{ + ctx: ctx, + // Remove the lock of the given key from the lock group when unlock to + // keep minimal working set, which is suited for low qps, non-time-critical + // and non-consecutive large key space scenarios. One of scenarios for + // last use case is keyspace group split loads non-consecutive keyspace meta + // in batches and lock all loaded keyspace meta within a batch at the same time. + metaLock: syncutil.NewLockGroup(syncutil.WithRemoveEntryOnUnlock(true)), + idAllocator: idAllocator, + store: store, + cluster: cluster, + config: config, + kgm: kgm, + nextPatrolStartID: utils.DefaultKeyspaceID, + } +} + +// Bootstrap saves default keyspace info. +func (manager *Manager) Bootstrap() error { + // Split Keyspace Region for default keyspace. + if err := manager.splitKeyspaceRegion(utils.DefaultKeyspaceID, false); err != nil { + return err + } + now := time.Now().Unix() + defaultKeyspaceMeta := &keyspacepb.KeyspaceMeta{ + Id: utils.DefaultKeyspaceID, + Name: utils.DefaultKeyspaceName, + State: keyspacepb.KeyspaceState_ENABLED, + CreatedAt: now, + StateChangedAt: now, + } + + config, err := manager.kgm.GetKeyspaceConfigByKind(endpoint.Basic) + if err != nil { + return err + } + defaultKeyspaceMeta.Config = config + err = manager.saveNewKeyspace(defaultKeyspaceMeta) + // It's possible that default keyspace already exists in the storage (e.g. PD restart/recover), + // so we ignore the keyspaceExists error. + if err != nil && err != ErrKeyspaceExists { + return err + } + if err := manager.kgm.UpdateKeyspaceForGroup(endpoint.Basic, config[TSOKeyspaceGroupIDKey], defaultKeyspaceMeta.GetId(), opAdd); err != nil { + return err + } + // Initialize pre-alloc keyspace. + preAlloc := manager.config.GetPreAlloc() + for _, keyspaceName := range preAlloc { + config, err := manager.kgm.GetKeyspaceConfigByKind(endpoint.Basic) + if err != nil { + return err + } + req := &CreateKeyspaceRequest{ + Name: keyspaceName, + CreateTime: now, + IsPreAlloc: true, + Config: config, + } + keyspace, err := manager.CreateKeyspace(req) + // Ignore the keyspaceExists error for the same reason as saving default keyspace. + if err != nil && err != ErrKeyspaceExists { + return err + } + if err := manager.kgm.UpdateKeyspaceForGroup(endpoint.Basic, config[TSOKeyspaceGroupIDKey], keyspace.GetId(), opAdd); err != nil { + return err + } + } + return nil +} + +// UpdateConfig update keyspace manager's config. +func (manager *Manager) UpdateConfig(cfg Config) { + manager.config = cfg +} + +// CreateKeyspace create a keyspace meta with given config and save it to storage. +func (manager *Manager) CreateKeyspace(request *CreateKeyspaceRequest) (*keyspacepb.KeyspaceMeta, error) { + // Validate purposed name's legality. + if err := validateName(request.Name); err != nil { + return nil, err + } + // Allocate new keyspaceID. + newID, err := manager.allocID() + if err != nil { + return nil, err + } + userKind := endpoint.StringUserKind(request.Config[UserKindKey]) + config, err := manager.kgm.GetKeyspaceConfigByKind(userKind) + if err != nil { + return nil, err + } + if len(config) != 0 { + if request.Config == nil { + request.Config = config + } else { + request.Config[TSOKeyspaceGroupIDKey] = config[TSOKeyspaceGroupIDKey] + request.Config[UserKindKey] = config[UserKindKey] + } + } + // Create a disabled keyspace meta for tikv-server to get the config on keyspace split. + keyspace := &keyspacepb.KeyspaceMeta{ + Id: newID, + Name: request.Name, + State: keyspacepb.KeyspaceState_DISABLED, + CreatedAt: request.CreateTime, + StateChangedAt: request.CreateTime, + Config: request.Config, + } + err = manager.saveNewKeyspace(keyspace) + if err != nil { + log.Warn("[keyspace] failed to save keyspace before split", + zap.Uint32("keyspace-id", keyspace.GetId()), + zap.String("name", keyspace.GetName()), + zap.Error(err), + ) + return nil, err + } + // If the request to create a keyspace is pre-allocated when the PD starts, + // there is no need to wait for the region split, because TiKV has not started. + waitRegionSplit := !request.IsPreAlloc && manager.config.ToWaitRegionSplit() + // Split keyspace region. + err = manager.splitKeyspaceRegion(newID, waitRegionSplit) + if err != nil { + err2 := manager.store.RunInTxn(manager.ctx, func(txn kv.Txn) error { + idPath := endpoint.KeyspaceIDPath(request.Name) + metaPath := endpoint.KeyspaceMetaPath(newID) + e := txn.Remove(idPath) + if e != nil { + return e + } + return txn.Remove(metaPath) + }) + if err2 != nil { + log.Warn("[keyspace] failed to remove pre-created keyspace after split failed", + zap.Uint32("keyspace-id", keyspace.GetId()), + zap.String("name", keyspace.GetName()), + zap.Error(err2), + ) + } + return nil, err + } + // enable the keyspace metadata after split. + keyspace.State = keyspacepb.KeyspaceState_ENABLED + _, err = manager.UpdateKeyspaceStateByID(newID, keyspacepb.KeyspaceState_ENABLED, request.CreateTime) + if err != nil { + log.Warn("[keyspace] failed to create keyspace", + zap.Uint32("keyspace-id", keyspace.GetId()), + zap.String("name", keyspace.GetName()), + zap.Error(err), + ) + return nil, err + } + if err := manager.kgm.UpdateKeyspaceForGroup(userKind, config[TSOKeyspaceGroupIDKey], keyspace.GetId(), opAdd); err != nil { + return nil, err + } + log.Info("[keyspace] keyspace created", + zap.Uint32("keyspace-id", keyspace.GetId()), + zap.String("name", keyspace.GetName()), + ) + return keyspace, nil +} + +func (manager *Manager) saveNewKeyspace(keyspace *keyspacepb.KeyspaceMeta) error { + manager.metaLock.Lock(keyspace.Id) + defer manager.metaLock.Unlock(keyspace.Id) + + return manager.store.RunInTxn(manager.ctx, func(txn kv.Txn) error { + // Save keyspace ID. + // Check if keyspace with that name already exists. + nameExists, _, err := manager.store.LoadKeyspaceID(txn, keyspace.Name) + if err != nil { + return err + } + if nameExists { + return ErrKeyspaceExists + } + err = manager.store.SaveKeyspaceID(txn, keyspace.Id, keyspace.Name) + if err != nil { + return err + } + // Save keyspace meta. + // Check if keyspace with that id already exists. + loadedMeta, err := manager.store.LoadKeyspaceMeta(txn, keyspace.Id) + if err != nil { + return err + } + if loadedMeta != nil { + return ErrKeyspaceExists + } + return manager.store.SaveKeyspaceMeta(txn, keyspace) + }) +} + +// splitKeyspaceRegion add keyspace's boundaries to region label. The corresponding +// region will then be split by Coordinator's patrolRegion. +func (manager *Manager) splitKeyspaceRegion(id uint32, waitRegionSplit bool) (err error) { + failpoint.Inject("skipSplitRegion", func() { + failpoint.Return(nil) + }) + + start := time.Now() + keyspaceRule := makeLabelRule(id) + cl, ok := manager.cluster.(interface{ GetRegionLabeler() *labeler.RegionLabeler }) + if !ok { + return errors.New("cluster does not support region label") + } + err = cl.GetRegionLabeler().SetLabelRule(keyspaceRule) + if err != nil { + log.Warn("[keyspace] failed to add region label for keyspace", + zap.Uint32("keyspace-id", id), + zap.Error(err), + ) + return err + } + defer func() { + if err != nil { + cl.GetRegionLabeler().DeleteLabelRule(keyspaceRule.ID) + } + }() + + if waitRegionSplit { + ranges := keyspaceRule.Data.([]*labeler.KeyRangeRule) + if len(ranges) < 2 { + log.Warn("[keyspace] failed to split keyspace region with insufficient range", zap.Any("label-rule", keyspaceRule)) + return ErrRegionSplitFailed + } + rawLeftBound, rawRightBound := ranges[0].StartKey, ranges[0].EndKey + txnLeftBound, txnRightBound := ranges[1].StartKey, ranges[1].EndKey + + ticker := time.NewTicker(manager.config.GetCheckRegionSplitInterval()) + timer := time.NewTimer(manager.config.GetWaitRegionSplitTimeout()) + defer func() { + ticker.Stop() + timer.Stop() + }() + for { + select { + case <-ticker.C: + regionsInfo := manager.cluster.GetBasicCluster().RegionsInfo + region := regionsInfo.GetRegionByKey(rawLeftBound) + if region == nil || !bytes.Equal(region.GetStartKey(), rawLeftBound) { + continue + } + region = regionsInfo.GetRegionByKey(rawRightBound) + if region == nil || !bytes.Equal(region.GetStartKey(), rawRightBound) { + continue + } + region = regionsInfo.GetRegionByKey(txnLeftBound) + if region == nil || !bytes.Equal(region.GetStartKey(), txnLeftBound) { + continue + } + region = regionsInfo.GetRegionByKey(txnRightBound) + if region == nil || !bytes.Equal(region.GetStartKey(), txnRightBound) { + continue + } + // Note: we reset the ticker here to support updating configuration dynamically. + ticker.Reset(manager.config.GetCheckRegionSplitInterval()) + case <-timer.C: + log.Warn("[keyspace] wait region split timeout", + zap.Uint32("keyspace-id", id), + zap.Error(err), + ) + err = ErrRegionSplitTimeout + return + } + log.Info("[keyspace] wait region split successfully", zap.Uint32("keyspace-id", id)) + break + } + } + + log.Info("[keyspace] added region label for keyspace", + zap.Uint32("keyspace-id", id), + zap.Any("label-rule", keyspaceRule), + zap.Duration("takes", time.Since(start)), + ) + return +} + +// LoadKeyspace returns the keyspace specified by name. +// It returns error if loading or unmarshalling met error or if keyspace does not exist. +func (manager *Manager) LoadKeyspace(name string) (*keyspacepb.KeyspaceMeta, error) { + var meta *keyspacepb.KeyspaceMeta + err := manager.store.RunInTxn(manager.ctx, func(txn kv.Txn) error { + loaded, id, err := manager.store.LoadKeyspaceID(txn, name) + if err != nil { + return err + } + if !loaded { + return ErrKeyspaceNotFound + } + meta, err = manager.store.LoadKeyspaceMeta(txn, id) + if err != nil { + return err + } + if meta == nil { + return ErrKeyspaceNotFound + } + return nil + }) + return meta, err +} + +// LoadKeyspaceByID returns the keyspace specified by id. +// It returns error if loading or unmarshalling met error or if keyspace does not exist. +func (manager *Manager) LoadKeyspaceByID(spaceID uint32) (*keyspacepb.KeyspaceMeta, error) { + var ( + meta *keyspacepb.KeyspaceMeta + err error + ) + err = manager.store.RunInTxn(manager.ctx, func(txn kv.Txn) error { + meta, err = manager.store.LoadKeyspaceMeta(txn, spaceID) + if err != nil { + return err + } + if meta == nil { + return ErrKeyspaceNotFound + } + return nil + }) + return meta, err +} + +// Mutation represents a single operation to be applied on keyspace config. +type Mutation struct { + Op OpType + Key string + Value string +} + +// OpType defines the type of keyspace config operation. +type OpType int + +const ( + // OpPut denotes a put operation onto the given config. + // If target key exists, it will put a new value, + // otherwise, it creates a new config entry. + OpPut OpType = iota + 1 // Operation type starts at 1. + // OpDel denotes a deletion operation onto the given config. + // Note: OpDel is idempotent, deleting a non-existing key + // will not result in error. + OpDel +) + +// UpdateKeyspaceConfig changes target keyspace's config in the order specified in mutations. +// It returns error if saving failed, operation not allowed, or if keyspace not exists. +func (manager *Manager) UpdateKeyspaceConfig(name string, mutations []*Mutation) (*keyspacepb.KeyspaceMeta, error) { + var meta *keyspacepb.KeyspaceMeta + oldConfig := make(map[string]string) + err := manager.store.RunInTxn(manager.ctx, func(txn kv.Txn) error { + // First get KeyspaceID from Name. + loaded, id, err := manager.store.LoadKeyspaceID(txn, name) + if err != nil { + return err + } + if !loaded { + return ErrKeyspaceNotFound + } + manager.metaLock.Lock(id) + defer manager.metaLock.Unlock(id) + // Load keyspace by id. + meta, err = manager.store.LoadKeyspaceMeta(txn, id) + if err != nil { + return err + } + if meta == nil { + return ErrKeyspaceNotFound + } + // Only keyspace with state listed in allowChangeConfig are allowed to change their config. + if !slice.Contains(allowChangeConfig, meta.GetState()) { + return errors.Errorf("cannot change config for keyspace with state %s", meta.GetState().String()) + } + // Initialize meta's config map if it's nil. + if meta.GetConfig() == nil { + meta.Config = map[string]string{} + } + for k, v := range meta.GetConfig() { + oldConfig[k] = v + } + // Update keyspace config according to mutations. + for _, mutation := range mutations { + switch mutation.Op { + case OpPut: + meta.Config[mutation.Key] = mutation.Value + case OpDel: + delete(meta.Config, mutation.Key) + default: + return errIllegalOperation + } + } + newConfig := meta.GetConfig() + oldUserKind := endpoint.StringUserKind(oldConfig[UserKindKey]) + newUserKind := endpoint.StringUserKind(newConfig[UserKindKey]) + oldID := oldConfig[TSOKeyspaceGroupIDKey] + newID := newConfig[TSOKeyspaceGroupIDKey] + needUpdate := oldUserKind != newUserKind || oldID != newID + if needUpdate { + if err := manager.kgm.UpdateKeyspaceGroup(oldID, newID, oldUserKind, newUserKind, meta.GetId()); err != nil { + return err + } + } + // Save the updated keyspace meta. + if err := manager.store.SaveKeyspaceMeta(txn, meta); err != nil { + if needUpdate { + if err := manager.kgm.UpdateKeyspaceGroup(newID, oldID, newUserKind, oldUserKind, meta.GetId()); err != nil { + log.Error("failed to revert keyspace group", zap.Error(err)) + } + } + return err + } + return nil + }) + + if err != nil { + log.Warn("[keyspace] failed to update keyspace config", + zap.Uint32("keyspace-id", meta.GetId()), + zap.String("name", meta.GetName()), + zap.Error(err), + ) + return nil, err + } + log.Info("[keyspace] keyspace config updated", + zap.Uint32("keyspace-id", meta.GetId()), + zap.String("name", meta.GetName()), + zap.Any("new-config", meta.GetConfig()), + ) + return meta, nil +} + +// UpdateKeyspaceState updates target keyspace to the given state if it's not already in that state. +// It returns error if saving failed, operation not allowed, or if keyspace not exists. +func (manager *Manager) UpdateKeyspaceState(name string, newState keyspacepb.KeyspaceState, now int64) (*keyspacepb.KeyspaceMeta, error) { + // Changing the state of default keyspace is not allowed. + if name == utils.DefaultKeyspaceName { + log.Warn("[keyspace] failed to update keyspace config", + zap.Error(ErrModifyDefaultKeyspace), + ) + return nil, ErrModifyDefaultKeyspace + } + var meta *keyspacepb.KeyspaceMeta + err := manager.store.RunInTxn(manager.ctx, func(txn kv.Txn) error { + // First get KeyspaceID from Name. + loaded, id, err := manager.store.LoadKeyspaceID(txn, name) + if err != nil { + return err + } + if !loaded { + return ErrKeyspaceNotFound + } + manager.metaLock.Lock(id) + defer manager.metaLock.Unlock(id) + // Load keyspace by id. + meta, err = manager.store.LoadKeyspaceMeta(txn, id) + if err != nil { + return err + } + if meta == nil { + return ErrKeyspaceNotFound + } + // Update keyspace meta. + if err = updateKeyspaceState(meta, newState, now); err != nil { + return err + } + return manager.store.SaveKeyspaceMeta(txn, meta) + }) + if err != nil { + log.Warn("[keyspace] failed to update keyspace config", + zap.Uint32("keyspace-id", meta.GetId()), + zap.String("name", meta.GetName()), + zap.Error(err), + ) + return nil, err + } + log.Info("[keyspace] keyspace state updated", + zap.Uint32("ID", meta.GetId()), + zap.String("keyspace-id", meta.GetName()), + zap.String("new-state", newState.String()), + ) + return meta, nil +} + +// UpdateKeyspaceStateByID updates target keyspace to the given state if it's not already in that state. +// It returns error if saving failed, operation not allowed, or if keyspace not exists. +func (manager *Manager) UpdateKeyspaceStateByID(id uint32, newState keyspacepb.KeyspaceState, now int64) (*keyspacepb.KeyspaceMeta, error) { + // Changing the state of default keyspace is not allowed. + if id == utils.DefaultKeyspaceID { + log.Warn("[keyspace] failed to update keyspace config", + zap.Error(ErrModifyDefaultKeyspace), + ) + return nil, ErrModifyDefaultKeyspace + } + var meta *keyspacepb.KeyspaceMeta + var err error + err = manager.store.RunInTxn(manager.ctx, func(txn kv.Txn) error { + manager.metaLock.Lock(id) + defer manager.metaLock.Unlock(id) + // Load keyspace by id. + meta, err = manager.store.LoadKeyspaceMeta(txn, id) + if err != nil { + return err + } + if meta == nil { + return ErrKeyspaceNotFound + } + // Update keyspace meta. + if err = updateKeyspaceState(meta, newState, now); err != nil { + return err + } + return manager.store.SaveKeyspaceMeta(txn, meta) + }) + if err != nil { + log.Warn("[keyspace] failed to update keyspace config", + zap.Uint32("keyspace-id", meta.GetId()), + zap.String("name", meta.GetName()), + zap.Error(err), + ) + return nil, err + } + log.Info("[keyspace] keyspace state updated", + zap.Uint32("keyspace-id", meta.GetId()), + zap.String("name", meta.GetName()), + zap.String("new-state", newState.String()), + ) + return meta, nil +} + +// updateKeyspaceState updates keyspace meta and record the update time. +func updateKeyspaceState(meta *keyspacepb.KeyspaceMeta, newState keyspacepb.KeyspaceState, now int64) error { + // If already in the target state, do nothing and return. + if meta.GetState() == newState { + return nil + } + // Consult state transition table to check if the operation is legal. + if !slice.Contains(stateTransitionTable[meta.GetState()], newState) { + return errors.Errorf("cannot change keyspace state from %s to %s", meta.GetState().String(), newState.String()) + } + // If the operation is legal, update keyspace state and change time. + meta.State = newState + meta.StateChangedAt = now + return nil +} + +// LoadRangeKeyspace load up to limit keyspaces starting from keyspace with startID. +func (manager *Manager) LoadRangeKeyspace(startID uint32, limit int) ([]*keyspacepb.KeyspaceMeta, error) { + // Load Start should fall within acceptable ID range. + if startID > spaceIDMax { + return nil, errors.Errorf("startID of the scan %d exceeds spaceID Max %d", startID, spaceIDMax) + } + var ( + keyspaces []*keyspacepb.KeyspaceMeta + err error + ) + err = manager.store.RunInTxn(manager.ctx, func(txn kv.Txn) error { + keyspaces, err = manager.store.LoadRangeKeyspace(txn, startID, limit) + return err + }) + if err != nil { + return nil, err + } + return keyspaces, nil +} + +// allocID allocate a new keyspace id. +func (manager *Manager) allocID() (uint32, error) { + id64, err := manager.idAllocator.Alloc() + if err != nil { + return 0, err + } + id32 := uint32(id64) + if err = validateID(id32); err != nil { + return 0, err + } + return id32, nil +} + +// PatrolKeyspaceAssignment is used to patrol all keyspaces and assign them to the keyspace groups. +func (manager *Manager) PatrolKeyspaceAssignment(startKeyspaceID, endKeyspaceID uint32) error { + if startKeyspaceID > manager.nextPatrolStartID { + manager.nextPatrolStartID = startKeyspaceID + } + if endKeyspaceID != 0 && endKeyspaceID < manager.nextPatrolStartID { + log.Info("[keyspace] end keyspace id is smaller than the next patrol start id, skip patrol", + zap.Uint32("end-keyspace-id", endKeyspaceID), + zap.Uint32("next-patrol-start-id", manager.nextPatrolStartID)) + return nil + } + var ( + // Some statistics info. + start = time.Now() + patrolledKeyspaceCount uint64 + assignedKeyspaceCount uint64 + // The current start ID of the patrol, used for logging. + currentStartID = manager.nextPatrolStartID + // The next start ID of the patrol, used for the next patrol. + nextStartID = currentStartID + moreToPatrol = true + err error + ) + defer func() { + log.Debug("[keyspace] patrol keyspace assignment finished", + zap.Duration("cost", time.Since(start)), + zap.Uint64("patrolled-keyspace-count", patrolledKeyspaceCount), + zap.Uint64("assigned-keyspace-count", assignedKeyspaceCount), + zap.Int("batch-size", MaxEtcdTxnOps), + zap.Uint32("start-keyspace-id", startKeyspaceID), + zap.Uint32("end-keyspace-id", endKeyspaceID), + zap.Uint32("current-start-id", currentStartID), + zap.Uint32("next-start-id", nextStartID), + ) + }() + for moreToPatrol { + var defaultKeyspaceGroup *endpoint.KeyspaceGroup + err = manager.store.RunInTxn(manager.ctx, func(txn kv.Txn) error { + var err error + defaultKeyspaceGroup, err = manager.kgm.store.LoadKeyspaceGroup(txn, utils.DefaultKeyspaceGroupID) + if err != nil { + return err + } + if defaultKeyspaceGroup == nil { + return errors.Errorf("default keyspace group %d not found", utils.DefaultKeyspaceGroupID) + } + if defaultKeyspaceGroup.IsSplitting() { + return ErrKeyspaceGroupInSplit(utils.DefaultKeyspaceGroupID) + } + if defaultKeyspaceGroup.IsMerging() { + return ErrKeyspaceGroupInMerging(utils.DefaultKeyspaceGroupID) + } + keyspaces, err := manager.store.LoadRangeKeyspace(txn, manager.nextPatrolStartID, MaxEtcdTxnOps) + if err != nil { + return err + } + keyspaceNum := len(keyspaces) + // If there are more than one keyspace, update the current and next start IDs. + if keyspaceNum > 0 { + currentStartID = keyspaces[0].GetId() + nextStartID = keyspaces[keyspaceNum-1].GetId() + 1 + } + // If there are less than `MaxEtcdTxnOps` keyspaces or the next start ID reaches the end, + // there is no need to patrol again. + moreToPatrol = keyspaceNum == MaxEtcdTxnOps + var ( + assigned = false + keyspaceIDsToUnlock = make([]uint32, 0, keyspaceNum) + ) + defer func() { + for _, id := range keyspaceIDsToUnlock { + manager.metaLock.Unlock(id) + } + }() + for _, ks := range keyspaces { + if ks == nil { + continue + } + if endKeyspaceID != 0 && ks.Id > endKeyspaceID { + moreToPatrol = false + break + } + patrolledKeyspaceCount++ + manager.metaLock.Lock(ks.Id) + if ks.Config == nil { + ks.Config = make(map[string]string, 1) + } else if _, ok := ks.Config[TSOKeyspaceGroupIDKey]; ok { + // If the keyspace already has a group ID, skip it. + manager.metaLock.Unlock(ks.Id) + continue + } + // Unlock the keyspace meta lock after the whole txn. + keyspaceIDsToUnlock = append(keyspaceIDsToUnlock, ks.Id) + // If the keyspace doesn't have a group ID, assign it to the default keyspace group. + if !slice.Contains(defaultKeyspaceGroup.Keyspaces, ks.Id) { + defaultKeyspaceGroup.Keyspaces = append(defaultKeyspaceGroup.Keyspaces, ks.Id) + // Only save the keyspace group meta if any keyspace is assigned to it. + assigned = true + } + ks.Config[TSOKeyspaceGroupIDKey] = strconv.FormatUint(uint64(utils.DefaultKeyspaceGroupID), 10) + err = manager.store.SaveKeyspaceMeta(txn, ks) + if err != nil { + log.Error("[keyspace] failed to save keyspace meta during patrol", + zap.Int("batch-size", MaxEtcdTxnOps), + zap.Uint32("start-keyspace-id", startKeyspaceID), + zap.Uint32("end-keyspace-id", endKeyspaceID), + zap.Uint32("current-start-id", currentStartID), + zap.Uint32("next-start-id", nextStartID), + zap.Uint32("keyspace-id", ks.Id), zap.Error(err)) + return err + } + assignedKeyspaceCount++ + } + if assigned { + err = manager.kgm.store.SaveKeyspaceGroup(txn, defaultKeyspaceGroup) + if err != nil { + log.Error("[keyspace] failed to save default keyspace group meta during patrol", + zap.Int("batch-size", MaxEtcdTxnOps), + zap.Uint32("start-keyspace-id", startKeyspaceID), + zap.Uint32("end-keyspace-id", endKeyspaceID), + zap.Uint32("current-start-id", currentStartID), + zap.Uint32("next-start-id", nextStartID), zap.Error(err)) + return err + } + } + return nil + }) + if err != nil { + return err + } + manager.kgm.Lock() + manager.kgm.groups[endpoint.StringUserKind(defaultKeyspaceGroup.UserKind)].Put(defaultKeyspaceGroup) + manager.kgm.Unlock() + // If all keyspaces in the current batch are assigned, update the next start ID. + manager.nextPatrolStartID = nextStartID + } + return nil +} diff --git a/server/keyspace/keyspace_test.go b/pkg/keyspace/keyspace_test.go similarity index 59% rename from server/keyspace/keyspace_test.go rename to pkg/keyspace/keyspace_test.go index d478736052e..27e7de359ee 100644 --- a/server/keyspace/keyspace_test.go +++ b/pkg/keyspace/keyspace_test.go @@ -15,6 +15,7 @@ package keyspace import ( + "context" "fmt" "math" "strconv" @@ -26,10 +27,11 @@ import ( "github.com/pingcap/kvproto/pkg/keyspacepb" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" + "github.com/tikv/pd/pkg/mcs/utils" "github.com/tikv/pd/pkg/mock/mockid" "github.com/tikv/pd/pkg/storage/endpoint" "github.com/tikv/pd/pkg/storage/kv" - "github.com/tikv/pd/server/config" + "github.com/tikv/pd/pkg/utils/typeutil" ) const ( @@ -40,6 +42,8 @@ const ( type keyspaceTestSuite struct { suite.Suite + ctx context.Context + cancel context.CancelFunc manager *Manager } @@ -47,18 +51,49 @@ func TestKeyspaceTestSuite(t *testing.T) { suite.Run(t, new(keyspaceTestSuite)) } +type mockConfig struct { + PreAlloc []string + WaitRegionSplit bool + WaitRegionSplitTimeout typeutil.Duration + CheckRegionSplitInterval typeutil.Duration +} + +func (m *mockConfig) GetPreAlloc() []string { + return m.PreAlloc +} + +func (m *mockConfig) ToWaitRegionSplit() bool { + return m.WaitRegionSplit +} + +func (m *mockConfig) GetWaitRegionSplitTimeout() time.Duration { + return m.WaitRegionSplitTimeout.Duration +} + +func (m *mockConfig) GetCheckRegionSplitInterval() time.Duration { + return m.CheckRegionSplitInterval.Duration +} + func (suite *keyspaceTestSuite) SetupTest() { + suite.ctx, suite.cancel = context.WithCancel(context.Background()) store := endpoint.NewStorageEndpoint(kv.NewMemoryKV(), nil) allocator := mockid.NewIDAllocator() - suite.manager = NewKeyspaceManager(store, nil, allocator, config.KeyspaceConfig{}) + kgm := NewKeyspaceGroupManager(suite.ctx, store, nil, 0) + suite.manager = NewKeyspaceManager(suite.ctx, store, nil, allocator, &mockConfig{}, kgm) + suite.NoError(kgm.Bootstrap(suite.ctx)) suite.NoError(suite.manager.Bootstrap()) } +func (suite *keyspaceTestSuite) TearDownTest() { + suite.cancel() +} + func (suite *keyspaceTestSuite) SetupSuite() { - suite.NoError(failpoint.Enable("github.com/tikv/pd/server/keyspace/skipSplitRegion", "return(true)")) + suite.NoError(failpoint.Enable("github.com/tikv/pd/pkg/keyspace/skipSplitRegion", "return(true)")) } + func (suite *keyspaceTestSuite) TearDownSuite() { - suite.NoError(failpoint.Disable("github.com/tikv/pd/server/keyspace/skipSplitRegion")) + suite.NoError(failpoint.Disable("github.com/tikv/pd/pkg/keyspace/skipSplitRegion")) } func makeCreateKeyspaceRequests(count int) []*CreateKeyspaceRequest { @@ -66,12 +101,13 @@ func makeCreateKeyspaceRequests(count int) []*CreateKeyspaceRequest { requests := make([]*CreateKeyspaceRequest, count) for i := 0; i < count; i++ { requests[i] = &CreateKeyspaceRequest{ - Name: fmt.Sprintf("test_keyspace%d", i), + Name: fmt.Sprintf("test_keyspace_%d", i), Config: map[string]string{ testConfig1: "100", testConfig2: "200", }, - Now: now, + CreateTime: now, + IsPreAlloc: true, // skip wait region split } } return requests @@ -147,8 +183,11 @@ func (suite *keyspaceTestSuite) TestUpdateKeyspaceConfig() { re.Error(err) } // Changing config of DEFAULT keyspace is allowed. - updated, err := manager.UpdateKeyspaceConfig(DefaultKeyspaceName, mutations) + updated, err := manager.UpdateKeyspaceConfig(utils.DefaultKeyspaceName, mutations) re.NoError(err) + // remove auto filled fields + delete(updated.Config, TSOKeyspaceGroupIDKey) + delete(updated.Config, UserKindKey) checkMutations(re, nil, updated.Config, mutations) } @@ -184,7 +223,7 @@ func (suite *keyspaceTestSuite) TestUpdateKeyspaceState() { _, err = manager.UpdateKeyspaceState(createRequest.Name, keyspacepb.KeyspaceState_ENABLED, newTime) re.Error(err) // Changing state of DEFAULT keyspace is not allowed. - _, err = manager.UpdateKeyspaceState(DefaultKeyspaceName, keyspacepb.KeyspaceState_DISABLED, newTime) + _, err = manager.UpdateKeyspaceState(utils.DefaultKeyspaceName, keyspacepb.KeyspaceState_DISABLED, newTime) re.Error(err) } } @@ -293,8 +332,8 @@ func (suite *keyspaceTestSuite) TestUpdateMultipleKeyspace() { // checkCreateRequest verifies a keyspace meta matches a create request. func checkCreateRequest(re *require.Assertions, request *CreateKeyspaceRequest, meta *keyspacepb.KeyspaceMeta) { re.Equal(request.Name, meta.GetName()) - re.Equal(request.Now, meta.GetCreatedAt()) - re.Equal(request.Now, meta.GetStateChangedAt()) + re.Equal(request.CreateTime, meta.GetCreatedAt()) + re.Equal(request.CreateTime, meta.GetStateChangedAt()) re.Equal(keyspacepb.KeyspaceState_ENABLED, meta.GetState()) re.Equal(request.Config, meta.GetConfig()) } @@ -335,3 +374,150 @@ func updateKeyspaceConfig(re *require.Assertions, manager *Manager, name string, oldMeta = updatedMeta } } + +func (suite *keyspaceTestSuite) TestPatrolKeyspaceAssignment() { + re := suite.Require() + // Create a keyspace without any keyspace group. + now := time.Now().Unix() + err := suite.manager.saveNewKeyspace(&keyspacepb.KeyspaceMeta{ + Id: 111, + Name: "111", + State: keyspacepb.KeyspaceState_ENABLED, + CreatedAt: now, + StateChangedAt: now, + }) + re.NoError(err) + // Check if the keyspace is not attached to the default group. + defaultKeyspaceGroup, err := suite.manager.kgm.GetKeyspaceGroupByID(utils.DefaultKeyspaceGroupID) + re.NoError(err) + re.NotNil(defaultKeyspaceGroup) + re.NotContains(defaultKeyspaceGroup.Keyspaces, uint32(111)) + // Patrol the keyspace assignment. + err = suite.manager.PatrolKeyspaceAssignment(0, 0) + re.NoError(err) + // Check if the keyspace is attached to the default group. + defaultKeyspaceGroup, err = suite.manager.kgm.GetKeyspaceGroupByID(utils.DefaultKeyspaceGroupID) + re.NoError(err) + re.NotNil(defaultKeyspaceGroup) + re.Contains(defaultKeyspaceGroup.Keyspaces, uint32(111)) +} + +func (suite *keyspaceTestSuite) TestPatrolKeyspaceAssignmentInBatch() { + re := suite.Require() + // Create some keyspaces without any keyspace group. + for i := 1; i < MaxEtcdTxnOps*2+1; i++ { + now := time.Now().Unix() + err := suite.manager.saveNewKeyspace(&keyspacepb.KeyspaceMeta{ + Id: uint32(i), + Name: strconv.Itoa(i), + State: keyspacepb.KeyspaceState_ENABLED, + CreatedAt: now, + StateChangedAt: now, + }) + re.NoError(err) + } + // Check if all the keyspaces are not attached to the default group. + defaultKeyspaceGroup, err := suite.manager.kgm.GetKeyspaceGroupByID(utils.DefaultKeyspaceGroupID) + re.NoError(err) + re.NotNil(defaultKeyspaceGroup) + for i := 1; i < MaxEtcdTxnOps*2+1; i++ { + re.NotContains(defaultKeyspaceGroup.Keyspaces, uint32(i)) + } + // Patrol the keyspace assignment. + err = suite.manager.PatrolKeyspaceAssignment(0, 0) + re.NoError(err) + // Check if all the keyspaces are attached to the default group. + defaultKeyspaceGroup, err = suite.manager.kgm.GetKeyspaceGroupByID(utils.DefaultKeyspaceGroupID) + re.NoError(err) + re.NotNil(defaultKeyspaceGroup) + for i := 1; i < MaxEtcdTxnOps*2+1; i++ { + re.Contains(defaultKeyspaceGroup.Keyspaces, uint32(i)) + } +} + +func (suite *keyspaceTestSuite) TestPatrolKeyspaceAssignmentWithRange() { + re := suite.Require() + // Create some keyspaces without any keyspace group. + for i := 1; i < MaxEtcdTxnOps*2+1; i++ { + now := time.Now().Unix() + err := suite.manager.saveNewKeyspace(&keyspacepb.KeyspaceMeta{ + Id: uint32(i), + Name: strconv.Itoa(i), + State: keyspacepb.KeyspaceState_ENABLED, + CreatedAt: now, + StateChangedAt: now, + }) + re.NoError(err) + } + // Check if all the keyspaces are not attached to the default group. + defaultKeyspaceGroup, err := suite.manager.kgm.GetKeyspaceGroupByID(utils.DefaultKeyspaceGroupID) + re.NoError(err) + re.NotNil(defaultKeyspaceGroup) + for i := 1; i < MaxEtcdTxnOps*2+1; i++ { + re.NotContains(defaultKeyspaceGroup.Keyspaces, uint32(i)) + } + // Patrol the keyspace assignment with range [MaxEtcdTxnOps/2, MaxEtcdTxnOps/2+MaxEtcdTxnOps+1] + // to make sure the range crossing the boundary of etcd transaction operation limit. + var ( + startKeyspaceID = uint32(MaxEtcdTxnOps / 2) + endKeyspaceID = startKeyspaceID + MaxEtcdTxnOps + 1 + ) + err = suite.manager.PatrolKeyspaceAssignment(startKeyspaceID, endKeyspaceID) + re.NoError(err) + // Check if only the keyspaces within the range are attached to the default group. + defaultKeyspaceGroup, err = suite.manager.kgm.GetKeyspaceGroupByID(utils.DefaultKeyspaceGroupID) + re.NoError(err) + re.NotNil(defaultKeyspaceGroup) + for i := 1; i < MaxEtcdTxnOps*2+1; i++ { + keyspaceID := uint32(i) + if keyspaceID >= startKeyspaceID && keyspaceID <= endKeyspaceID { + re.Contains(defaultKeyspaceGroup.Keyspaces, keyspaceID) + } else { + re.NotContains(defaultKeyspaceGroup.Keyspaces, keyspaceID) + } + } +} + +// Benchmark the keyspace assignment patrol. +func BenchmarkPatrolKeyspaceAssignment1000(b *testing.B) { + benchmarkPatrolKeyspaceAssignmentN(1000, b) +} + +func BenchmarkPatrolKeyspaceAssignment10000(b *testing.B) { + benchmarkPatrolKeyspaceAssignmentN(10000, b) +} + +func BenchmarkPatrolKeyspaceAssignment100000(b *testing.B) { + benchmarkPatrolKeyspaceAssignmentN(100000, b) +} + +func benchmarkPatrolKeyspaceAssignmentN( + n int, b *testing.B, +) { + suite := new(keyspaceTestSuite) + suite.SetT(&testing.T{}) + suite.SetupSuite() + suite.SetupTest() + re := suite.Require() + // Create some keyspaces without any keyspace group. + for i := 1; i <= n; i++ { + now := time.Now().Unix() + err := suite.manager.saveNewKeyspace(&keyspacepb.KeyspaceMeta{ + Id: uint32(i), + Name: strconv.Itoa(i), + State: keyspacepb.KeyspaceState_ENABLED, + CreatedAt: now, + StateChangedAt: now, + }) + re.NoError(err) + } + // Benchmark the keyspace assignment patrol. + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := suite.manager.PatrolKeyspaceAssignment(0, 0) + re.NoError(err) + } + b.StopTimer() + suite.TearDownTest() + suite.TearDownSuite() +} diff --git a/pkg/keyspace/tso_keyspace_group.go b/pkg/keyspace/tso_keyspace_group.go new file mode 100644 index 00000000000..22c265140fd --- /dev/null +++ b/pkg/keyspace/tso_keyspace_group.go @@ -0,0 +1,1151 @@ +// Copyright 2023 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package keyspace + +import ( + "context" + "encoding/json" + "sort" + "strconv" + "strings" + "sync" + "time" + + "github.com/pingcap/errors" + "github.com/pingcap/failpoint" + "github.com/pingcap/kvproto/pkg/tsopb" + "github.com/pingcap/log" + "github.com/tikv/pd/pkg/balancer" + "github.com/tikv/pd/pkg/mcs/discovery" + "github.com/tikv/pd/pkg/mcs/utils" + "github.com/tikv/pd/pkg/slice" + "github.com/tikv/pd/pkg/storage/endpoint" + "github.com/tikv/pd/pkg/storage/kv" + "github.com/tikv/pd/pkg/utils/etcdutil" + "github.com/tikv/pd/pkg/utils/logutil" + "go.etcd.io/etcd/clientv3" + "go.etcd.io/etcd/mvcc/mvccpb" + "go.uber.org/zap" +) + +const ( + defaultBalancerPolicy = balancer.PolicyRoundRobin + allocNodesToKeyspaceGroupsInterval = 1 * time.Second + allocNodesTimeout = 1 * time.Second + allocNodesInterval = 10 * time.Millisecond +) + +const ( + opAdd int = iota + opDelete +) + +// GroupManager is the manager of keyspace group related data. +type GroupManager struct { + ctx context.Context + cancel context.CancelFunc + wg sync.WaitGroup + client *clientv3.Client + clusterID uint64 + + sync.RWMutex + // groups is the cache of keyspace group related information. + // user kind -> keyspace group + groups map[endpoint.UserKind]*indexedHeap + + // store is the storage for keyspace group related information. + store endpoint.KeyspaceGroupStorage + + // nodeBalancer is the balancer for tso nodes. + // TODO: add user kind with different balancer when we ensure where the correspondence between tso node and user kind will be found + nodesBalancer balancer.Balancer[string] + // serviceRegistryMap stores the mapping from the service registry key to the service address. + // Note: it is only used in tsoNodesWatcher. + serviceRegistryMap map[string]string + // tsoNodesWatcher is the watcher for the registered tso servers. + tsoNodesWatcher *etcdutil.LoopWatcher +} + +// NewKeyspaceGroupManager creates a Manager of keyspace group related data. +func NewKeyspaceGroupManager( + ctx context.Context, + store endpoint.KeyspaceGroupStorage, + client *clientv3.Client, + clusterID uint64, +) *GroupManager { + ctx, cancel := context.WithCancel(ctx) + groups := make(map[endpoint.UserKind]*indexedHeap) + for i := 0; i < int(endpoint.UserKindCount); i++ { + groups[endpoint.UserKind(i)] = newIndexedHeap(int(utils.MaxKeyspaceGroupCountInUse)) + } + m := &GroupManager{ + ctx: ctx, + cancel: cancel, + store: store, + groups: groups, + client: client, + clusterID: clusterID, + nodesBalancer: balancer.GenByPolicy[string](defaultBalancerPolicy), + serviceRegistryMap: make(map[string]string), + } + + // If the etcd client is not nil, start the watch loop for the registered tso servers. + // The PD(TSO) Client relies on this info to discover tso servers. + if m.client != nil { + m.initTSONodesWatcher(m.client, m.clusterID) + m.tsoNodesWatcher.StartWatchLoop() + } + return m +} + +// Bootstrap saves default keyspace group info and init group mapping in the memory. +func (m *GroupManager) Bootstrap(ctx context.Context) error { + // Force the membership restriction that the default keyspace must belong to default keyspace group. + // Have no information to specify the distribution of the default keyspace group replicas, so just + // leave the replica/member list empty. The TSO service will assign the default keyspace group replica + // to every tso node/pod by default. + defaultKeyspaceGroup := &endpoint.KeyspaceGroup{ + ID: utils.DefaultKeyspaceGroupID, + UserKind: endpoint.Basic.String(), + Keyspaces: []uint32{utils.DefaultKeyspaceID}, + } + + m.Lock() + defer m.Unlock() + + // Ignore the error if default keyspace group already exists in the storage (e.g. PD restart/recover). + err := m.saveKeyspaceGroups([]*endpoint.KeyspaceGroup{defaultKeyspaceGroup}, false) + if err != nil && err != ErrKeyspaceGroupExists { + return err + } + + // Load all the keyspace groups from the storage and add to the respective userKind groups. + groups, err := m.store.LoadKeyspaceGroups(utils.DefaultKeyspaceGroupID, 0) + if err != nil { + return err + } + for _, group := range groups { + userKind := endpoint.StringUserKind(group.UserKind) + m.groups[userKind].Put(group) + } + + // It will only alloc node when the group manager is on API leader. + if m.client != nil { + m.wg.Add(1) + go m.allocNodesToAllKeyspaceGroups(ctx) + } + return nil +} + +// Close closes the manager. +func (m *GroupManager) Close() { + m.cancel() + m.wg.Wait() +} + +func (m *GroupManager) allocNodesToAllKeyspaceGroups(ctx context.Context) { + defer logutil.LogPanic() + defer m.wg.Done() + ticker := time.NewTicker(allocNodesToKeyspaceGroupsInterval) + failpoint.Inject("acceleratedAllocNodes", func() { + ticker.Stop() + ticker = time.NewTicker(time.Millisecond * 100) + }) + defer ticker.Stop() + log.Info("start to alloc nodes to all keyspace groups") + for { + select { + case <-ctx.Done(): + log.Info("stop to alloc nodes to all keyspace groups") + return + case <-ticker.C: + } + countOfNodes := m.GetNodesCount() + if countOfNodes < utils.DefaultKeyspaceGroupReplicaCount { + continue + } + groups, err := m.store.LoadKeyspaceGroups(utils.DefaultKeyspaceGroupID, 0) + if err != nil { + log.Error("failed to load all keyspace groups", zap.Error(err)) + continue + } + // if the default keyspace is not initialized, we should wait for the default keyspace to be initialized. + if len(groups) == 0 { + continue + } + withError := false + for _, group := range groups { + if len(group.Members) < utils.DefaultKeyspaceGroupReplicaCount { + nodes, err := m.AllocNodesForKeyspaceGroup(group.ID, utils.DefaultKeyspaceGroupReplicaCount) + if err != nil { + withError = true + log.Error("failed to alloc nodes for keyspace group", zap.Uint32("keyspace-group-id", group.ID), zap.Error(err)) + continue + } + group.Members = nodes + } + } + if !withError { + // all keyspace groups have equal or more than default replica count + log.Info("all keyspace groups have equal or more than default replica count, stop to alloc node") + return + } + } +} + +func (m *GroupManager) initTSONodesWatcher(client *clientv3.Client, clusterID uint64) { + tsoServiceKey := discovery.TSOPath(clusterID) + tsoServiceEndKey := clientv3.GetPrefixRangeEnd(tsoServiceKey) + + putFn := func(kv *mvccpb.KeyValue) error { + s := &discovery.ServiceRegistryEntry{} + if err := json.Unmarshal(kv.Value, s); err != nil { + log.Warn("failed to unmarshal service registry entry", + zap.String("event-kv-key", string(kv.Key)), zap.Error(err)) + return err + } + m.nodesBalancer.Put(s.ServiceAddr) + m.serviceRegistryMap[string(kv.Key)] = s.ServiceAddr + return nil + } + deleteFn := func(kv *mvccpb.KeyValue) error { + key := string(kv.Key) + if serviceAddr, ok := m.serviceRegistryMap[key]; ok { + delete(m.serviceRegistryMap, key) + m.nodesBalancer.Delete(serviceAddr) + return nil + } + return errors.Errorf("failed to find the service address for key %s", key) + } + + m.tsoNodesWatcher = etcdutil.NewLoopWatcher( + m.ctx, + &m.wg, + client, + "tso-nodes-watcher", + tsoServiceKey, + putFn, + deleteFn, + func() error { return nil }, + clientv3.WithRange(tsoServiceEndKey), + ) +} + +// CreateKeyspaceGroups creates keyspace groups. +func (m *GroupManager) CreateKeyspaceGroups(keyspaceGroups []*endpoint.KeyspaceGroup) error { + m.Lock() + defer m.Unlock() + if err := m.saveKeyspaceGroups(keyspaceGroups, false); err != nil { + return err + } + + for _, keyspaceGroup := range keyspaceGroups { + userKind := endpoint.StringUserKind(keyspaceGroup.UserKind) + m.groups[userKind].Put(keyspaceGroup) + } + + return nil +} + +// GetTSOServiceAddrs gets all TSO service addresses. +func (m *GroupManager) GetTSOServiceAddrs() []string { + if m == nil || m.nodesBalancer == nil { + return nil + } + return m.nodesBalancer.GetAll() +} + +// GetKeyspaceGroups gets keyspace groups from the start ID with limit. +// If limit is 0, it will load all keyspace groups from the start ID. +func (m *GroupManager) GetKeyspaceGroups(startID uint32, limit int) ([]*endpoint.KeyspaceGroup, error) { + return m.store.LoadKeyspaceGroups(startID, limit) +} + +// GetKeyspaceGroupByID returns the keyspace group by ID. +func (m *GroupManager) GetKeyspaceGroupByID(id uint32) (*endpoint.KeyspaceGroup, error) { + var ( + kg *endpoint.KeyspaceGroup + err error + ) + + if err := m.store.RunInTxn(m.ctx, func(txn kv.Txn) error { + kg, err = m.store.LoadKeyspaceGroup(txn, id) + if err != nil { + return err + } + return nil + }); err != nil { + return nil, err + } + return kg, nil +} + +// DeleteKeyspaceGroupByID deletes the keyspace group by ID. +func (m *GroupManager) DeleteKeyspaceGroupByID(id uint32) (*endpoint.KeyspaceGroup, error) { + var ( + kg *endpoint.KeyspaceGroup + err error + ) + + m.Lock() + defer m.Unlock() + if err := m.store.RunInTxn(m.ctx, func(txn kv.Txn) error { + kg, err = m.store.LoadKeyspaceGroup(txn, id) + if err != nil { + return err + } + if kg == nil { + return nil + } + if kg.IsSplitting() { + return ErrKeyspaceGroupInSplit(id) + } + return m.store.DeleteKeyspaceGroup(txn, id) + }); err != nil { + return nil, err + } + + userKind := endpoint.StringUserKind(kg.UserKind) + // TODO: move out the keyspace to another group + // we don't need the keyspace group as the return value + m.groups[userKind].Remove(id) + + return kg, nil +} + +// saveKeyspaceGroups will try to save the given keyspace groups into the storage. +// If any keyspace group already exists and `overwrite` is false, it will return ErrKeyspaceGroupExists. +func (m *GroupManager) saveKeyspaceGroups(keyspaceGroups []*endpoint.KeyspaceGroup, overwrite bool) error { + return m.store.RunInTxn(m.ctx, func(txn kv.Txn) error { + for _, keyspaceGroup := range keyspaceGroups { + // Check if keyspace group has already existed. + oldKG, err := m.store.LoadKeyspaceGroup(txn, keyspaceGroup.ID) + if err != nil { + return err + } + if oldKG != nil && !overwrite { + return ErrKeyspaceGroupExists + } + if oldKG.IsSplitting() && overwrite { + return ErrKeyspaceGroupInSplit(keyspaceGroup.ID) + } + if oldKG.IsMerging() && overwrite { + return ErrKeyspaceGroupInMerging(keyspaceGroup.ID) + } + newKG := &endpoint.KeyspaceGroup{ + ID: keyspaceGroup.ID, + UserKind: keyspaceGroup.UserKind, + Members: keyspaceGroup.Members, + Keyspaces: keyspaceGroup.Keyspaces, + } + err = m.store.SaveKeyspaceGroup(txn, newKG) + if err != nil { + return err + } + } + return nil + }) +} + +// GetKeyspaceConfigByKind returns the keyspace config for the given user kind. +func (m *GroupManager) GetKeyspaceConfigByKind(userKind endpoint.UserKind) (map[string]string, error) { + // when server is not in API mode, we don't need to return the keyspace config + if m == nil { + return map[string]string{}, nil + } + m.RLock() + defer m.RUnlock() + return m.getKeyspaceConfigByKindLocked(userKind) +} + +func (m *GroupManager) getKeyspaceConfigByKindLocked(userKind endpoint.UserKind) (map[string]string, error) { + groups, ok := m.groups[userKind] + if !ok { + return map[string]string{}, errors.Errorf("user kind %s not found", userKind) + } + kg := groups.Top() + if kg == nil { + return map[string]string{}, errors.Errorf("no keyspace group for user kind %s", userKind) + } + id := strconv.FormatUint(uint64(kg.ID), 10) + config := map[string]string{ + UserKindKey: userKind.String(), + TSOKeyspaceGroupIDKey: id, + } + return config, nil +} + +// GetGroupByKeyspaceID returns the keyspace group ID for the given keyspace ID. +func (m *GroupManager) GetGroupByKeyspaceID(id uint32) (uint32, error) { + m.RLock() + defer m.RUnlock() + for _, groups := range m.groups { + for _, group := range groups.GetAll() { + if slice.Contains(group.Keyspaces, id) { + return group.ID, nil + } + } + } + return 0, ErrKeyspaceNotInAnyKeyspaceGroup +} + +var failpointOnce sync.Once + +// UpdateKeyspaceForGroup updates the keyspace field for the keyspace group. +func (m *GroupManager) UpdateKeyspaceForGroup(userKind endpoint.UserKind, groupID string, keyspaceID uint32, mutation int) error { + // when server is not in API mode, we don't need to update the keyspace for keyspace group + if m == nil { + return nil + } + id, err := strconv.ParseUint(groupID, 10, 64) + if err != nil { + return err + } + + failpoint.Inject("externalAllocNode", func(val failpoint.Value) { + failpointOnce.Do(func() { + addrs := val.(string) + m.SetNodesForKeyspaceGroup(utils.DefaultKeyspaceGroupID, strings.Split(addrs, ",")) + }) + }) + m.Lock() + defer m.Unlock() + return m.updateKeyspaceForGroupLocked(userKind, id, keyspaceID, mutation) +} + +func (m *GroupManager) updateKeyspaceForGroupLocked(userKind endpoint.UserKind, groupID uint64, keyspaceID uint32, mutation int) error { + kg := m.groups[userKind].Get(uint32(groupID)) + if kg == nil { + return ErrKeyspaceGroupNotExists(uint32(groupID)) + } + if kg.IsSplitting() { + return ErrKeyspaceGroupInSplit(uint32(groupID)) + } + if kg.IsMerging() { + return ErrKeyspaceGroupInMerging(uint32(groupID)) + } + + changed := false + + switch mutation { + case opAdd: + if !slice.Contains(kg.Keyspaces, keyspaceID) { + kg.Keyspaces = append(kg.Keyspaces, keyspaceID) + changed = true + } + case opDelete: + lenOfKeyspaces := len(kg.Keyspaces) + kg.Keyspaces = slice.Remove(kg.Keyspaces, keyspaceID) + if lenOfKeyspaces != len(kg.Keyspaces) { + changed = true + } + } + + if changed { + if err := m.saveKeyspaceGroups([]*endpoint.KeyspaceGroup{kg}, true); err != nil { + return err + } + m.groups[userKind].Put(kg) + } + return nil +} + +// UpdateKeyspaceGroup updates the keyspace group. +func (m *GroupManager) UpdateKeyspaceGroup(oldGroupID, newGroupID string, oldUserKind, newUserKind endpoint.UserKind, keyspaceID uint32) error { + // when server is not in API mode, we don't need to update the keyspace group + if m == nil { + return nil + } + oldID, err := strconv.ParseUint(oldGroupID, 10, 64) + if err != nil { + return err + } + newID, err := strconv.ParseUint(newGroupID, 10, 64) + if err != nil { + return err + } + + m.Lock() + defer m.Unlock() + oldKG := m.groups[oldUserKind].Get(uint32(oldID)) + if oldKG == nil { + return errors.Errorf("keyspace group %s not found in %s group", oldGroupID, oldUserKind) + } + newKG := m.groups[newUserKind].Get(uint32(newID)) + if newKG == nil { + return errors.Errorf("keyspace group %s not found in %s group", newGroupID, newUserKind) + } + if oldKG.IsSplitting() { + return ErrKeyspaceGroupInSplit(uint32(oldID)) + } else if newKG.IsSplitting() { + return ErrKeyspaceGroupInSplit(uint32(newID)) + } else if oldKG.IsMerging() { + return ErrKeyspaceGroupInMerging(uint32(oldID)) + } else if newKG.IsMerging() { + return ErrKeyspaceGroupInMerging(uint32(newID)) + } + + var updateOld, updateNew bool + if !slice.Contains(newKG.Keyspaces, keyspaceID) { + newKG.Keyspaces = append(newKG.Keyspaces, keyspaceID) + updateNew = true + } + + lenOfOldKeyspaces := len(oldKG.Keyspaces) + oldKG.Keyspaces = slice.Remove(oldKG.Keyspaces, keyspaceID) + if lenOfOldKeyspaces != len(oldKG.Keyspaces) { + updateOld = true + } + + if err := m.saveKeyspaceGroups([]*endpoint.KeyspaceGroup{oldKG, newKG}, true); err != nil { + return err + } + + if updateOld { + m.groups[oldUserKind].Put(oldKG) + } + + if updateNew { + m.groups[newUserKind].Put(newKG) + } + + return nil +} + +// SplitKeyspaceGroupByID splits the keyspace group by ID into a new keyspace group with the given new ID. +// And the keyspaces in the old keyspace group will be moved to the new keyspace group. +func (m *GroupManager) SplitKeyspaceGroupByID( + splitSourceID, splitTargetID uint32, + keyspaces []uint32, keyspaceIDRange ...uint32, +) error { + var splitSourceKg, splitTargetKg *endpoint.KeyspaceGroup + m.Lock() + defer m.Unlock() + if err := m.store.RunInTxn(m.ctx, func(txn kv.Txn) (err error) { + // Load the old keyspace group first. + splitSourceKg, err = m.store.LoadKeyspaceGroup(txn, splitSourceID) + if err != nil { + return err + } + if splitSourceKg == nil { + return ErrKeyspaceGroupNotExists(splitSourceID) + } + // A keyspace group can not take part in multiple split processes. + if splitSourceKg.IsSplitting() { + return ErrKeyspaceGroupInSplit(splitSourceID) + } + // A keyspace group can not be split when it is in merging. + if splitSourceKg.IsMerging() { + return ErrKeyspaceGroupInMerging(splitSourceID) + } + // Build the new keyspace groups for split source and target. + var startKeyspaceID, endKeyspaceID uint32 + if len(keyspaceIDRange) >= 2 { + startKeyspaceID, endKeyspaceID = keyspaceIDRange[0], keyspaceIDRange[1] + } + splitSourceKeyspaces, splitTargetKeyspaces, err := buildSplitKeyspaces( + splitSourceKg.Keyspaces, keyspaces, startKeyspaceID, endKeyspaceID) + if err != nil { + return err + } + // Check if the source keyspace group has enough replicas. + if len(splitSourceKg.Members) < utils.DefaultKeyspaceGroupReplicaCount { + return ErrKeyspaceGroupNotEnoughReplicas + } + // Check if the new keyspace group already exists. + splitTargetKg, err = m.store.LoadKeyspaceGroup(txn, splitTargetID) + if err != nil { + return err + } + if splitTargetKg != nil { + return ErrKeyspaceGroupExists + } + // Update the old keyspace group. + splitSourceKg.Keyspaces = splitSourceKeyspaces + splitSourceKg.SplitState = &endpoint.SplitState{ + SplitSource: splitSourceKg.ID, + } + if err = m.store.SaveKeyspaceGroup(txn, splitSourceKg); err != nil { + return err + } + splitTargetKg = &endpoint.KeyspaceGroup{ + ID: splitTargetID, + // Keep the same user kind and members as the old keyspace group. + UserKind: splitSourceKg.UserKind, + Members: splitSourceKg.Members, + Keyspaces: splitTargetKeyspaces, + SplitState: &endpoint.SplitState{ + SplitSource: splitSourceKg.ID, + }, + } + // Create the new split keyspace group. + return m.store.SaveKeyspaceGroup(txn, splitTargetKg) + }); err != nil { + return err + } + // Update the keyspace group cache. + m.groups[endpoint.StringUserKind(splitSourceKg.UserKind)].Put(splitSourceKg) + m.groups[endpoint.StringUserKind(splitTargetKg.UserKind)].Put(splitTargetKg) + return nil +} + +func buildSplitKeyspaces( + // `old` is the original keyspace list which will be split out, + // `new` is the keyspace list which will be split from the old keyspace list. + old, new []uint32, + startKeyspaceID, endKeyspaceID uint32, +) ([]uint32, []uint32, error) { + oldNum, newNum := len(old), len(new) + // Split according to the new keyspace list. + if newNum != 0 { + if newNum > oldNum { + return nil, nil, ErrKeyspaceNotInKeyspaceGroup + } + var ( + oldKeyspaceMap = make(map[uint32]struct{}, oldNum) + newKeyspaceMap = make(map[uint32]struct{}, newNum) + ) + for _, keyspace := range old { + oldKeyspaceMap[keyspace] = struct{}{} + } + for _, keyspace := range new { + if keyspace == utils.DefaultKeyspaceID { + return nil, nil, ErrModifyDefaultKeyspace + } + if _, ok := oldKeyspaceMap[keyspace]; !ok { + return nil, nil, ErrKeyspaceNotInKeyspaceGroup + } + newKeyspaceMap[keyspace] = struct{}{} + } + // Get the split keyspace list for the old keyspace group. + oldSplit := make([]uint32, 0, oldNum-newNum) + for _, keyspace := range old { + if _, ok := newKeyspaceMap[keyspace]; !ok { + oldSplit = append(oldSplit, keyspace) + } + } + // If newNum != len(newKeyspaceMap), it means the provided new keyspace list contains + // duplicate keyspaces, and we need to dedup them (https://github.com/tikv/pd/issues/6687); + // otherwise, we can just return the old split and new keyspace list. + if newNum == len(newKeyspaceMap) { + return oldSplit, new, nil + } + newSplit := make([]uint32, 0, len(newKeyspaceMap)) + for keyspace := range newKeyspaceMap { + newSplit = append(newSplit, keyspace) + } + return oldSplit, newSplit, nil + } + // Split according to the start and end keyspace ID. + if startKeyspaceID == 0 && endKeyspaceID == 0 { + return nil, nil, ErrKeyspaceNotInKeyspaceGroup + } + var ( + newSplit = make([]uint32, 0, oldNum) + newKeyspaceMap = make(map[uint32]struct{}, newNum) + ) + for _, keyspace := range old { + if keyspace == utils.DefaultKeyspaceID { + // The source keyspace group must be the default keyspace group and we always keep the default + // keyspace in the default keyspace group. + continue + } + if startKeyspaceID <= keyspace && keyspace <= endKeyspaceID { + newSplit = append(newSplit, keyspace) + newKeyspaceMap[keyspace] = struct{}{} + } + } + // Get the split keyspace list for the old keyspace group. + oldSplit := make([]uint32, 0, oldNum-len(newSplit)) + for _, keyspace := range old { + if _, ok := newKeyspaceMap[keyspace]; !ok { + oldSplit = append(oldSplit, keyspace) + } + } + return oldSplit, newSplit, nil +} + +// FinishSplitKeyspaceByID finishes the split keyspace group by the split target ID. +func (m *GroupManager) FinishSplitKeyspaceByID(splitTargetID uint32) error { + var splitTargetKg, splitSourceKg *endpoint.KeyspaceGroup + m.Lock() + defer m.Unlock() + if err := m.store.RunInTxn(m.ctx, func(txn kv.Txn) (err error) { + // Load the split target keyspace group first. + splitTargetKg, err = m.store.LoadKeyspaceGroup(txn, splitTargetID) + if err != nil { + return err + } + if splitTargetKg == nil { + return ErrKeyspaceGroupNotExists(splitTargetID) + } + // Check if it's in the split state. + if !splitTargetKg.IsSplitTarget() { + return ErrKeyspaceGroupNotInSplit(splitTargetID) + } + // Load the split source keyspace group then. + splitSourceKg, err = m.store.LoadKeyspaceGroup(txn, splitTargetKg.SplitSource()) + if err != nil { + return err + } + if splitSourceKg == nil { + return ErrKeyspaceGroupNotExists(splitTargetKg.SplitSource()) + } + if !splitSourceKg.IsSplitSource() { + return ErrKeyspaceGroupNotInSplit(splitTargetKg.SplitSource()) + } + splitTargetKg.SplitState = nil + splitSourceKg.SplitState = nil + err = m.store.SaveKeyspaceGroup(txn, splitTargetKg) + if err != nil { + return err + } + return m.store.SaveKeyspaceGroup(txn, splitSourceKg) + }); err != nil { + return err + } + // Update the keyspace group cache. + m.groups[endpoint.StringUserKind(splitTargetKg.UserKind)].Put(splitTargetKg) + m.groups[endpoint.StringUserKind(splitSourceKg.UserKind)].Put(splitSourceKg) + log.Info("finish split keyspace group", zap.Uint32("split-source-id", splitSourceKg.ID), zap.Uint32("split-target-id", splitTargetID)) + return nil +} + +// GetNodesCount returns the count of nodes. +func (m *GroupManager) GetNodesCount() int { + if m.nodesBalancer == nil { + return 0 + } + return m.nodesBalancer.Len() +} + +// AllocNodesForKeyspaceGroup allocates nodes for the keyspace group. +func (m *GroupManager) AllocNodesForKeyspaceGroup(id uint32, desiredReplicaCount int) ([]endpoint.KeyspaceGroupMember, error) { + m.Lock() + defer m.Unlock() + ctx, cancel := context.WithTimeout(m.ctx, allocNodesTimeout) + defer cancel() + ticker := time.NewTicker(allocNodesInterval) + defer ticker.Stop() + + var kg *endpoint.KeyspaceGroup + nodes := make([]endpoint.KeyspaceGroupMember, 0, desiredReplicaCount) + err := m.store.RunInTxn(m.ctx, func(txn kv.Txn) error { + var err error + kg, err = m.store.LoadKeyspaceGroup(txn, id) + if err != nil { + return err + } + if kg == nil { + return ErrKeyspaceGroupNotExists(id) + } + if kg.IsSplitting() { + return ErrKeyspaceGroupInSplit(id) + } + if kg.IsMerging() { + return ErrKeyspaceGroupInMerging(id) + } + exists := make(map[string]struct{}) + for _, member := range kg.Members { + exists[member.Address] = struct{}{} + nodes = append(nodes, member) + } + if len(exists) >= desiredReplicaCount { + return nil + } + for len(exists) < desiredReplicaCount { + select { + case <-ctx.Done(): + return nil + case <-ticker.C: + } + countOfNodes := m.GetNodesCount() + if countOfNodes < desiredReplicaCount || countOfNodes == 0 { // double check + return ErrNoAvailableNode + } + addr := m.nodesBalancer.Next() + if addr == "" { + return ErrNoAvailableNode + } + if _, ok := exists[addr]; ok { + continue + } + exists[addr] = struct{}{} + nodes = append(nodes, endpoint.KeyspaceGroupMember{ + Address: addr, + Priority: utils.DefaultKeyspaceGroupReplicaPriority, + }) + } + kg.Members = nodes + return m.store.SaveKeyspaceGroup(txn, kg) + }) + if err != nil { + return nil, err + } + m.groups[endpoint.StringUserKind(kg.UserKind)].Put(kg) + log.Info("alloc nodes for keyspace group", + zap.Uint32("keyspace-group-id", id), + zap.Reflect("nodes", nodes)) + return nodes, nil +} + +// SetNodesForKeyspaceGroup sets the nodes for the keyspace group. +func (m *GroupManager) SetNodesForKeyspaceGroup(id uint32, nodes []string) error { + m.Lock() + defer m.Unlock() + var kg *endpoint.KeyspaceGroup + err := m.store.RunInTxn(m.ctx, func(txn kv.Txn) error { + var err error + kg, err = m.store.LoadKeyspaceGroup(txn, id) + if err != nil { + return err + } + if kg == nil { + return ErrKeyspaceGroupNotExists(id) + } + if kg.IsSplitting() { + return ErrKeyspaceGroupInSplit(id) + } + if kg.IsMerging() { + return ErrKeyspaceGroupInMerging(id) + } + members := make([]endpoint.KeyspaceGroupMember, 0, len(nodes)) + for _, node := range nodes { + members = append(members, endpoint.KeyspaceGroupMember{ + Address: node, + Priority: utils.DefaultKeyspaceGroupReplicaPriority, + }) + } + kg.Members = members + return m.store.SaveKeyspaceGroup(txn, kg) + }) + if err != nil { + return err + } + m.groups[endpoint.StringUserKind(kg.UserKind)].Put(kg) + return nil +} + +// SetPriorityForKeyspaceGroup sets the priority of node for the keyspace group. +func (m *GroupManager) SetPriorityForKeyspaceGroup(id uint32, node string, priority int) error { + m.Lock() + defer m.Unlock() + var kg *endpoint.KeyspaceGroup + err := m.store.RunInTxn(m.ctx, func(txn kv.Txn) error { + var err error + kg, err = m.store.LoadKeyspaceGroup(txn, id) + if err != nil { + return err + } + if kg == nil { + return ErrKeyspaceGroupNotExists(id) + } + if kg.IsSplitting() { + return ErrKeyspaceGroupInSplit(id) + } + if kg.IsMerging() { + return ErrKeyspaceGroupInMerging(id) + } + inKeyspaceGroup := false + members := make([]endpoint.KeyspaceGroupMember, 0, len(kg.Members)) + for _, member := range kg.Members { + if member.Address == node { + inKeyspaceGroup = true + member.Priority = priority + } + members = append(members, member) + } + if !inKeyspaceGroup { + return ErrNodeNotInKeyspaceGroup + } + kg.Members = members + return m.store.SaveKeyspaceGroup(txn, kg) + }) + if err != nil { + return err + } + m.groups[endpoint.StringUserKind(kg.UserKind)].Put(kg) + return nil +} + +// IsExistNode checks if the node exists. +func (m *GroupManager) IsExistNode(addr string) bool { + nodes := m.nodesBalancer.GetAll() + for _, node := range nodes { + if node == addr { + return true + } + } + return false +} + +// MergeKeyspaceGroups merges the keyspace group in the list into the target keyspace group. +func (m *GroupManager) MergeKeyspaceGroups(mergeTargetID uint32, mergeList []uint32) error { + mergeListNum := len(mergeList) + if mergeListNum == 0 { + return nil + } + // The transaction below will: + // - Load and delete the keyspace groups in the merge list. + // - Load and update the target keyspace group. + // So we pre-check the number of operations to avoid exceeding the maximum number of etcd transaction. + if (mergeListNum+1)*2 > MaxEtcdTxnOps { + return ErrExceedMaxEtcdTxnOps + } + if slice.Contains(mergeList, utils.DefaultKeyspaceGroupID) { + return ErrModifyDefaultKeyspaceGroup + } + var ( + groups = make(map[uint32]*endpoint.KeyspaceGroup, mergeListNum+1) + mergeTargetKg *endpoint.KeyspaceGroup + ) + m.Lock() + defer m.Unlock() + if err := m.store.RunInTxn(m.ctx, func(txn kv.Txn) (err error) { + // Load and check all keyspace groups first. + for _, kgID := range append(mergeList, mergeTargetID) { + kg, err := m.store.LoadKeyspaceGroup(txn, kgID) + if err != nil { + return err + } + if kg == nil { + return ErrKeyspaceGroupNotExists(kgID) + } + // A keyspace group can not be merged if it's in splitting. + if kg.IsSplitting() { + return ErrKeyspaceGroupInSplit(kgID) + } + // A keyspace group can not be split when it is in merging. + if kg.IsMerging() { + return ErrKeyspaceGroupInMerging(kgID) + } + groups[kgID] = kg + } + // Build the new keyspaces for the merge target keyspace group. + mergeTargetKg = groups[mergeTargetID] + keyspaces := make(map[uint32]struct{}) + for _, keyspace := range mergeTargetKg.Keyspaces { + keyspaces[keyspace] = struct{}{} + } + for _, kgID := range mergeList { + kg := groups[kgID] + for _, keyspace := range kg.Keyspaces { + keyspaces[keyspace] = struct{}{} + } + } + mergedKeyspaces := make([]uint32, 0, len(keyspaces)) + for keyspace := range keyspaces { + mergedKeyspaces = append(mergedKeyspaces, keyspace) + } + sort.Slice(mergedKeyspaces, func(i, j int) bool { + return mergedKeyspaces[i] < mergedKeyspaces[j] + }) + mergeTargetKg.Keyspaces = mergedKeyspaces + // Update the merge state of the target keyspace group. + mergeTargetKg.MergeState = &endpoint.MergeState{ + MergeList: mergeList, + } + err = m.store.SaveKeyspaceGroup(txn, mergeTargetKg) + if err != nil { + return err + } + // Delete the keyspace groups in merge list and move the keyspaces in it to the target keyspace group. + for _, kgID := range mergeList { + if err := m.store.DeleteKeyspaceGroup(txn, kgID); err != nil { + return err + } + } + return nil + }); err != nil { + return err + } + // Update the keyspace group cache. + m.groups[endpoint.StringUserKind(mergeTargetKg.UserKind)].Put(mergeTargetKg) + for _, kgID := range mergeList { + kg := groups[kgID] + m.groups[endpoint.StringUserKind(kg.UserKind)].Remove(kgID) + } + return nil +} + +// FinishMergeKeyspaceByID finishes the merging keyspace group by the merge target ID. +func (m *GroupManager) FinishMergeKeyspaceByID(mergeTargetID uint32) error { + var ( + mergeTargetKg *endpoint.KeyspaceGroup + mergeList []uint32 + ) + m.Lock() + defer m.Unlock() + if err := m.store.RunInTxn(m.ctx, func(txn kv.Txn) (err error) { + // Load the merge target keyspace group first. + mergeTargetKg, err = m.store.LoadKeyspaceGroup(txn, mergeTargetID) + if err != nil { + return err + } + if mergeTargetKg == nil { + return ErrKeyspaceGroupNotExists(mergeTargetID) + } + // Check if it's in the merging state. + if !mergeTargetKg.IsMergeTarget() { + return ErrKeyspaceGroupNotInMerging(mergeTargetID) + } + // Make sure all merging keyspace groups are deleted. + for _, kgID := range mergeTargetKg.MergeState.MergeList { + kg, err := m.store.LoadKeyspaceGroup(txn, kgID) + if err != nil { + return err + } + if kg != nil { + return ErrKeyspaceGroupNotInMerging(kgID) + } + } + mergeList = mergeTargetKg.MergeState.MergeList + mergeTargetKg.MergeState = nil + return m.store.SaveKeyspaceGroup(txn, mergeTargetKg) + }); err != nil { + return err + } + // Update the keyspace group cache. + m.groups[endpoint.StringUserKind(mergeTargetKg.UserKind)].Put(mergeTargetKg) + log.Info("finish merge keyspace group", + zap.Uint32("merge-target-id", mergeTargetKg.ID), + zap.Reflect("merge-list", mergeList)) + return nil +} + +// MergeAllIntoDefaultKeyspaceGroup merges all other keyspace groups into the default keyspace group. +func (m *GroupManager) MergeAllIntoDefaultKeyspaceGroup() error { + defer logutil.LogPanic() + // Since we don't take the default keyspace group into account, + // the number of unmerged keyspace groups is -1. + unmergedGroupNum := -1 + // Calculate the total number of keyspace groups to merge. + for _, groups := range m.groups { + unmergedGroupNum += groups.Len() + } + mergedGroupNum := 0 + // Start to merge all keyspace groups into the default one. + for userKind, groups := range m.groups { + mergeNum := groups.Len() + log.Info("start to merge all keyspace groups into the default one", + zap.Stringer("user-kind", userKind), + zap.Int("merge-num", mergeNum), + zap.Int("merged-group-num", mergedGroupNum), + zap.Int("unmerged-group-num", unmergedGroupNum)) + if mergeNum == 0 { + continue + } + var ( + maxBatchSize = MaxEtcdTxnOps/2 - 1 + groupsToMerge = make([]uint32, 0, maxBatchSize) + ) + for idx, group := range groups.GetAll() { + if group.ID == utils.DefaultKeyspaceGroupID { + continue + } + groupsToMerge = append(groupsToMerge, group.ID) + if len(groupsToMerge) < maxBatchSize && idx < mergeNum-1 { + continue + } + log.Info("merge keyspace groups into the default one", + zap.Int("index", idx), + zap.Int("batch-size", len(groupsToMerge)), + zap.Int("merge-num", mergeNum), + zap.Int("merged-group-num", mergedGroupNum), + zap.Int("unmerged-group-num", unmergedGroupNum)) + // Reach the batch size, merge them into the default keyspace group. + if err := m.MergeKeyspaceGroups(utils.DefaultKeyspaceGroupID, groupsToMerge); err != nil { + log.Error("failed to merge all keyspace groups into the default one", + zap.Int("index", idx), + zap.Int("batch-size", len(groupsToMerge)), + zap.Int("merge-num", mergeNum), + zap.Int("merged-group-num", mergedGroupNum), + zap.Int("unmerged-group-num", unmergedGroupNum), + zap.Error(err)) + return err + } + // Wait for the merge to finish. + ctx, cancel := context.WithTimeout(m.ctx, time.Minute) + ticker := time.NewTicker(time.Second) + checkLoop: + for { + select { + case <-ctx.Done(): + log.Info("cancel merging all keyspace groups into the default one", + zap.Int("index", idx), + zap.Int("batch-size", len(groupsToMerge)), + zap.Int("merge-num", mergeNum), + zap.Int("merged-group-num", mergedGroupNum), + zap.Int("unmerged-group-num", unmergedGroupNum)) + cancel() + ticker.Stop() + return nil + case <-ticker.C: + kg, err := m.GetKeyspaceGroupByID(utils.DefaultKeyspaceGroupID) + if err != nil { + log.Error("failed to check the default keyspace group merge state", + zap.Int("index", idx), + zap.Int("batch-size", len(groupsToMerge)), + zap.Int("merge-num", mergeNum), + zap.Int("merged-group-num", mergedGroupNum), + zap.Int("unmerged-group-num", unmergedGroupNum), + zap.Error(err)) + cancel() + ticker.Stop() + return err + } + if !kg.IsMergeTarget() { + break checkLoop + } + } + } + cancel() + ticker.Stop() + mergedGroupNum += len(groupsToMerge) + unmergedGroupNum -= len(groupsToMerge) + groupsToMerge = groupsToMerge[:0] + } + } + log.Info("finish merging all keyspace groups into the default one", + zap.Int("merged-group-num", mergedGroupNum), + zap.Int("unmerged-group-num", unmergedGroupNum)) + return nil +} + +// GetKeyspaceGroupPrimaryByID returns the primary node of the keyspace group by ID. +func (m *GroupManager) GetKeyspaceGroupPrimaryByID(id uint32) (string, error) { + // check if the keyspace group exists + kg, err := m.GetKeyspaceGroupByID(id) + if err != nil { + return "", err + } + if kg == nil { + return "", ErrKeyspaceGroupNotExists(id) + } + + rootPath := endpoint.TSOSvcRootPath(m.clusterID) + primaryPath := endpoint.KeyspaceGroupPrimaryPath(rootPath, id) + leader := &tsopb.Participant{} + ok, _, err := etcdutil.GetProtoMsgWithModRev(m.client, primaryPath, leader) + if err != nil { + return "", err + } + if !ok { + return "", ErrKeyspaceGroupPrimaryNotFound + } + // The format of leader name is address-groupID. + contents := strings.Split(leader.GetName(), "-") + return contents[0], err +} diff --git a/pkg/keyspace/tso_keyspace_group_test.go b/pkg/keyspace/tso_keyspace_group_test.go new file mode 100644 index 00000000000..5f01146eb96 --- /dev/null +++ b/pkg/keyspace/tso_keyspace_group_test.go @@ -0,0 +1,563 @@ +// Copyright 2023 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package keyspace + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + "github.com/tikv/pd/pkg/mcs/utils" + "github.com/tikv/pd/pkg/mock/mockcluster" + "github.com/tikv/pd/pkg/mock/mockconfig" + "github.com/tikv/pd/pkg/mock/mockid" + "github.com/tikv/pd/pkg/storage/endpoint" + "github.com/tikv/pd/pkg/storage/kv" +) + +type keyspaceGroupTestSuite struct { + suite.Suite + ctx context.Context + cancel context.CancelFunc + kgm *GroupManager + kg *Manager +} + +func TestKeyspaceGroupTestSuite(t *testing.T) { + suite.Run(t, new(keyspaceGroupTestSuite)) +} + +func (suite *keyspaceGroupTestSuite) SetupTest() { + suite.ctx, suite.cancel = context.WithCancel(context.Background()) + store := endpoint.NewStorageEndpoint(kv.NewMemoryKV(), nil) + suite.kgm = NewKeyspaceGroupManager(suite.ctx, store, nil, 0) + idAllocator := mockid.NewIDAllocator() + cluster := mockcluster.NewCluster(suite.ctx, mockconfig.NewTestOptions()) + suite.kg = NewKeyspaceManager(suite.ctx, store, cluster, idAllocator, &mockConfig{}, suite.kgm) + suite.NoError(suite.kgm.Bootstrap(suite.ctx)) +} + +func (suite *keyspaceGroupTestSuite) TearDownTest() { + suite.cancel() +} + +func (suite *keyspaceGroupTestSuite) TestKeyspaceGroupOperations() { + re := suite.Require() + + keyspaceGroups := []*endpoint.KeyspaceGroup{ + { + ID: uint32(1), + UserKind: endpoint.Standard.String(), + }, + { + ID: uint32(2), + UserKind: endpoint.Standard.String(), + Keyspaces: []uint32{111, 222, 333}, + }, + { + ID: uint32(3), + UserKind: endpoint.Standard.String(), + }, + } + err := suite.kgm.CreateKeyspaceGroups(keyspaceGroups) + re.NoError(err) + // list all keyspace groups + kgs, err := suite.kgm.GetKeyspaceGroups(uint32(0), 0) + re.NoError(err) + re.Len(kgs, 4) + // list part of keyspace groups + kgs, err = suite.kgm.GetKeyspaceGroups(uint32(1), 2) + re.NoError(err) + re.Len(kgs, 2) + // get the default keyspace group + kg, err := suite.kgm.GetKeyspaceGroupByID(utils.DefaultKeyspaceGroupID) + re.NoError(err) + re.Equal(uint32(0), kg.ID) + re.Equal(endpoint.Basic.String(), kg.UserKind) + re.False(kg.IsSplitting()) + // get the keyspace group 3 + kg, err = suite.kgm.GetKeyspaceGroupByID(3) + re.NoError(err) + re.Equal(uint32(3), kg.ID) + re.Equal(endpoint.Standard.String(), kg.UserKind) + re.False(kg.IsSplitting()) + // remove the keyspace group 3 + kg, err = suite.kgm.DeleteKeyspaceGroupByID(3) + re.NoError(err) + re.Equal(uint32(3), kg.ID) + // get non-existing keyspace group + kg, err = suite.kgm.GetKeyspaceGroupByID(3) + re.NoError(err) + re.Empty(kg) + // create an existing keyspace group + keyspaceGroups = []*endpoint.KeyspaceGroup{{ID: uint32(1), UserKind: endpoint.Standard.String()}} + err = suite.kgm.CreateKeyspaceGroups(keyspaceGroups) + re.Error(err) +} + +func (suite *keyspaceGroupTestSuite) TestKeyspaceAssignment() { + re := suite.Require() + + keyspaceGroups := []*endpoint.KeyspaceGroup{ + { + ID: uint32(1), + UserKind: endpoint.Standard.String(), + }, + { + ID: uint32(2), + UserKind: endpoint.Standard.String(), + }, + { + ID: uint32(3), + UserKind: endpoint.Standard.String(), + }, + } + err := suite.kgm.CreateKeyspaceGroups(keyspaceGroups) + re.NoError(err) + // list all keyspace groups + kgs, err := suite.kgm.GetKeyspaceGroups(uint32(0), 0) + re.NoError(err) + re.Len(kgs, 4) + + for i := 0; i < 99; i++ { + _, err := suite.kg.CreateKeyspace(&CreateKeyspaceRequest{ + Name: fmt.Sprintf("test%d", i), + Config: map[string]string{ + UserKindKey: endpoint.Standard.String(), + }, + CreateTime: time.Now().Unix(), + }) + re.NoError(err) + } + + for i := 1; i <= 3; i++ { + kg, err := suite.kgm.GetKeyspaceGroupByID(uint32(i)) + re.NoError(err) + re.Len(kg.Keyspaces, 33) + } +} + +func (suite *keyspaceGroupTestSuite) TestUpdateKeyspace() { + re := suite.Require() + + keyspaceGroups := []*endpoint.KeyspaceGroup{ + { + ID: uint32(1), + UserKind: endpoint.Basic.String(), + }, + { + ID: uint32(2), + UserKind: endpoint.Standard.String(), + }, + { + ID: uint32(3), + UserKind: endpoint.Enterprise.String(), + }, + } + err := suite.kgm.CreateKeyspaceGroups(keyspaceGroups) + re.NoError(err) + // list all keyspace groups + _, err = suite.kgm.GetKeyspaceGroups(uint32(0), 0) + re.NoError(err) + re.Equal(2, suite.kgm.groups[endpoint.Basic].Len()) + re.Equal(1, suite.kgm.groups[endpoint.Standard].Len()) + re.Equal(1, suite.kgm.groups[endpoint.Enterprise].Len()) + + _, err = suite.kg.CreateKeyspace(&CreateKeyspaceRequest{ + Name: "test", + Config: map[string]string{ + UserKindKey: endpoint.Standard.String(), + }, + CreateTime: time.Now().Unix(), + }) + re.NoError(err) + kg2, err := suite.kgm.GetKeyspaceGroupByID(2) + re.NoError(err) + re.Len(kg2.Keyspaces, 1) + kg3, err := suite.kgm.GetKeyspaceGroupByID(3) + re.NoError(err) + re.Len(kg3.Keyspaces, 0) + + _, err = suite.kg.UpdateKeyspaceConfig("test", []*Mutation{ + { + Op: OpPut, + Key: UserKindKey, + Value: endpoint.Enterprise.String(), + }, + { + Op: OpPut, + Key: TSOKeyspaceGroupIDKey, + Value: "2", + }, + }) + re.Error(err) + kg2, err = suite.kgm.GetKeyspaceGroupByID(2) + re.NoError(err) + re.Len(kg2.Keyspaces, 1) + kg3, err = suite.kgm.GetKeyspaceGroupByID(3) + re.NoError(err) + re.Len(kg3.Keyspaces, 0) + _, err = suite.kg.UpdateKeyspaceConfig("test", []*Mutation{ + { + Op: OpPut, + Key: UserKindKey, + Value: endpoint.Enterprise.String(), + }, + { + Op: OpPut, + Key: TSOKeyspaceGroupIDKey, + Value: "3", + }, + }) + re.NoError(err) + kg2, err = suite.kgm.GetKeyspaceGroupByID(2) + re.NoError(err) + re.Len(kg2.Keyspaces, 0) + kg3, err = suite.kgm.GetKeyspaceGroupByID(3) + re.NoError(err) + re.Len(kg3.Keyspaces, 1) +} + +func (suite *keyspaceGroupTestSuite) TestKeyspaceGroupSplit() { + re := suite.Require() + + keyspaceGroups := []*endpoint.KeyspaceGroup{ + { + ID: uint32(1), + UserKind: endpoint.Basic.String(), + Keyspaces: []uint32{444}, + }, + { + ID: uint32(2), + UserKind: endpoint.Standard.String(), + Keyspaces: []uint32{111, 222, 333}, + Members: make([]endpoint.KeyspaceGroupMember, utils.DefaultKeyspaceGroupReplicaCount), + }, + } + err := suite.kgm.CreateKeyspaceGroups(keyspaceGroups) + re.NoError(err) + // split the default keyspace + err = suite.kgm.SplitKeyspaceGroupByID(0, 4, []uint32{utils.DefaultKeyspaceID}) + re.ErrorIs(err, ErrModifyDefaultKeyspace) + // split the keyspace group 1 to 4 + err = suite.kgm.SplitKeyspaceGroupByID(1, 4, []uint32{444}) + re.ErrorIs(err, ErrKeyspaceGroupNotEnoughReplicas) + // split the keyspace group 2 to 4 without giving any keyspace + err = suite.kgm.SplitKeyspaceGroupByID(2, 4, []uint32{}) + re.ErrorIs(err, ErrKeyspaceNotInKeyspaceGroup) + // split the keyspace group 2 to 4 + err = suite.kgm.SplitKeyspaceGroupByID(2, 4, []uint32{333}) + re.NoError(err) + kg2, err := suite.kgm.GetKeyspaceGroupByID(2) + re.NoError(err) + re.Equal(uint32(2), kg2.ID) + re.Equal([]uint32{111, 222}, kg2.Keyspaces) + re.True(kg2.IsSplitSource()) + re.Equal(kg2.ID, kg2.SplitSource()) + kg4, err := suite.kgm.GetKeyspaceGroupByID(4) + re.NoError(err) + re.Equal(uint32(4), kg4.ID) + re.Equal([]uint32{333}, kg4.Keyspaces) + re.True(kg4.IsSplitTarget()) + re.Equal(kg2.ID, kg4.SplitSource()) + re.Equal(kg2.UserKind, kg4.UserKind) + re.Equal(kg2.Members, kg4.Members) + + // finish the split of the keyspace group 2 + err = suite.kgm.FinishSplitKeyspaceByID(2) + re.ErrorContains(err, ErrKeyspaceGroupNotInSplit(2).Error()) + // finish the split of a non-existing keyspace group + err = suite.kgm.FinishSplitKeyspaceByID(5) + re.ErrorContains(err, ErrKeyspaceGroupNotExists(5).Error()) + // split the in-split keyspace group + err = suite.kgm.SplitKeyspaceGroupByID(2, 4, []uint32{333}) + re.ErrorContains(err, ErrKeyspaceGroupInSplit(2).Error()) + // remove the in-split keyspace group + kg2, err = suite.kgm.DeleteKeyspaceGroupByID(2) + re.Nil(kg2) + re.ErrorContains(err, ErrKeyspaceGroupInSplit(2).Error()) + kg4, err = suite.kgm.DeleteKeyspaceGroupByID(4) + re.Nil(kg4) + re.ErrorContains(err, ErrKeyspaceGroupInSplit(4).Error()) + // update the in-split keyspace group + err = suite.kg.kgm.UpdateKeyspaceForGroup(endpoint.Standard, "2", 444, opAdd) + re.ErrorContains(err, ErrKeyspaceGroupInSplit(2).Error()) + err = suite.kg.kgm.UpdateKeyspaceForGroup(endpoint.Standard, "4", 444, opAdd) + re.ErrorContains(err, ErrKeyspaceGroupInSplit(4).Error()) + + // finish the split of keyspace group 4 + err = suite.kgm.FinishSplitKeyspaceByID(4) + re.NoError(err) + kg2, err = suite.kgm.GetKeyspaceGroupByID(2) + re.NoError(err) + re.Equal(uint32(2), kg2.ID) + re.Equal([]uint32{111, 222}, kg2.Keyspaces) + re.False(kg2.IsSplitting()) + kg4, err = suite.kgm.GetKeyspaceGroupByID(4) + re.NoError(err) + re.Equal(uint32(4), kg4.ID) + re.Equal([]uint32{333}, kg4.Keyspaces) + re.False(kg4.IsSplitting()) + re.Equal(kg2.UserKind, kg4.UserKind) + re.Equal(kg2.Members, kg4.Members) + + // split a non-existing keyspace group + err = suite.kgm.SplitKeyspaceGroupByID(3, 5, nil) + re.ErrorContains(err, ErrKeyspaceGroupNotExists(3).Error()) + // split into an existing keyspace group + err = suite.kgm.SplitKeyspaceGroupByID(2, 4, []uint32{111}) + re.ErrorIs(err, ErrKeyspaceGroupExists) + // split with the wrong keyspaces. + err = suite.kgm.SplitKeyspaceGroupByID(2, 5, []uint32{111, 222, 444}) + re.ErrorIs(err, ErrKeyspaceNotInKeyspaceGroup) +} + +func (suite *keyspaceGroupTestSuite) TestKeyspaceGroupSplitRange() { + re := suite.Require() + + keyspaceGroups := []*endpoint.KeyspaceGroup{ + { + ID: uint32(1), + UserKind: endpoint.Basic.String(), + }, + { + ID: uint32(2), + UserKind: endpoint.Standard.String(), + Keyspaces: []uint32{111, 333, 444, 555, 666}, + Members: make([]endpoint.KeyspaceGroupMember, utils.DefaultKeyspaceGroupReplicaCount), + }, + } + err := suite.kgm.CreateKeyspaceGroups(keyspaceGroups) + re.NoError(err) + // split the keyspace group 2 to 4 with keyspace range [222, 555] + err = suite.kgm.SplitKeyspaceGroupByID(2, 4, nil, 222, 555) + re.NoError(err) + kg2, err := suite.kgm.GetKeyspaceGroupByID(2) + re.NoError(err) + re.Equal(uint32(2), kg2.ID) + re.Equal([]uint32{111, 666}, kg2.Keyspaces) + re.True(kg2.IsSplitSource()) + re.Equal(kg2.ID, kg2.SplitSource()) + kg4, err := suite.kgm.GetKeyspaceGroupByID(4) + re.NoError(err) + re.Equal(uint32(4), kg4.ID) + re.Equal([]uint32{333, 444, 555}, kg4.Keyspaces) + re.True(kg4.IsSplitTarget()) + re.Equal(kg2.ID, kg4.SplitSource()) + re.Equal(kg2.UserKind, kg4.UserKind) + re.Equal(kg2.Members, kg4.Members) + // finish the split of keyspace group 4 + err = suite.kgm.FinishSplitKeyspaceByID(4) + re.NoError(err) + kg2, err = suite.kgm.GetKeyspaceGroupByID(2) + re.NoError(err) + re.Equal(uint32(2), kg2.ID) + re.Equal([]uint32{111, 666}, kg2.Keyspaces) + re.False(kg2.IsSplitting()) + kg4, err = suite.kgm.GetKeyspaceGroupByID(4) + re.NoError(err) + re.Equal(uint32(4), kg4.ID) + re.Equal([]uint32{333, 444, 555}, kg4.Keyspaces) + re.False(kg4.IsSplitting()) + re.Equal(kg2.UserKind, kg4.UserKind) + re.Equal(kg2.Members, kg4.Members) +} + +func (suite *keyspaceGroupTestSuite) TestKeyspaceGroupMerge() { + re := suite.Require() + + keyspaceGroups := []*endpoint.KeyspaceGroup{ + { + ID: uint32(1), + UserKind: endpoint.Basic.String(), + Keyspaces: []uint32{111, 222, 333}, + Members: make([]endpoint.KeyspaceGroupMember, utils.DefaultKeyspaceGroupReplicaCount), + }, + { + ID: uint32(3), + UserKind: endpoint.Basic.String(), + Keyspaces: []uint32{444, 555}, + }, + } + err := suite.kgm.CreateKeyspaceGroups(keyspaceGroups) + re.NoError(err) + // split the keyspace group 1 to 2 + err = suite.kgm.SplitKeyspaceGroupByID(1, 2, []uint32{333}) + re.NoError(err) + // finish the split of the keyspace group 2 + err = suite.kgm.FinishSplitKeyspaceByID(2) + re.NoError(err) + // check the keyspace group 1 and 2 + kg1, err := suite.kgm.GetKeyspaceGroupByID(1) + re.NoError(err) + re.Equal(uint32(1), kg1.ID) + re.Equal([]uint32{111, 222}, kg1.Keyspaces) + re.False(kg1.IsSplitting()) + re.False(kg1.IsMerging()) + kg2, err := suite.kgm.GetKeyspaceGroupByID(2) + re.NoError(err) + re.Equal(uint32(2), kg2.ID) + re.Equal([]uint32{333}, kg2.Keyspaces) + re.False(kg2.IsSplitting()) + re.False(kg2.IsMerging()) + re.Equal(kg1.UserKind, kg2.UserKind) + re.Equal(kg1.Members, kg2.Members) + // merge the keyspace group 2 and 3 back into 1 + err = suite.kgm.MergeKeyspaceGroups(1, []uint32{2, 3}) + re.NoError(err) + // check the keyspace group 2 and 3 + kg2, err = suite.kgm.GetKeyspaceGroupByID(2) + re.NoError(err) + re.Nil(kg2) + kg3, err := suite.kgm.GetKeyspaceGroupByID(3) + re.NoError(err) + re.Nil(kg3) + // check the keyspace group 1 + kg1, err = suite.kgm.GetKeyspaceGroupByID(1) + re.NoError(err) + re.Equal(uint32(1), kg1.ID) + re.Equal([]uint32{111, 222, 333, 444, 555}, kg1.Keyspaces) + re.False(kg1.IsSplitting()) + re.True(kg1.IsMerging()) + // finish the merging + err = suite.kgm.FinishMergeKeyspaceByID(1) + re.NoError(err) + kg1, err = suite.kgm.GetKeyspaceGroupByID(1) + re.NoError(err) + re.Equal(uint32(1), kg1.ID) + re.Equal([]uint32{111, 222, 333, 444, 555}, kg1.Keyspaces) + re.False(kg1.IsSplitting()) + re.False(kg1.IsMerging()) + + // merge a non-existing keyspace group + err = suite.kgm.MergeKeyspaceGroups(4, []uint32{5}) + re.ErrorContains(err, ErrKeyspaceGroupNotExists(5).Error()) + // merge with the number of keyspace groups exceeds the limit + err = suite.kgm.MergeKeyspaceGroups(1, make([]uint32, MaxEtcdTxnOps/2)) + re.ErrorIs(err, ErrExceedMaxEtcdTxnOps) + // merge the default keyspace group + err = suite.kgm.MergeKeyspaceGroups(1, []uint32{utils.DefaultKeyspaceGroupID}) + re.ErrorIs(err, ErrModifyDefaultKeyspaceGroup) +} + +func TestBuildSplitKeyspaces(t *testing.T) { + re := require.New(t) + testCases := []struct { + old []uint32 + new []uint32 + startKeyspaceID uint32 + endKeyspaceID uint32 + expectedOld []uint32 + expectedNew []uint32 + err error + }{ + { + old: []uint32{1, 2, 3, 4, 5}, + new: []uint32{1, 2, 3, 4, 5}, + expectedOld: []uint32{}, + expectedNew: []uint32{1, 2, 3, 4, 5}, + }, + { + old: []uint32{1, 2, 3, 4, 5}, + new: []uint32{1}, + expectedOld: []uint32{2, 3, 4, 5}, + expectedNew: []uint32{1}, + }, + { + old: []uint32{1, 2, 3, 4, 5}, + new: []uint32{6}, + err: ErrKeyspaceNotInKeyspaceGroup, + }, + { + old: []uint32{1, 2}, + new: []uint32{2, 2}, + expectedOld: []uint32{1}, + expectedNew: []uint32{2}, + }, + { + old: []uint32{0, 1, 2, 3, 4, 5}, + startKeyspaceID: 2, + endKeyspaceID: 4, + expectedOld: []uint32{0, 1, 5}, + expectedNew: []uint32{2, 3, 4}, + }, + { + old: []uint32{0, 1, 2, 3, 4, 5}, + startKeyspaceID: 0, + endKeyspaceID: 4, + expectedOld: []uint32{0, 5}, + expectedNew: []uint32{1, 2, 3, 4}, + }, + { + old: []uint32{1, 2, 3, 4, 5}, + startKeyspaceID: 2, + endKeyspaceID: 4, + expectedOld: []uint32{1, 5}, + expectedNew: []uint32{2, 3, 4}, + }, + { + old: []uint32{1, 2, 3, 4, 5}, + startKeyspaceID: 5, + endKeyspaceID: 6, + expectedOld: []uint32{1, 2, 3, 4}, + expectedNew: []uint32{5}, + }, + { + old: []uint32{1, 2, 3, 4, 5}, + startKeyspaceID: 2, + endKeyspaceID: 6, + expectedOld: []uint32{1}, + expectedNew: []uint32{2, 3, 4, 5}, + }, + { + old: []uint32{1, 2, 3, 4, 5}, + startKeyspaceID: 1, + endKeyspaceID: 1, + expectedOld: []uint32{2, 3, 4, 5}, + expectedNew: []uint32{1}, + }, + { + old: []uint32{1, 2, 3, 4, 5}, + startKeyspaceID: 0, + endKeyspaceID: 6, + expectedOld: []uint32{}, + expectedNew: []uint32{1, 2, 3, 4, 5}, + }, + { + old: []uint32{1, 2, 3, 4, 5}, + startKeyspaceID: 7, + endKeyspaceID: 10, + expectedOld: []uint32{1, 2, 3, 4, 5}, + expectedNew: []uint32{}, + }, + { + old: []uint32{1, 2, 3, 4, 5}, + err: ErrKeyspaceNotInKeyspaceGroup, + }, + } + for idx, testCase := range testCases { + old, new, err := buildSplitKeyspaces(testCase.old, testCase.new, testCase.startKeyspaceID, testCase.endKeyspaceID) + if testCase.err != nil { + re.ErrorIs(testCase.err, err, "test case %d", idx) + } else { + re.NoError(err, "test case %d", idx) + re.Equal(testCase.expectedOld, old, "test case %d", idx) + re.Equal(testCase.expectedNew, new, "test case %d", idx) + } + } +} diff --git a/pkg/keyspace/util.go b/pkg/keyspace/util.go new file mode 100644 index 00000000000..2923dc7053f --- /dev/null +++ b/pkg/keyspace/util.go @@ -0,0 +1,305 @@ +// Copyright 2022 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package keyspace + +import ( + "container/heap" + "encoding/binary" + "encoding/hex" + "regexp" + "strconv" + + "github.com/pingcap/errors" + "github.com/pingcap/kvproto/pkg/keyspacepb" + "github.com/tikv/pd/pkg/codec" + "github.com/tikv/pd/pkg/mcs/utils" + "github.com/tikv/pd/pkg/schedule/labeler" + "github.com/tikv/pd/pkg/storage/endpoint" +) + +const ( + spaceIDMax = ^uint32(0) >> 8 // 16777215 (Uint24Max) is the maximum value of spaceID. + // namePattern is a regex that specifies acceptable characters of the keyspace name. + // Name must be non-empty and contains only alphanumerical, `_` and `-`. + namePattern = "^[-A-Za-z0-9_]+$" +) + +var ( + // ErrKeyspaceNotFound is used to indicate target keyspace does not exist. + ErrKeyspaceNotFound = errors.New("keyspace does not exist") + // ErrRegionSplitTimeout indices to split region timeout + ErrRegionSplitTimeout = errors.New("region split timeout") + // ErrRegionSplitFailed indices to split region failed + ErrRegionSplitFailed = errors.New("region split failed") + // ErrKeyspaceExists indicates target keyspace already exists. + // It's used when creating a new keyspace. + ErrKeyspaceExists = errors.New("keyspace already exists") + // ErrKeyspaceGroupExists indicates target keyspace group already exists. + ErrKeyspaceGroupExists = errors.New("keyspace group already exists") + // ErrKeyspaceGroupNotExists is used to indicate target keyspace group does not exist. + ErrKeyspaceGroupNotExists = func(groupID uint32) error { + return errors.Errorf("keyspace group %v does not exist", groupID) + } + // ErrKeyspaceGroupInSplit is used to indicate target keyspace group is in split state. + ErrKeyspaceGroupInSplit = func(groupID uint32) error { + return errors.Errorf("keyspace group %v is in split state", groupID) + } + // ErrKeyspaceGroupNotInSplit is used to indicate target keyspace group is not in split state. + ErrKeyspaceGroupNotInSplit = func(groupID uint32) error { + return errors.Errorf("keyspace group %v is not in split state", groupID) + } + // ErrKeyspaceGroupInMerging is used to indicate target keyspace group is in merging state. + ErrKeyspaceGroupInMerging = func(groupID uint32) error { + return errors.Errorf("keyspace group %v is in merging state", groupID) + } + // ErrKeyspaceGroupNotInMerging is used to indicate target keyspace group is not in merging state. + ErrKeyspaceGroupNotInMerging = func(groupID uint32) error { + return errors.Errorf("keyspace group %v is not in merging state", groupID) + } + // ErrKeyspaceNotInKeyspaceGroup is used to indicate target keyspace is not in this keyspace group. + ErrKeyspaceNotInKeyspaceGroup = errors.New("keyspace is not in this keyspace group") + // ErrKeyspaceNotInAnyKeyspaceGroup is used to indicate target keyspace is not in any keyspace group. + ErrKeyspaceNotInAnyKeyspaceGroup = errors.New("keyspace is not in any keyspace group") + // ErrNodeNotInKeyspaceGroup is used to indicate the tso node is not in this keyspace group. + ErrNodeNotInKeyspaceGroup = errors.New("the tso node is not in this keyspace group") + // ErrKeyspaceGroupNotEnoughReplicas is used to indicate not enough replicas in the keyspace group. + ErrKeyspaceGroupNotEnoughReplicas = errors.New("not enough replicas in the keyspace group") + // ErrModifyDefaultKeyspaceGroup is used to indicate that default keyspace group cannot be modified. + ErrModifyDefaultKeyspaceGroup = errors.New("default keyspace group cannot be modified") + // ErrNoAvailableNode is used to indicate no available node in the keyspace group. + ErrNoAvailableNode = errors.New("no available node") + // ErrExceedMaxEtcdTxnOps is used to indicate the number of etcd txn operations exceeds the limit. + ErrExceedMaxEtcdTxnOps = errors.New("exceed max etcd txn operations") + // ErrModifyDefaultKeyspace is used to indicate that default keyspace cannot be modified. + ErrModifyDefaultKeyspace = errors.New("cannot modify default keyspace's state") + errIllegalOperation = errors.New("unknown operation") + + // stateTransitionTable lists all allowed next state for the given current state. + // Note that transit from any state to itself is allowed for idempotence. + stateTransitionTable = map[keyspacepb.KeyspaceState][]keyspacepb.KeyspaceState{ + keyspacepb.KeyspaceState_ENABLED: {keyspacepb.KeyspaceState_ENABLED, keyspacepb.KeyspaceState_DISABLED}, + keyspacepb.KeyspaceState_DISABLED: {keyspacepb.KeyspaceState_DISABLED, keyspacepb.KeyspaceState_ENABLED, keyspacepb.KeyspaceState_ARCHIVED}, + keyspacepb.KeyspaceState_ARCHIVED: {keyspacepb.KeyspaceState_ARCHIVED, keyspacepb.KeyspaceState_TOMBSTONE}, + keyspacepb.KeyspaceState_TOMBSTONE: {keyspacepb.KeyspaceState_TOMBSTONE}, + } + // Only keyspaces in the state specified by allowChangeConfig are allowed to change their config. + allowChangeConfig = []keyspacepb.KeyspaceState{keyspacepb.KeyspaceState_ENABLED, keyspacepb.KeyspaceState_DISABLED} + + // ErrKeyspaceGroupPrimaryNotFound is used to indicate primary of target keyspace group does not exist. + ErrKeyspaceGroupPrimaryNotFound = errors.New("primary of keyspace group does not exist") +) + +// validateID check if keyspace falls within the acceptable range. +// It throws errIllegalID when input id is our of range, +// or if it collides with reserved id. +func validateID(id uint32) error { + if id > spaceIDMax { + return errors.Errorf("illegal keyspace id %d, larger than spaceID Max %d", id, spaceIDMax) + } + if id == utils.DefaultKeyspaceID { + return errors.Errorf("illegal keyspace id %d, collides with default keyspace id", id) + } + return nil +} + +// validateName check if user provided name is legal. +// It throws errIllegalName when name contains illegal character, +// or if it collides with reserved name. +func validateName(name string) error { + isValid, err := regexp.MatchString(namePattern, name) + if err != nil { + return err + } + if !isValid { + return errors.Errorf("illegal keyspace name %s, should contain only alphanumerical and underline", name) + } + if name == utils.DefaultKeyspaceName { + return errors.Errorf("illegal keyspace name %s, collides with default keyspace name", name) + } + return nil +} + +// MaskKeyspaceID is used to hash the spaceID inside the lockGroup. +// A simple mask is applied to spaceID to use its last byte as map key, +// limiting the maximum map length to 256. +// Since keyspaceID is sequentially allocated, this can also reduce the chance +// of collision when comparing with random hashes. +func MaskKeyspaceID(id uint32) uint32 { + return id & 0xFF +} + +// RegionBound represents the region boundary of the given keyspace. +// For a keyspace with id ['a', 'b', 'c'], it has four boundaries: +// +// Lower bound for raw mode: ['r', 'a', 'b', 'c'] +// Upper bound for raw mode: ['r', 'a', 'b', 'c + 1'] +// Lower bound for txn mode: ['x', 'a', 'b', 'c'] +// Upper bound for txn mode: ['x', 'a', 'b', 'c + 1'] +// +// From which it shares the lower bound with keyspace with id ['a', 'b', 'c-1']. +// And shares upper bound with keyspace with id ['a', 'b', 'c + 1']. +// These repeated bound will not cause any problem, as repetitive bound will be ignored during rangeListBuild, +// but provides guard against hole in keyspace allocations should it occur. +type RegionBound struct { + RawLeftBound []byte + RawRightBound []byte + TxnLeftBound []byte + TxnRightBound []byte +} + +// MakeRegionBound constructs the correct region boundaries of the given keyspace. +func MakeRegionBound(id uint32) *RegionBound { + keyspaceIDBytes := make([]byte, 4) + nextKeyspaceIDBytes := make([]byte, 4) + binary.BigEndian.PutUint32(keyspaceIDBytes, id) + binary.BigEndian.PutUint32(nextKeyspaceIDBytes, id+1) + return &RegionBound{ + RawLeftBound: codec.EncodeBytes(append([]byte{'r'}, keyspaceIDBytes[1:]...)), + RawRightBound: codec.EncodeBytes(append([]byte{'r'}, nextKeyspaceIDBytes[1:]...)), + TxnLeftBound: codec.EncodeBytes(append([]byte{'x'}, keyspaceIDBytes[1:]...)), + TxnRightBound: codec.EncodeBytes(append([]byte{'x'}, nextKeyspaceIDBytes[1:]...)), + } +} + +// makeKeyRanges encodes keyspace ID to correct LabelRule data. +func makeKeyRanges(id uint32) []interface{} { + regionBound := MakeRegionBound(id) + return []interface{}{ + map[string]interface{}{ + "start_key": hex.EncodeToString(regionBound.RawLeftBound), + "end_key": hex.EncodeToString(regionBound.RawRightBound), + }, + map[string]interface{}{ + "start_key": hex.EncodeToString(regionBound.TxnLeftBound), + "end_key": hex.EncodeToString(regionBound.TxnRightBound), + }, + } +} + +// getRegionLabelID returns the region label id of the target keyspace. +func getRegionLabelID(id uint32) string { + return regionLabelIDPrefix + strconv.FormatUint(uint64(id), endpoint.SpaceIDBase) +} + +// makeLabelRule makes the label rule for the given keyspace id. +func makeLabelRule(id uint32) *labeler.LabelRule { + return &labeler.LabelRule{ + ID: getRegionLabelID(id), + Index: 0, + Labels: []labeler.RegionLabel{ + { + Key: regionLabelKey, + Value: strconv.FormatUint(uint64(id), endpoint.SpaceIDBase), + }, + }, + RuleType: labeler.KeyRange, + Data: makeKeyRanges(id), + } +} + +// indexedHeap is a heap with index. +type indexedHeap struct { + items []*endpoint.KeyspaceGroup + // keyspace group id -> position in items + index map[uint32]int +} + +func newIndexedHeap(hint int) *indexedHeap { + return &indexedHeap{ + items: make([]*endpoint.KeyspaceGroup, 0, hint), + index: map[uint32]int{}, + } +} + +// Implementing heap.Interface. +func (hp *indexedHeap) Len() int { + return len(hp.items) +} + +// Implementing heap.Interface. +func (hp *indexedHeap) Less(i, j int) bool { + // Gives the keyspace group with the least number of keyspaces first + return len(hp.items[j].Keyspaces) > len(hp.items[i].Keyspaces) +} + +// Implementing heap.Interface. +func (hp *indexedHeap) Swap(i, j int) { + lid := hp.items[i].ID + rid := hp.items[j].ID + hp.items[i], hp.items[j] = hp.items[j], hp.items[i] + hp.index[lid] = j + hp.index[rid] = i +} + +// Implementing heap.Interface. +func (hp *indexedHeap) Push(x interface{}) { + item := x.(*endpoint.KeyspaceGroup) + hp.index[item.ID] = hp.Len() + hp.items = append(hp.items, item) +} + +// Implementing heap.Interface. +func (hp *indexedHeap) Pop() interface{} { + l := hp.Len() + item := hp.items[l-1] + hp.items = hp.items[:l-1] + delete(hp.index, item.ID) + return item +} + +// Top returns the top item. +func (hp *indexedHeap) Top() *endpoint.KeyspaceGroup { + if hp.Len() <= 0 { + return nil + } + return hp.items[0] +} + +// Get returns item with the given ID. +func (hp *indexedHeap) Get(id uint32) *endpoint.KeyspaceGroup { + idx, ok := hp.index[id] + if !ok { + return nil + } + item := hp.items[idx] + return item +} + +// GetAll returns all the items. +func (hp *indexedHeap) GetAll() []*endpoint.KeyspaceGroup { + all := make([]*endpoint.KeyspaceGroup, len(hp.items)) + copy(all, hp.items) + return all +} + +// Put inserts item or updates the old item if it exists. +func (hp *indexedHeap) Put(item *endpoint.KeyspaceGroup) (isUpdate bool) { + if idx, ok := hp.index[item.ID]; ok { + hp.items[idx] = item + heap.Fix(hp, idx) + return true + } + heap.Push(hp, item) + return false +} + +// Remove deletes item by ID and returns it. +func (hp *indexedHeap) Remove(id uint32) *endpoint.KeyspaceGroup { + if idx, ok := hp.index[id]; ok { + item := heap.Remove(hp, idx) + return item.(*endpoint.KeyspaceGroup) + } + return nil +} diff --git a/server/keyspace/util_test.go b/pkg/keyspace/util_test.go similarity index 94% rename from server/keyspace/util_test.go rename to pkg/keyspace/util_test.go index 40277e298b6..c7b3738a811 100644 --- a/server/keyspace/util_test.go +++ b/pkg/keyspace/util_test.go @@ -21,6 +21,7 @@ import ( "github.com/stretchr/testify/require" "github.com/tikv/pd/pkg/codec" + "github.com/tikv/pd/pkg/mcs/utils" "github.com/tikv/pd/pkg/schedule/labeler" ) @@ -30,7 +31,7 @@ func TestValidateID(t *testing.T) { id uint32 hasErr bool }{ - {DefaultKeyspaceID, true}, // Reserved id should result in error. + {utils.DefaultKeyspaceID, true}, // Reserved id should result in error. {100, false}, {spaceIDMax - 1, false}, {spaceIDMax, false}, @@ -48,7 +49,7 @@ func TestValidateName(t *testing.T) { name string hasErr bool }{ - {DefaultKeyspaceName, true}, // Reserved name should result in error. + {utils.DefaultKeyspaceName, true}, // Reserved name should result in error. {"keyspaceName1", false}, {"keyspace_name_1", false}, {"10", false}, diff --git a/pkg/mcs/discovery/discover.go b/pkg/mcs/discovery/discover.go index f6716e924cf..00e168114b0 100644 --- a/pkg/mcs/discovery/discover.go +++ b/pkg/mcs/discovery/discover.go @@ -20,9 +20,9 @@ import ( ) // Discover is used to get all the service instances of the specified service name. -func Discover(cli *clientv3.Client, serviceName string) ([]string, error) { - key := discoveryPath(serviceName) + "/" - endKey := clientv3.GetPrefixRangeEnd(key) + "/" +func Discover(cli *clientv3.Client, clusterID, serviceName string) ([]string, error) { + key := ServicePath(clusterID, serviceName) + endKey := clientv3.GetPrefixRangeEnd(key) withRange := clientv3.WithRange(endKey) resp, err := etcdutil.EtcdKVGet(cli, key, withRange) diff --git a/pkg/mcs/discovery/discover_test.go b/pkg/mcs/discovery/discover_test.go index fe556bac073..fed1d7844a0 100644 --- a/pkg/mcs/discovery/discover_test.go +++ b/pkg/mcs/discovery/discover_test.go @@ -41,14 +41,14 @@ func TestDiscover(t *testing.T) { re.NoError(err) <-etcd.Server.ReadyNotify() - sr1 := NewServiceRegister(context.Background(), client, "test_service", "127.0.0.1:1", "127.0.0.1:1", 1) + sr1 := NewServiceRegister(context.Background(), client, "12345", "test_service", "127.0.0.1:1", "127.0.0.1:1", 1) err = sr1.Register() re.NoError(err) - sr2 := NewServiceRegister(context.Background(), client, "test_service", "127.0.0.1:2", "127.0.0.1:2", 1) + sr2 := NewServiceRegister(context.Background(), client, "12345", "test_service", "127.0.0.1:2", "127.0.0.1:2", 1) err = sr2.Register() re.NoError(err) - endpoints, err := Discover(client, "test_service") + endpoints, err := Discover(client, "12345", "test_service") re.NoError(err) re.Len(endpoints, 2) re.Equal("127.0.0.1:1", endpoints[0]) @@ -57,7 +57,54 @@ func TestDiscover(t *testing.T) { sr1.cancel() sr2.cancel() time.Sleep(3 * time.Second) - endpoints, err = Discover(client, "test_service") + endpoints, err = Discover(client, "12345", "test_service") + re.NoError(err) + re.Empty(endpoints) +} + +func TestServiceRegistryEntry(t *testing.T) { + re := require.New(t) + cfg := etcdutil.NewTestSingleConfig(t) + etcd, err := embed.StartEtcd(cfg) + defer func() { + etcd.Close() + }() + re.NoError(err) + + ep := cfg.LCUrls[0].String() + re.NoError(err) + + client, err := clientv3.NewFromURL(ep) + re.NoError(err) + + <-etcd.Server.ReadyNotify() + entry1 := &ServiceRegistryEntry{ServiceAddr: "127.0.0.1:1"} + s1, err := entry1.Serialize() + re.NoError(err) + sr1 := NewServiceRegister(context.Background(), client, "12345", "test_service", "127.0.0.1:1", s1, 1) + err = sr1.Register() + re.NoError(err) + entry2 := &ServiceRegistryEntry{ServiceAddr: "127.0.0.1:2"} + s2, err := entry2.Serialize() + re.NoError(err) + sr2 := NewServiceRegister(context.Background(), client, "12345", "test_service", "127.0.0.1:2", s2, 1) + err = sr2.Register() + re.NoError(err) + + endpoints, err := Discover(client, "12345", "test_service") + re.NoError(err) + re.Len(endpoints, 2) + returnedEntry1 := &ServiceRegistryEntry{} + returnedEntry1.Deserialize([]byte(endpoints[0])) + re.Equal("127.0.0.1:1", returnedEntry1.ServiceAddr) + returnedEntry2 := &ServiceRegistryEntry{} + returnedEntry2.Deserialize([]byte(endpoints[1])) + re.Equal("127.0.0.1:2", returnedEntry2.ServiceAddr) + + sr1.cancel() + sr2.cancel() + time.Sleep(3 * time.Second) + endpoints, err = Discover(client, "12345", "test_service") re.NoError(err) re.Empty(endpoints) } diff --git a/pkg/mcs/discovery/key_path.go b/pkg/mcs/discovery/key_path.go index 171dbff736b..b7bf9d1cac3 100644 --- a/pkg/mcs/discovery/key_path.go +++ b/pkg/mcs/discovery/key_path.go @@ -14,17 +14,28 @@ package discovery -import "path" +import ( + "strconv" + "strings" + + "github.com/tikv/pd/pkg/mcs/utils" +) const ( - registryPrefix = "/pd/microservice" - registryKey = "registry" + registryKey = "registry" ) -func registryPath(serviceName, serviceAddr string) string { - return path.Join(registryPrefix, serviceName, registryKey, serviceAddr) +// RegistryPath returns the full path to store microservice addresses. +func RegistryPath(clusterID, serviceName, serviceAddr string) string { + return strings.Join([]string{utils.MicroserviceRootPath, clusterID, serviceName, registryKey, serviceAddr}, "/") +} + +// ServicePath returns the path to store microservice addresses. +func ServicePath(clusterID, serviceName string) string { + return strings.Join([]string{utils.MicroserviceRootPath, clusterID, serviceName, registryKey, ""}, "/") } -func discoveryPath(serviceName string) string { - return path.Join(registryPrefix, serviceName, registryKey) +// TSOPath returns the path to store TSO addresses. +func TSOPath(clusterID uint64) string { + return ServicePath(strconv.FormatUint(clusterID, 10), "tso") } diff --git a/pkg/mcs/discovery/register.go b/pkg/mcs/discovery/register.go index e7e0254a5c4..1e7800295de 100644 --- a/pkg/mcs/discovery/register.go +++ b/pkg/mcs/discovery/register.go @@ -20,6 +20,7 @@ import ( "time" "github.com/pingcap/log" + "github.com/tikv/pd/pkg/utils/logutil" "go.etcd.io/etcd/clientv3" "go.uber.org/zap" ) @@ -38,9 +39,9 @@ type ServiceRegister struct { } // NewServiceRegister creates a new ServiceRegister. -func NewServiceRegister(ctx context.Context, cli *clientv3.Client, serviceName, serviceAddr, serializedValue string, ttl int64) *ServiceRegister { +func NewServiceRegister(ctx context.Context, cli *clientv3.Client, clusterID, serviceName, serviceAddr, serializedValue string, ttl int64) *ServiceRegister { cctx, cancel := context.WithCancel(ctx) - serviceKey := registryPath(serviceName, serviceAddr) + serviceKey := RegistryPath(clusterID, serviceName, serviceAddr) return &ServiceRegister{ ctx: cctx, cancel: cancel, @@ -70,6 +71,7 @@ func (sr *ServiceRegister) Register() error { return fmt.Errorf("keepalive failed: %v", err) } go func() { + defer logutil.LogPanic() for { select { case <-sr.ctx.Done(): @@ -84,6 +86,7 @@ func (sr *ServiceRegister) Register() error { select { case <-sr.ctx.Done(): log.Info("exit register process", zap.String("key", sr.key)) + t.Stop() return default: } @@ -92,11 +95,13 @@ func (sr *ServiceRegister) Register() error { resp, err := sr.cli.Grant(sr.ctx, sr.ttl) if err != nil { log.Error("grant lease failed", zap.String("key", sr.key), zap.Error(err)) + t.Stop() continue } if _, err := sr.cli.Put(sr.ctx, sr.key, sr.value, clientv3.WithLease(resp.ID)); err != nil { log.Error("put the key failed", zap.String("key", sr.key), zap.Error(err)) + t.Stop() continue } } diff --git a/pkg/mcs/discovery/register_test.go b/pkg/mcs/discovery/register_test.go index c8594ba0162..6d0bf8a7066 100644 --- a/pkg/mcs/discovery/register_test.go +++ b/pkg/mcs/discovery/register_test.go @@ -39,13 +39,15 @@ func TestRegister(t *testing.T) { re.NoError(err) <-etcd.Server.ReadyNotify() - sr := NewServiceRegister(context.Background(), client, "test_service", "127.0.0.1:1", "127.0.0.1:1", 10) + // with http prefix + sr := NewServiceRegister(context.Background(), client, "12345", "test_service", "http://127.0.0.1:1", "http://127.0.0.1:1", 10) re.NoError(err) err = sr.Register() re.NoError(err) + re.Equal("/ms/12345/test_service/registry/http://127.0.0.1:1", sr.key) resp, err := client.Get(context.Background(), sr.key) re.NoError(err) - re.Equal("127.0.0.1:1", string(resp.Kvs[0].Value)) + re.Equal("http://127.0.0.1:1", string(resp.Kvs[0].Value)) err = sr.Deregister() re.NoError(err) @@ -53,7 +55,7 @@ func TestRegister(t *testing.T) { re.NoError(err) re.Empty(resp.Kvs) - sr = NewServiceRegister(context.Background(), client, "test_service", "127.0.0.1:2", "127.0.0.1:2", 1) + sr = NewServiceRegister(context.Background(), client, "12345", "test_service", "127.0.0.1:2", "127.0.0.1:2", 1) re.NoError(err) err = sr.Register() re.NoError(err) diff --git a/pkg/mcs/discovery/registry_entry.go b/pkg/mcs/discovery/registry_entry.go new file mode 100644 index 00000000000..52751b430c4 --- /dev/null +++ b/pkg/mcs/discovery/registry_entry.go @@ -0,0 +1,46 @@ +// Copyright 2023 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package discovery + +import ( + "encoding/json" + + "github.com/pingcap/log" + "go.uber.org/zap" +) + +// ServiceRegistryEntry is the registry entry of a service +type ServiceRegistryEntry struct { + ServiceAddr string `json:"service-addr"` +} + +// Serialize this service registry entry +func (e *ServiceRegistryEntry) Serialize() (serializedValue string, err error) { + data, err := json.Marshal(e) + if err != nil { + log.Error("json marshal the service registry entry failed", zap.Error(err)) + return "", err + } + return string(data), nil +} + +// Deserialize the data to this service registry entry +func (e *ServiceRegistryEntry) Deserialize(data []byte) error { + if err := json.Unmarshal(data, e); err != nil { + log.Error("json unmarshal the service registry entry failed", zap.Error(err)) + return err + } + return nil +} diff --git a/pkg/mcs/meta_storage/server/grpc_service.go b/pkg/mcs/metastorage/server/grpc_service.go similarity index 91% rename from pkg/mcs/meta_storage/server/grpc_service.go rename to pkg/mcs/metastorage/server/grpc_service.go index 2c7acaa8e78..e9d35fbf14b 100644 --- a/pkg/mcs/meta_storage/server/grpc_service.go +++ b/pkg/mcs/metastorage/server/grpc_service.go @@ -75,7 +75,7 @@ func (s *Service) RegisterRESTHandler(userDefineHandlers map[string]http.Handler } func (s *Service) checkServing() error { - if !s.manager.srv.IsServing() { + if s.manager == nil || s.manager.srv == nil || !s.manager.srv.IsServing() { return errNotLeader } return nil @@ -126,7 +126,14 @@ func (s *Service) Watch(req *meta_storagepb.WatchRequest, server meta_storagepb. events := make([]*meta_storagepb.Event, 0, len(res.Events)) for _, e := range res.Events { - event := &meta_storagepb.Event{Kv: &meta_storagepb.KeyValue{Key: e.Kv.Key, Value: e.Kv.Value}, Type: meta_storagepb.Event_EventType(e.Type)} + event := &meta_storagepb.Event{Kv: &meta_storagepb.KeyValue{ + Key: e.Kv.Key, + Value: e.Kv.Value, + ModRevision: e.Kv.ModRevision, + CreateRevision: e.Kv.CreateRevision, + Version: e.Kv.Version, + Lease: e.Kv.Lease, + }, Type: meta_storagepb.Event_EventType(e.Type)} if e.PrevKv != nil { event.PrevKv = &meta_storagepb.KeyValue{Key: e.PrevKv.Key, Value: e.PrevKv.Value} } @@ -163,11 +170,15 @@ func (s *Service) Get(ctx context.Context, req *meta_storagepb.GetRequest) (*met } cli := s.manager.GetClient() res, err := cli.Get(ctx, key, options...) + var revision int64 + if res != nil { + revision = res.Header.GetRevision() + } if err != nil { - return &meta_storagepb.GetResponse{Header: s.wrapErrorAndRevision(res.Header.GetRevision(), meta_storagepb.ErrorType_UNKNOWN, err.Error())}, nil + return &meta_storagepb.GetResponse{Header: s.wrapErrorAndRevision(revision, meta_storagepb.ErrorType_UNKNOWN, err.Error())}, nil } resp := &meta_storagepb.GetResponse{ - Header: &meta_storagepb.ResponseHeader{ClusterId: s.manager.ClusterID(), Revision: res.Header.GetRevision()}, + Header: &meta_storagepb.ResponseHeader{ClusterId: s.manager.ClusterID(), Revision: revision}, Count: res.Count, More: res.More, } @@ -197,12 +208,16 @@ func (s *Service) Put(ctx context.Context, req *meta_storagepb.PutRequest) (*met cli := s.manager.GetClient() res, err := cli.Put(ctx, key, value, options...) + var revision int64 + if res != nil { + revision = res.Header.GetRevision() + } if err != nil { - return &meta_storagepb.PutResponse{Header: s.wrapErrorAndRevision(res.Header.GetRevision(), meta_storagepb.ErrorType_UNKNOWN, err.Error())}, nil + return &meta_storagepb.PutResponse{Header: s.wrapErrorAndRevision(revision, meta_storagepb.ErrorType_UNKNOWN, err.Error())}, nil } resp := &meta_storagepb.PutResponse{ - Header: &meta_storagepb.ResponseHeader{ClusterId: s.manager.ClusterID(), Revision: res.Header.GetRevision()}, + Header: &meta_storagepb.ResponseHeader{ClusterId: s.manager.ClusterID(), Revision: revision}, } if res.PrevKv != nil { resp.PrevKv = &meta_storagepb.KeyValue{Key: res.PrevKv.Key, Value: res.PrevKv.Value} diff --git a/pkg/mcs/meta_storage/server/install/install.go b/pkg/mcs/metastorage/server/install/install.go similarity index 93% rename from pkg/mcs/meta_storage/server/install/install.go rename to pkg/mcs/metastorage/server/install/install.go index f7efda1322a..defc24d5f99 100644 --- a/pkg/mcs/meta_storage/server/install/install.go +++ b/pkg/mcs/metastorage/server/install/install.go @@ -15,7 +15,7 @@ package install import ( - ms_server "github.com/tikv/pd/pkg/mcs/meta_storage/server" + ms_server "github.com/tikv/pd/pkg/mcs/metastorage/server" "github.com/tikv/pd/pkg/mcs/registry" ) diff --git a/pkg/mcs/meta_storage/server/manager.go b/pkg/mcs/metastorage/server/manager.go similarity index 100% rename from pkg/mcs/meta_storage/server/manager.go rename to pkg/mcs/metastorage/server/manager.go diff --git a/pkg/mcs/resource_manager/server/resource_group_test.go b/pkg/mcs/resource_manager/server/resource_group_test.go deleted file mode 100644 index 236c756d4e5..00000000000 --- a/pkg/mcs/resource_manager/server/resource_group_test.go +++ /dev/null @@ -1,64 +0,0 @@ -package server - -import ( - "encoding/json" - "testing" - - rmpb "github.com/pingcap/kvproto/pkg/resource_manager" - "github.com/stretchr/testify/require" -) - -func TestPatchResourceGroup(t *testing.T) { - re := require.New(t) - rg1 := &ResourceGroup{Name: "test", Mode: rmpb.GroupMode_RUMode, RUSettings: &RequestUnitSettings{}} - err := rg1.CheckAndInit() - re.NoError(err) - testCaseRU := []struct { - patchJSONString string - expectJSONString string - }{ - {`{"name":"test", "mode":1, "r_u_settings": {"r_u":{"settings":{"fill_rate": 200000}}}}`, - `{"name":"test","mode":1,"r_u_settings":{"r_u":{"settings":{"fill_rate":200000},"state":{"initialized":false}}}}`}, - {`{"name":"test", "mode":1, "r_u_settings": {"r_u":{"settings":{"fill_rate": 200000, "burst_limit": -1}}}}`, - `{"name":"test","mode":1,"r_u_settings":{"r_u":{"settings":{"fill_rate":200000,"burst_limit":-1},"state":{"initialized":false}}}}`}, - } - - for _, ca := range testCaseRU { - rg := rg1.Copy() - patch := &rmpb.ResourceGroup{} - err := json.Unmarshal([]byte(ca.patchJSONString), patch) - re.NoError(err) - err = rg.PatchSettings(patch) - re.NoError(err) - res, err := json.Marshal(rg) - re.NoError(err) - re.Equal(ca.expectJSONString, string(res)) - } - - rg2 := &ResourceGroup{Name: "test", Mode: rmpb.GroupMode_RawMode, RawResourceSettings: &RawResourceSettings{}} - err = rg2.CheckAndInit() - re.NoError(err) - testCaseResource := []struct { - patchJSONString string - expectJSONString string - }{ - {`{"name":"test", "mode":2, "raw_resource_settings": {"cpu":{"settings":{"fill_rate": 200000}}}}`, - `{"name":"test","mode":2,"raw_resource_settings":{"cpu":{"settings":{"fill_rate":200000},"state":{"initialized":false}},"io_read_bandwidth":{"state":{"initialized":false}},"io_write_bandwidth":{"state":{"initialized":false}}}}`}, - {`{"name":"test", "mode":2, "raw_resource_settings": {"io_read":{"settings":{"fill_rate": 200000,"burst_limit":1000000}}}}`, - `{"name":"test","mode":2,"raw_resource_settings":{"cpu":{"state":{"initialized":false}},"io_read_bandwidth":{"settings":{"fill_rate":200000,"burst_limit":1000000},"state":{"initialized":false}},"io_write_bandwidth":{"state":{"initialized":false}}}}`}, - {`{"name":"test", "mode":2, "raw_resource_settings": {"io_write":{"settings":{"fill_rate": 200000}}}}`, - `{"name":"test","mode":2,"raw_resource_settings":{"cpu":{"state":{"initialized":false}},"io_read_bandwidth":{"state":{"initialized":false}},"io_write_bandwidth":{"settings":{"fill_rate":200000},"state":{"initialized":false}}}}`}, - } - - for _, ca := range testCaseResource { - rg := rg2.Copy() - patch := &rmpb.ResourceGroup{} - err := json.Unmarshal([]byte(ca.patchJSONString), patch) - re.NoError(err) - err = rg.PatchSettings(patch) - re.NoError(err) - res, err := json.Marshal(rg) - re.NoError(err) - re.Equal(ca.expectJSONString, string(res)) - } -} diff --git a/pkg/mcs/resource_manager/server/testutil.go b/pkg/mcs/resource_manager/server/testutil.go deleted file mode 100644 index 5a1aa37abcb..00000000000 --- a/pkg/mcs/resource_manager/server/testutil.go +++ /dev/null @@ -1,60 +0,0 @@ -// Copyright 2023 TiKV Project Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package server - -import ( - "context" - "os" - - "github.com/pingcap/log" - "github.com/spf13/cobra" - "github.com/stretchr/testify/require" - "github.com/tikv/pd/pkg/utils/logutil" -) - -// CleanupFunc closes test resource manager server(s) and deletes any files left behind. -type CleanupFunc func() - -// NewTestServer creates a resource manager server for testing. -func NewTestServer(ctx context.Context, re *require.Assertions, cfg *Config) (*Server, CleanupFunc, error) { - // New zap logger - err := logutil.SetupLogger(cfg.Log, &cfg.Logger, &cfg.LogProps, cfg.Security.RedactInfoLog) - re.NoError(err) - log.ReplaceGlobals(cfg.Logger, cfg.LogProps) - // Flushing any buffered log entries - defer log.Sync() - - s := NewServer(ctx, cfg) - if err = s.Run(); err != nil { - return nil, nil, err - } - - cleanup := func() { - s.Close() - os.RemoveAll(cfg.DataDir) - } - return s, cleanup, nil -} - -// NewTestDefaultConfig creates a new default config for testing. -func NewTestDefaultConfig() (*Config, error) { - cmd := &cobra.Command{ - Use: "resource_manager", - Short: "Run the resource manager service", - } - cfg := NewConfig() - flagSet := cmd.Flags() - return cfg, cfg.Parse(flagSet) -} diff --git a/pkg/mcs/resource_manager/server/token_bukets.go b/pkg/mcs/resource_manager/server/token_bukets.go deleted file mode 100644 index 47bc8b36c92..00000000000 --- a/pkg/mcs/resource_manager/server/token_bukets.go +++ /dev/null @@ -1,237 +0,0 @@ -// Copyright 2022 TiKV Project Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS,g -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package server - -import ( - "math" - "time" - - "github.com/gogo/protobuf/proto" - rmpb "github.com/pingcap/kvproto/pkg/resource_manager" -) - -const ( - defaultRefillRate = 10000 - defaultInitialTokens = 10 * 10000 -) - -const ( - defaultReserveRatio = 0.05 - defaultLoanCoefficient = 2 -) - -// GroupTokenBucket is a token bucket for a resource group. -// Now we don't save consumption in `GroupTokenBucket`, only statistics it in prometheus. -type GroupTokenBucket struct { - // Settings is the setting of TokenBucket. - // BurstLimit is used as below: - // - If b == 0, that means the limiter is unlimited capacity. default use in resource controller (burst with a rate within an unlimited capacity). - // - If b < 0, that means the limiter is unlimited capacity and fillrate(r) is ignored, can be seen as r == Inf (burst within a unlimited capacity). - // - If b > 0, that means the limiter is limited capacity. - // MaxTokens limits the number of tokens that can be accumulated - Settings *rmpb.TokenLimitSettings `json:"settings,omitempty"` - GroupTokenBucketState `json:"state,omitempty"` -} - -// GroupTokenBucketState is the running state of TokenBucket. -type GroupTokenBucketState struct { - Tokens float64 `json:"tokens,omitempty"` - LastUpdate *time.Time `json:"last_update,omitempty"` - Initialized bool `json:"initialized"` - // settingChanged is used to avoid that the number of tokens returned is jitter because of changing fill rate. - settingChanged bool -} - -// Clone returns the copy of GroupTokenBucketState -func (s *GroupTokenBucketState) Clone() *GroupTokenBucketState { - return &GroupTokenBucketState{ - Tokens: s.Tokens, - LastUpdate: s.LastUpdate, - Initialized: s.Initialized, - } -} - -// NewGroupTokenBucket returns a new GroupTokenBucket -func NewGroupTokenBucket(tokenBucket *rmpb.TokenBucket) GroupTokenBucket { - if tokenBucket == nil || tokenBucket.Settings == nil { - return GroupTokenBucket{} - } - return GroupTokenBucket{ - Settings: tokenBucket.Settings, - GroupTokenBucketState: GroupTokenBucketState{ - Tokens: tokenBucket.Tokens, - }, - } -} - -// GetTokenBucket returns the grpc protoc struct of GroupTokenBucket. -func (t *GroupTokenBucket) GetTokenBucket() *rmpb.TokenBucket { - if t.Settings == nil { - return nil - } - return &rmpb.TokenBucket{ - Settings: t.Settings, - Tokens: t.Tokens, - } -} - -// patch patches the token bucket settings. -func (t *GroupTokenBucket) patch(tb *rmpb.TokenBucket) { - if tb == nil { - return - } - if setting := proto.Clone(tb.GetSettings()).(*rmpb.TokenLimitSettings); setting != nil { - t.Settings = setting - t.settingChanged = true - } - - // the settings in token is delta of the last update and now. - t.Tokens += tb.GetTokens() -} - -// init initializes the group token bucket. -func (t *GroupTokenBucket) init(now time.Time) { - if t.Settings.FillRate == 0 { - t.Settings.FillRate = defaultRefillRate - } - if t.Tokens < defaultInitialTokens { - t.Tokens = defaultInitialTokens - } - t.LastUpdate = &now - t.Initialized = true -} - -// request requests tokens from the group token bucket. -func (t *GroupTokenBucket) request(now time.Time, neededTokens float64, targetPeriodMs uint64) (*rmpb.TokenBucket, int64) { - if !t.Initialized { - t.init(now) - } else { - delta := now.Sub(*t.LastUpdate) - if delta > 0 { - t.Tokens += float64(t.Settings.FillRate) * delta.Seconds() - t.LastUpdate = &now - } - } - // reloan when setting changed - if t.settingChanged && t.Tokens <= 0 { - t.Tokens = 0 - } - t.settingChanged = false - if t.Settings.BurstLimit != 0 { - if burst := float64(t.Settings.BurstLimit); t.Tokens > burst { - t.Tokens = burst - } - } - - var res rmpb.TokenBucket - res.Settings = &rmpb.TokenLimitSettings{BurstLimit: t.Settings.GetBurstLimit()} - // If BurstLimit is -1, just return. - if res.Settings.BurstLimit < 0 { - res.Tokens = neededTokens - return &res, 0 - } - // FillRate is used for the token server unavailable in abnormal situation. - if neededTokens <= 0 { - return &res, 0 - } - // If the current tokens can directly meet the requirement, returns the need token - if t.Tokens >= neededTokens { - t.Tokens -= neededTokens - // granted the total request tokens - res.Tokens = neededTokens - return &res, 0 - } - - // Firstly allocate the remaining tokens - var grantedTokens float64 - hasRemaining := false - if t.Tokens > 0 { - grantedTokens = t.Tokens - neededTokens -= grantedTokens - t.Tokens = 0 - hasRemaining = true - } - - var targetPeriodTime = time.Duration(targetPeriodMs) * time.Millisecond - var trickleTime = 0. - - LoanCoefficient := defaultLoanCoefficient - // when BurstLimit less or equal FillRate, the server does not accumulate a significant number of tokens. - // So we don't need to smooth the token allocation speed. - if t.Settings.BurstLimit > 0 && t.Settings.BurstLimit <= int64(t.Settings.FillRate) { - LoanCoefficient = 1 - } - // When there are loan, the allotment will match the fill rate. - // We will have k threshold, beyond which the token allocation will be a minimum. - // The threshold unit is `fill rate * target period`. - // | - // k*fill_rate |* * * * * * * - // | * - // *** | * - // | * - // | * - // fill_rate | * - // reserve_rate | * - // | - // grant_rate 0 ------------------------------------------------------------------------------------ - // loan *** k*period_token (k+k-1)*period_token *** (k+k+1...+1)*period_token - p := make([]float64, LoanCoefficient) - p[0] = float64(LoanCoefficient) * float64(t.Settings.FillRate) * targetPeriodTime.Seconds() - for i := 1; i < LoanCoefficient; i++ { - p[i] = float64(LoanCoefficient-i)*float64(t.Settings.FillRate)*targetPeriodTime.Seconds() + p[i-1] - } - for i := 0; i < LoanCoefficient && neededTokens > 0 && trickleTime < targetPeriodTime.Seconds(); i++ { - loan := -t.Tokens - if loan > p[i] { - continue - } - roundReserveTokens := p[i] - loan - fillRate := float64(LoanCoefficient-i) * float64(t.Settings.FillRate) - if roundReserveTokens > neededTokens { - t.Tokens -= neededTokens - grantedTokens += neededTokens - trickleTime += grantedTokens / fillRate - neededTokens = 0 - } else { - roundReserveTime := roundReserveTokens / fillRate - if roundReserveTime+trickleTime >= targetPeriodTime.Seconds() { - roundTokens := (targetPeriodTime.Seconds() - trickleTime) * fillRate - neededTokens -= roundTokens - t.Tokens -= roundTokens - grantedTokens += roundTokens - trickleTime = targetPeriodTime.Seconds() - } else { - grantedTokens += roundReserveTokens - neededTokens -= roundReserveTokens - t.Tokens -= roundReserveTokens - trickleTime += roundReserveTime - } - } - } - if grantedTokens < defaultReserveRatio*float64(t.Settings.FillRate)*targetPeriodTime.Seconds() { - t.Tokens -= defaultReserveRatio*float64(t.Settings.FillRate)*targetPeriodTime.Seconds() - grantedTokens - grantedTokens = defaultReserveRatio * float64(t.Settings.FillRate) * targetPeriodTime.Seconds() - } - res.Tokens = grantedTokens - var trickleDuration time.Duration - // can't directly treat targetPeriodTime as trickleTime when there is a token remaining. - // If treat, client consumption will be slowed down (actually cloud be increased). - if hasRemaining { - trickleDuration = time.Duration(math.Min(trickleTime, targetPeriodTime.Seconds()) * float64(time.Second)) - } else { - trickleDuration = targetPeriodTime - } - return &res, trickleDuration.Milliseconds() -} diff --git a/pkg/mcs/resource_manager/server/apis/v1/api.go b/pkg/mcs/resourcemanager/server/apis/v1/api.go similarity index 89% rename from pkg/mcs/resource_manager/server/apis/v1/api.go rename to pkg/mcs/resourcemanager/server/apis/v1/api.go index 9b70761690a..1b50fa1bf21 100644 --- a/pkg/mcs/resource_manager/server/apis/v1/api.go +++ b/pkg/mcs/resourcemanager/server/apis/v1/api.go @@ -17,12 +17,16 @@ package apis import ( "errors" "net/http" + "sync" "github.com/gin-contrib/cors" "github.com/gin-contrib/gzip" + "github.com/gin-contrib/pprof" "github.com/gin-gonic/gin" + "github.com/joho/godotenv" rmpb "github.com/pingcap/kvproto/pkg/resource_manager" - rmserver "github.com/tikv/pd/pkg/mcs/resource_manager/server" + rmserver "github.com/tikv/pd/pkg/mcs/resourcemanager/server" + "github.com/tikv/pd/pkg/mcs/utils" "github.com/tikv/pd/pkg/utils/apiutil" "github.com/tikv/pd/pkg/utils/apiutil/multiservicesapi" ) @@ -31,6 +35,7 @@ import ( const APIPathPrefix = "/resource-manager/api/v1/" var ( + once sync.Once apiServiceGroup = apiutil.APIServiceGroup{ Name: "resource-manager", Version: "v1", @@ -56,6 +61,11 @@ type Service struct { // NewService returns a new Service. func NewService(srv *rmserver.Service) *Service { + once.Do(func() { + // These global modification will be effective only for the first invoke. + _ = godotenv.Load() + gin.SetMode(gin.ReleaseMode) + }) apiHandlerEngine := gin.New() apiHandlerEngine.Use(gin.Recovery()) apiHandlerEngine.Use(cors.Default()) @@ -67,6 +77,8 @@ func NewService(srv *rmserver.Service) *Service { c.Next() }) apiHandlerEngine.Use(multiservicesapi.ServiceRedirector()) + apiHandlerEngine.GET("metrics", utils.PromHandler()) + pprof.Register(apiHandlerEngine) endpoint := apiHandlerEngine.Group(APIPathPrefix) s := &Service{ manager: manager, @@ -108,8 +120,7 @@ func (s *Service) postResourceGroup(c *gin.Context) { c.String(http.StatusBadRequest, err.Error()) return } - nGroup := rmserver.FromProtoResourceGroup(&group) - if err := s.manager.AddResourceGroup(nGroup); err != nil { + if err := s.manager.AddResourceGroup(&group); err != nil { c.String(http.StatusInternalServerError, err.Error()) return } @@ -142,7 +153,7 @@ func (s *Service) putResourceGroup(c *gin.Context) { // // @Tags ResourceManager // @Summary Get resource group by name. -// @Success 200 {string} json format of rmpb.ResourceGroup +// @Success 200 {string} json format of rmserver.ResourceGroup // @Failure 404 {string} error // @Param name path string true "groupName" // @Router /config/group/{name} [GET] @@ -158,7 +169,7 @@ func (s *Service) getResourceGroup(c *gin.Context) { // // @Tags ResourceManager // @Summary get all resource group with a list. -// @Success 200 {string} json format of []rmpb.ResourceGroup +// @Success 200 {string} json format of []rmserver.ResourceGroup // @Failure 404 {string} error // @Router /config/groups [GET] func (s *Service) getResourceGroupList(c *gin.Context) { diff --git a/pkg/mcs/resource_manager/server/config.go b/pkg/mcs/resourcemanager/server/config.go similarity index 64% rename from pkg/mcs/resource_manager/server/config.go rename to pkg/mcs/resourcemanager/server/config.go index 5e082d7a5c8..51fbe388458 100644 --- a/pkg/mcs/resource_manager/server/config.go +++ b/pkg/mcs/resourcemanager/server/config.go @@ -19,15 +19,18 @@ import ( "os" "path/filepath" "strings" + "time" "github.com/BurntSushi/toml" "github.com/pingcap/errors" + "github.com/pingcap/failpoint" "github.com/pingcap/log" "github.com/spf13/pflag" "github.com/tikv/pd/pkg/mcs/utils" "github.com/tikv/pd/pkg/utils/configutil" "github.com/tikv/pd/pkg/utils/grpcutil" "github.com/tikv/pd/pkg/utils/metricutil" + "github.com/tikv/pd/pkg/utils/typeutil" "go.uber.org/zap" ) @@ -36,23 +39,34 @@ const ( defaultBackendEndpoints = "http://127.0.0.1:2379" defaultListenAddr = "http://127.0.0.1:3379" - defaultReadBaseCost = 0.25 + // 1 RU = 8 storage read requests + defaultReadBaseCost = 1. / 8 + // 1 RU = 2 storage read batch requests + defaultReadPerBatchBaseCost = 1. / 2 + // 1 RU = 1 storage write request defaultWriteBaseCost = 1 + // 1 RU = 1 storage write batch request + defaultWritePerBatchBaseCost = 1 // 1 RU = 64 KiB read bytes defaultReadCostPerByte = 1. / (64 * 1024) // 1 RU = 1 KiB written bytes defaultWriteCostPerByte = 1. / 1024 // 1 RU = 3 millisecond CPU time defaultCPUMsCost = 1. / 3 + + // Because the resource manager has not been deployed in microservice mode, + // do not enable this function. + defaultDegradedModeWaitDuration = time.Second * 0 ) // Config is the configuration for the resource manager. type Config struct { - BackendEndpoints string `toml:"backend-endpoints" json:"backend-endpoints"` - ListenAddr string `toml:"listen-addr" json:"listen-addr"` - Name string `toml:"name" json:"name"` - DataDir string `toml:"data-dir" json:"data-dir"` // TODO: remove this after refactoring - EnableGRPCGateway bool `json:"enable-grpc-gateway"` // TODO: use it + BackendEndpoints string `toml:"backend-endpoints" json:"backend-endpoints"` + ListenAddr string `toml:"listen-addr" json:"listen-addr"` + AdvertiseListenAddr string `toml:"advertise-listen-addr" json:"advertise-listen-addr"` + Name string `toml:"name" json:"name"` + DataDir string `toml:"data-dir" json:"data-dir"` // TODO: remove this after refactoring + EnableGRPCGateway bool `json:"enable-grpc-gateway"` // TODO: use it Metric metricutil.MetricConfig `toml:"metric" json:"metric"` @@ -63,28 +77,57 @@ type Config struct { Security configutil.SecurityConfig `toml:"security" json:"security"` + // WarningMsgs contains all warnings during parsing. + WarningMsgs []string + // LeaderLease defines the time within which a Resource Manager primary/leader must // update its TTL in etcd, otherwise etcd will expire the leader key and other servers // can campaign the primary/leader again. Etcd only supports seconds TTL, so here is // second too. LeaderLease int64 `toml:"lease" json:"lease"` + Controller ControllerConfig `toml:"controller" json:"controller"` +} + +// ControllerConfig is the configuration of the resource manager controller which includes some option for client needed. +type ControllerConfig struct { + // EnableDegradedMode is to control whether resource control client enable degraded mode when server is disconnect. + DegradedModeWaitDuration typeutil.Duration `toml:"degraded-mode-wait-duration" json:"degraded-mode-wait-duration"` + // RequestUnit is the configuration determines the coefficients of the RRU and WRU cost. // This configuration should be modified carefully. - RequestUnit RequestUnitConfig + RequestUnit RequestUnitConfig `toml:"request-unit" json:"request-unit"` +} + +// Adjust adjusts the configuration and initializes it with the default value if necessary. +func (rmc *ControllerConfig) Adjust(meta *configutil.ConfigMetaData) { + if rmc == nil { + return + } + rmc.RequestUnit.Adjust() + + configutil.AdjustDuration(&rmc.DegradedModeWaitDuration, defaultDegradedModeWaitDuration) + failpoint.Inject("enableDegradedMode", func() { + configutil.AdjustDuration(&rmc.DegradedModeWaitDuration, time.Second) + }) } // RequestUnitConfig is the configuration of the request units, which determines the coefficients of -// the RRU and WRU cost. +// the RRU and WRU cost. This configuration should be modified carefully. +// TODO: use common config with client size. type RequestUnitConfig struct { // ReadBaseCost is the base cost for a read request. No matter how many bytes read/written or // the CPU times taken for a request, this cost is inevitable. ReadBaseCost float64 `toml:"read-base-cost" json:"read-base-cost"` + // ReadPerBatchBaseCost is the base cost for a read request with batch. + ReadPerBatchBaseCost float64 `toml:"read-per-batch-base-cost" json:"read-per-batch-base-cost"` // ReadCostPerByte is the cost for each byte read. It's 1 RU = 64 KiB by default. ReadCostPerByte float64 `toml:"read-cost-per-byte" json:"read-cost-per-byte"` // WriteBaseCost is the base cost for a write request. No matter how many bytes read/written or // the CPU times taken for a request, this cost is inevitable. WriteBaseCost float64 `toml:"write-base-cost" json:"write-base-cost"` + // WritePerBatchBaseCost is the base cost for a write request with batch. + WritePerBatchBaseCost float64 `toml:"write-per-batch-base-cost" json:"write-per-batch-base-cost"` // WriteCostPerByte is the cost for each byte written. It's 1 RU = 1 KiB by default. WriteCostPerByte float64 `toml:"write-cost-per-byte" json:"write-cost-per-byte"` // CPUMsCost is the cost for each millisecond of CPU time taken. @@ -100,12 +143,18 @@ func (ruc *RequestUnitConfig) Adjust() { if ruc.ReadBaseCost == 0 { ruc.ReadBaseCost = defaultReadBaseCost } + if ruc.ReadPerBatchBaseCost == 0 { + ruc.ReadPerBatchBaseCost = defaultReadPerBatchBaseCost + } if ruc.ReadCostPerByte == 0 { ruc.ReadCostPerByte = defaultReadCostPerByte } if ruc.WriteBaseCost == 0 { ruc.WriteBaseCost = defaultWriteBaseCost } + if ruc.WritePerBatchBaseCost == 0 { + ruc.WritePerBatchBaseCost = defaultWritePerBatchBaseCost + } if ruc.WriteCostPerByte == 0 { ruc.WriteCostPerByte = defaultWriteCostPerByte } @@ -134,14 +183,15 @@ func (c *Config) Parse(flagSet *pflag.FlagSet) error { } // Ignore the error check here - configutil.AdjustCommandlineString(flagSet, &c.Log.Level, "log-level") - configutil.AdjustCommandlineString(flagSet, &c.Log.File.Filename, "log-file") - configutil.AdjustCommandlineString(flagSet, &c.Metric.PushAddress, "metrics-addr") - configutil.AdjustCommandlineString(flagSet, &c.Security.CAPath, "cacert") - configutil.AdjustCommandlineString(flagSet, &c.Security.CertPath, "cert") - configutil.AdjustCommandlineString(flagSet, &c.Security.KeyPath, "key") - configutil.AdjustCommandlineString(flagSet, &c.BackendEndpoints, "backend-endpoints") - configutil.AdjustCommandlineString(flagSet, &c.ListenAddr, "listen-addr") + configutil.AdjustCommandLineString(flagSet, &c.Log.Level, "log-level") + configutil.AdjustCommandLineString(flagSet, &c.Log.File.Filename, "log-file") + configutil.AdjustCommandLineString(flagSet, &c.Metric.PushAddress, "metrics-addr") + configutil.AdjustCommandLineString(flagSet, &c.Security.CAPath, "cacert") + configutil.AdjustCommandLineString(flagSet, &c.Security.CertPath, "cert") + configutil.AdjustCommandLineString(flagSet, &c.Security.KeyPath, "key") + configutil.AdjustCommandLineString(flagSet, &c.BackendEndpoints, "backend-endpoints") + configutil.AdjustCommandLineString(flagSet, &c.ListenAddr, "listen-addr") + configutil.AdjustCommandLineString(flagSet, &c.AdvertiseListenAddr, "advertise-listen-addr") return c.Adjust(meta, false) } @@ -149,11 +199,9 @@ func (c *Config) Parse(flagSet *pflag.FlagSet) error { // Adjust is used to adjust the resource manager configurations. func (c *Config) Adjust(meta *toml.MetaData, reloading bool) error { configMetaData := configutil.NewConfigMetadata(meta) - warningMsgs := make([]string, 0) if err := configMetaData.CheckUndecoded(); err != nil { - warningMsgs = append(warningMsgs, err.Error()) + c.WarningMsgs = append(c.WarningMsgs, err.Error()) } - configutil.PrintConfigCheckMsg(os.Stdout, warningMsgs) if c.Name == "" { hostname, err := os.Hostname() @@ -171,6 +219,7 @@ func (c *Config) Adjust(meta *toml.MetaData, reloading bool) error { configutil.AdjustString(&c.BackendEndpoints, defaultBackendEndpoints) configutil.AdjustString(&c.ListenAddr, defaultListenAddr) + configutil.AdjustString(&c.AdvertiseListenAddr, c.ListenAddr) if !configMetaData.IsDefined("enable-grpc-gateway") { c.EnableGRPCGateway = utils.DefaultEnableGRPCGateway @@ -183,10 +232,9 @@ func (c *Config) Adjust(meta *toml.MetaData, reloading bool) error { c.Log.Format = utils.DefaultLogFormat } + c.Controller.Adjust(configMetaData.Child("controller")) configutil.AdjustInt64(&c.LeaderLease, utils.DefaultLeaderLease) - c.RequestUnit.Adjust() - return nil } diff --git a/pkg/mcs/resourcemanager/server/config_test.go b/pkg/mcs/resourcemanager/server/config_test.go new file mode 100644 index 00000000000..c0cac4da9c0 --- /dev/null +++ b/pkg/mcs/resourcemanager/server/config_test.go @@ -0,0 +1,50 @@ +// Copyright 2022 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS,g +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "math" + "testing" + "time" + + "github.com/BurntSushi/toml" + "github.com/stretchr/testify/require" +) + +func TestControllerConfig(t *testing.T) { + re := require.New(t) + cfgData := ` +[controller] +degraded-mode-wait-duration = "2s" +[controller.request-unit] +read-base-cost = 1.0 +read-cost-per-byte = 2.0 +write-base-cost = 3.0 +write-cost-per-byte = 4.0 +read-cpu-ms-cost = 5.0 +` + cfg := NewConfig() + meta, err := toml.Decode(cfgData, &cfg) + re.NoError(err) + err = cfg.Adjust(&meta, false) + re.NoError(err) + + re.Equal(cfg.Controller.DegradedModeWaitDuration.Duration, time.Second*2) + re.LessOrEqual(math.Abs(cfg.Controller.RequestUnit.CPUMsCost-5), 1e-7) + re.LessOrEqual(math.Abs(cfg.Controller.RequestUnit.WriteCostPerByte-4), 1e-7) + re.LessOrEqual(math.Abs(cfg.Controller.RequestUnit.WriteBaseCost-3), 1e-7) + re.LessOrEqual(math.Abs(cfg.Controller.RequestUnit.ReadCostPerByte-2), 1e-7) + re.LessOrEqual(math.Abs(cfg.Controller.RequestUnit.ReadBaseCost-1), 1e-7) +} diff --git a/pkg/mcs/resource_manager/server/grpc_service.go b/pkg/mcs/resourcemanager/server/grpc_service.go similarity index 90% rename from pkg/mcs/resource_manager/server/grpc_service.go rename to pkg/mcs/resourcemanager/server/grpc_service.go index 72e0e8fc40c..cf198304dfc 100644 --- a/pkg/mcs/resource_manager/server/grpc_service.go +++ b/pkg/mcs/resourcemanager/server/grpc_service.go @@ -21,6 +21,7 @@ import ( "time" "github.com/pingcap/errors" + "github.com/pingcap/failpoint" rmpb "github.com/pingcap/kvproto/pkg/resource_manager" "github.com/pingcap/log" bs "github.com/tikv/pd/pkg/basicserver" @@ -59,7 +60,7 @@ type Service struct { } // NewService creates a new resource manager service. -func NewService[T RUConfigProvider](svr bs.Server) registry.RegistrableService { +func NewService[T ResourceManagerConfigProvider](svr bs.Server) registry.RegistrableService { manager := NewManager[T](svr) return &Service{ @@ -85,7 +86,7 @@ func (s *Service) GetManager() *Manager { } func (s *Service) checkServing() error { - if !s.manager.srv.IsServing() { + if s.manager == nil || s.manager.srv == nil || !s.manager.srv.IsServing() { return errNotLeader } return nil @@ -125,8 +126,7 @@ func (s *Service) AddResourceGroup(ctx context.Context, req *rmpb.PutResourceGro if err := s.checkServing(); err != nil { return nil, err } - rg := FromProtoResourceGroup(req.GetGroup()) - err := s.manager.AddResourceGroup(rg) + err := s.manager.AddResourceGroup(req.GetGroup()) if err != nil { return nil, err } @@ -169,6 +169,9 @@ func (s *Service) AcquireTokenBuckets(stream rmpb.ResourceManager_AcquireTokenBu if err == io.EOF { return nil } + failpoint.Inject("acquireFailed", func() { + err = errors.New("error") + }) if err != nil { return errors.WithStack(err) } @@ -176,6 +179,7 @@ func (s *Service) AcquireTokenBuckets(stream rmpb.ResourceManager_AcquireTokenBu return err } targetPeriodMs := request.GetTargetRequestPeriodMs() + clientUniqueID := request.GetClientUniqueId() resps := &rmpb.TokenBucketsResponse{} for _, req := range request.Requests { resourceGroupName := req.GetResourceGroupName() @@ -186,10 +190,15 @@ func (s *Service) AcquireTokenBuckets(stream rmpb.ResourceManager_AcquireTokenBu continue } // Send the consumption to update the metrics. + isBackground := req.GetIsBackground() s.manager.consumptionDispatcher <- struct { resourceGroupName string *rmpb.Consumption - }{resourceGroupName, req.GetConsumptionSinceLastRequest()} + isBackground bool + }{resourceGroupName, req.GetConsumptionSinceLastRequest(), isBackground} + if isBackground { + continue + } now := time.Now() resp := &rmpb.TokenBucketResponse{ ResourceGroupName: rg.Name, @@ -199,7 +208,7 @@ func (s *Service) AcquireTokenBuckets(stream rmpb.ResourceManager_AcquireTokenBu var tokens *rmpb.GrantedRUTokenBucket for _, re := range req.GetRuItems().GetRequestRU() { if re.Type == rmpb.RequestUnitType_RU { - tokens = rg.RequestRU(now, re.Value, targetPeriodMs) + tokens = rg.RequestRU(now, re.Value, targetPeriodMs, clientUniqueID) } if tokens == nil { continue @@ -210,7 +219,7 @@ func (s *Service) AcquireTokenBuckets(stream rmpb.ResourceManager_AcquireTokenBu log.Warn("not supports the resource type", zap.String("resource-group", resourceGroupName), zap.String("mode", rmpb.GroupMode_name[int32(rmpb.GroupMode_RawMode)])) continue } - log.Debug("finish token request from", zap.String("resource group", resourceGroupName)) + log.Debug("finish token request from", zap.String("resource-group", resourceGroupName)) resps.Responses = append(resps.Responses, resp) } stream.Send(resps) diff --git a/pkg/mcs/resource_manager/server/install/install.go b/pkg/mcs/resourcemanager/server/install/install.go similarity index 87% rename from pkg/mcs/resource_manager/server/install/install.go rename to pkg/mcs/resourcemanager/server/install/install.go index e8410f12b3b..89a4ac1914f 100644 --- a/pkg/mcs/resource_manager/server/install/install.go +++ b/pkg/mcs/resourcemanager/server/install/install.go @@ -16,10 +16,10 @@ package install import ( "github.com/tikv/pd/pkg/mcs/registry" - rm_server "github.com/tikv/pd/pkg/mcs/resource_manager/server" + rm_server "github.com/tikv/pd/pkg/mcs/resourcemanager/server" // init API group - _ "github.com/tikv/pd/pkg/mcs/resource_manager/server/apis/v1" + _ "github.com/tikv/pd/pkg/mcs/resourcemanager/server/apis/v1" ) func init() { diff --git a/pkg/mcs/resource_manager/server/manager.go b/pkg/mcs/resourcemanager/server/manager.go similarity index 68% rename from pkg/mcs/resource_manager/server/manager.go rename to pkg/mcs/resourcemanager/server/manager.go index 414ee392f82..18d8c1f4d3c 100644 --- a/pkg/mcs/resource_manager/server/manager.go +++ b/pkg/mcs/resourcemanager/server/manager.go @@ -17,18 +17,20 @@ package server import ( "context" "encoding/json" + "math" "sort" "sync" "time" "github.com/gogo/protobuf/proto" - "github.com/pingcap/errors" "github.com/pingcap/failpoint" rmpb "github.com/pingcap/kvproto/pkg/resource_manager" "github.com/pingcap/log" bs "github.com/tikv/pd/pkg/basicserver" + "github.com/tikv/pd/pkg/errs" "github.com/tikv/pd/pkg/storage/endpoint" "github.com/tikv/pd/pkg/storage/kv" + "github.com/tikv/pd/pkg/utils/logutil" "go.uber.org/zap" ) @@ -36,40 +38,46 @@ const ( defaultConsumptionChanSize = 1024 metricsCleanupInterval = time.Minute metricsCleanupTimeout = 20 * time.Minute + metricsAvailableRUInterval = 30 * time.Second + + reservedDefaultGroupName = "default" + middlePriority = 8 ) // Manager is the manager of resource group. type Manager struct { sync.RWMutex - srv bs.Server - ruConfig *RequestUnitConfig - groups map[string]*ResourceGroup - storage endpoint.ResourceGroupStorage + srv bs.Server + controllerConfig *ControllerConfig + groups map[string]*ResourceGroup + storage endpoint.ResourceGroupStorage // consumptionChan is used to send the consumption // info to the background metrics flusher. consumptionDispatcher chan struct { resourceGroupName string *rmpb.Consumption + isBackground bool } // record update time of each resource group consumptionRecord map[string]time.Time } -// RUConfigProvider is used to get RU config from the given +// ResourceManagerConfigProvider is used to get resource manager config from the given // `bs.server` without modifying its interface. -type RUConfigProvider interface { - GetRequestUnitConfig() *RequestUnitConfig +type ResourceManagerConfigProvider interface { + GetControllerConfig() *ControllerConfig } // NewManager returns a new manager base on the given server, -// which should implement the `RUConfigProvider` interface. -func NewManager[T RUConfigProvider](srv bs.Server) *Manager { +// which should implement the `ResourceManagerConfigProvider` interface. +func NewManager[T ResourceManagerConfigProvider](srv bs.Server) *Manager { m := &Manager{ - ruConfig: srv.(T).GetRequestUnitConfig(), - groups: make(map[string]*ResourceGroup), + controllerConfig: srv.(T).GetControllerConfig(), + groups: make(map[string]*ResourceGroup), consumptionDispatcher: make(chan struct { resourceGroupName string *rmpb.Consumption + isBackground bool }, defaultConsumptionChanSize), consumptionRecord: make(map[string]time.Time), } @@ -94,8 +102,9 @@ func (m *Manager) GetBasicServer() bs.Server { // Init initializes the resource group manager. func (m *Manager) Init(ctx context.Context) { - // Store the RU model config into the storage. - m.storage.SaveRequestUnitConfig(m.ruConfig) + // Todo: If we can modify following configs in the future, we should reload these configs. + // Store the controller config into the storage. + m.storage.SaveControllerConfig(m.controllerConfig) // Load resource group meta info from storage. m.groups = make(map[string]*ResourceGroup) handler := func(k, v string) { @@ -119,24 +128,49 @@ func (m *Manager) Init(ctx context.Context) { } } m.storage.LoadResourceGroupStates(tokenHandler) + + // Add default group if it's not inited. + if _, ok := m.groups[reservedDefaultGroupName]; !ok { + defaultGroup := &ResourceGroup{ + Name: reservedDefaultGroupName, + Mode: rmpb.GroupMode_RUMode, + RUSettings: &RequestUnitSettings{ + RU: &GroupTokenBucket{ + Settings: &rmpb.TokenLimitSettings{ + FillRate: math.MaxInt32, + BurstLimit: -1, + }, + }, + }, + Priority: middlePriority, + } + if err := m.AddResourceGroup(defaultGroup.IntoProtoResourceGroup()); err != nil { + log.Warn("init default group failed", zap.Error(err)) + } + } + // Start the background metrics flusher. go m.backgroundMetricsFlush(ctx) - go m.persistLoop(ctx) + go func() { + defer logutil.LogPanic() + m.persistLoop(ctx) + }() log.Info("resource group manager finishes initialization") } // AddResourceGroup puts a resource group. -func (m *Manager) AddResourceGroup(group *ResourceGroup) error { - m.RLock() - _, ok := m.groups[group.Name] - m.RUnlock() - if ok { - return errors.New("this group already exists") +// NOTE: AddResourceGroup should also be idempotent because tidb depends +// on this retry mechanism. +func (m *Manager) AddResourceGroup(grouppb *rmpb.ResourceGroup) error { + // Check the name. + if len(grouppb.Name) == 0 || len(grouppb.Name) > 32 { + return errs.ErrInvalidGroup } - err := group.CheckAndInit() - if err != nil { - return err + // Check the Priority. + if grouppb.GetPriority() > 16 { + return errs.ErrInvalidGroup } + group := FromProtoResourceGroup(grouppb) m.Lock() defer m.Unlock() if err := group.persistSettings(m.storage); err != nil { @@ -152,13 +186,13 @@ func (m *Manager) AddResourceGroup(group *ResourceGroup) error { // ModifyResourceGroup modifies an existing resource group. func (m *Manager) ModifyResourceGroup(group *rmpb.ResourceGroup) error { if group == nil || group.Name == "" { - return errors.New("invalid group name") + return errs.ErrInvalidGroup } m.Lock() curGroup, ok := m.groups[group.Name] m.Unlock() if !ok { - return errors.New("not exists the group") + return errs.ErrResourceGroupNotExists.FastGenByArgs(group.Name) } err := curGroup.PatchSettings(group) @@ -170,6 +204,9 @@ func (m *Manager) ModifyResourceGroup(group *rmpb.ResourceGroup) error { // DeleteResourceGroup deletes a resource group. func (m *Manager) DeleteResourceGroup(name string) error { + if name == reservedDefaultGroupName { + return errs.ErrDeleteReservedGroup + } if err := m.storage.DeleteResourceGroupSetting(name); err != nil { return err } @@ -242,15 +279,20 @@ func (m *Manager) persistResourceGroupRunningState() { group, ok := m.groups[keys[idx]] m.RUnlock() if ok { + m.Lock() group.persistStates(m.storage) + m.Unlock() } } } // Receive the consumption and flush it to the metrics. func (m *Manager) backgroundMetricsFlush(ctx context.Context) { - ticker := time.NewTicker(metricsCleanupInterval) - defer ticker.Stop() + defer logutil.LogPanic() + cleanUpTicker := time.NewTicker(metricsCleanupInterval) + defer cleanUpTicker.Stop() + availableRUTicker := time.NewTicker(metricsAvailableRUInterval) + defer availableRUTicker.Stop() for { select { case <-ctx.Done(): @@ -260,39 +302,44 @@ func (m *Manager) backgroundMetricsFlush(ctx context.Context) { if consumption == nil { continue } + backgroundType := "" + if consumptionInfo.isBackground { + backgroundType = backgroundTypeLabel + } + var ( name = consumptionInfo.resourceGroupName - rruMetrics = readRequestUnitCost.WithLabelValues(name) - wruMetrics = writeRequestUnitCost.WithLabelValues(name) + rruMetrics = readRequestUnitCost.WithLabelValues(name, backgroundType) + wruMetrics = writeRequestUnitCost.WithLabelValues(name, backgroundType) sqlLayerRuMetrics = sqlLayerRequestUnitCost.WithLabelValues(name) - readByteMetrics = readByteCost.WithLabelValues(name) - writeByteMetrics = writeByteCost.WithLabelValues(name) - kvCPUMetrics = kvCPUCost.WithLabelValues(name) - sqlCPUMetrics = sqlCPUCost.WithLabelValues(name) + readByteMetrics = readByteCost.WithLabelValues(name, backgroundType) + writeByteMetrics = writeByteCost.WithLabelValues(name, backgroundType) + kvCPUMetrics = kvCPUCost.WithLabelValues(name, backgroundType) + sqlCPUMetrics = sqlCPUCost.WithLabelValues(name, backgroundType) readRequestCountMetrics = requestCount.WithLabelValues(name, readTypeLabel) writeRequestCountMetrics = requestCount.WithLabelValues(name, writeTypeLabel) ) // RU info. if consumption.RRU != 0 { - rruMetrics.Observe(consumption.RRU) + rruMetrics.Add(consumption.RRU) } if consumption.WRU != 0 { - wruMetrics.Observe(consumption.WRU) + wruMetrics.Add(consumption.WRU) } // Byte info. if consumption.ReadBytes != 0 { - readByteMetrics.Observe(consumption.ReadBytes) + readByteMetrics.Add(consumption.ReadBytes) } if consumption.WriteBytes != 0 { - writeByteMetrics.Observe(consumption.WriteBytes) + writeByteMetrics.Add(consumption.WriteBytes) } // CPU time info. if consumption.TotalCpuTimeMs > 0 { if consumption.SqlLayerCpuTimeMs > 0 { - sqlLayerRuMetrics.Add(consumption.SqlLayerCpuTimeMs * m.ruConfig.CPUMsCost) - sqlCPUMetrics.Observe(consumption.SqlLayerCpuTimeMs) + sqlLayerRuMetrics.Add(consumption.SqlLayerCpuTimeMs * m.controllerConfig.RequestUnit.CPUMsCost) + sqlCPUMetrics.Add(consumption.SqlLayerCpuTimeMs) } - kvCPUMetrics.Observe(consumption.TotalCpuTimeMs - consumption.SqlLayerCpuTimeMs) + kvCPUMetrics.Add(consumption.TotalCpuTimeMs - consumption.SqlLayerCpuTimeMs) } // RPC count info. if consumption.KvReadRpcCount != 0 { @@ -304,7 +351,7 @@ func (m *Manager) backgroundMetricsFlush(ctx context.Context) { m.consumptionRecord[name] = time.Now() - case <-ticker.C: + case <-cleanUpTicker.C: // Clean up the metrics that have not been updated for a long time. for name, lastTime := range m.consumptionRecord { if time.Since(lastTime) > metricsCleanupTimeout { @@ -317,9 +364,23 @@ func (m *Manager) backgroundMetricsFlush(ctx context.Context) { sqlCPUCost.DeleteLabelValues(name) requestCount.DeleteLabelValues(name, readTypeLabel) requestCount.DeleteLabelValues(name, writeTypeLabel) + availableRUCounter.DeleteLabelValues(name) delete(m.consumptionRecord, name) } } + case <-availableRUTicker.C: + m.RLock() + for name, group := range m.groups { + if name == reservedDefaultGroupName { + continue + } + ru := group.getRUToken() + if ru < 0 { + ru = 0 + } + availableRUCounter.WithLabelValues(name).Set(ru) + } + m.RUnlock() } } } diff --git a/pkg/mcs/resource_manager/server/metrics.go b/pkg/mcs/resourcemanager/server/metrics.go similarity index 55% rename from pkg/mcs/resource_manager/server/metrics.go rename to pkg/mcs/resourcemanager/server/metrics.go index 6ecf93b1d45..083c44894ef 100644 --- a/pkg/mcs/resource_manager/server/metrics.go +++ b/pkg/mcs/resourcemanager/server/metrics.go @@ -14,9 +14,7 @@ package server -import ( - "github.com/prometheus/client_golang/prometheus" -) +import "github.com/prometheus/client_golang/prometheus" const ( namespace = "resource_manager" @@ -27,6 +25,7 @@ const ( typeLabel = "type" readTypeLabel = "read" writeTypeLabel = "write" + backgroundTypeLabel = "background" ) var ( @@ -39,70 +38,73 @@ var ( Help: "Indicate the resource manager server info, and the value is the start timestamp (s).", }, []string{"version", "hash"}) // RU cost metrics. - readRequestUnitCost = prometheus.NewHistogramVec( - prometheus.HistogramOpts{ + // `sum` is added to the name to maintain compatibility with the previous use of histogram. + readRequestUnitCost = prometheus.NewCounterVec( + prometheus.CounterOpts{ Namespace: namespace, Subsystem: ruSubsystem, - Name: "read_request_unit", - Help: "Bucketed histogram of the read request unit cost for all resource groups.", - Buckets: prometheus.ExponentialBuckets(1, 10, 5), // 1 ~ 100000 - }, []string{resourceGroupNameLabel}) - writeRequestUnitCost = prometheus.NewHistogramVec( - prometheus.HistogramOpts{ + Name: "read_request_unit_sum", + Help: "Counter of the read request unit cost for all resource groups.", + }, []string{resourceGroupNameLabel, typeLabel}) + writeRequestUnitCost = prometheus.NewCounterVec( + prometheus.CounterOpts{ Namespace: namespace, Subsystem: ruSubsystem, - Name: "write_request_unit", - Help: "Bucketed histogram of the write request unit cost for all resource groups.", - Buckets: prometheus.ExponentialBuckets(3, 10, 5), // 3 ~ 300000 - }, []string{resourceGroupNameLabel}) - sqlLayerRequestUnitCost = prometheus.NewGaugeVec( - prometheus.GaugeOpts{ + Name: "write_request_unit_sum", + Help: "Counter of the write request unit cost for all resource groups.", + }, []string{resourceGroupNameLabel, typeLabel}) + sqlLayerRequestUnitCost = prometheus.NewCounterVec( + prometheus.CounterOpts{ Namespace: namespace, Subsystem: ruSubsystem, - Name: "sql_layer_request_unit", + Name: "sql_layer_request_unit_sum", Help: "The number of the sql layer request unit cost for all resource groups.", }, []string{resourceGroupNameLabel}) // Resource cost metrics. - readByteCost = prometheus.NewHistogramVec( - prometheus.HistogramOpts{ + readByteCost = prometheus.NewCounterVec( + prometheus.CounterOpts{ Namespace: namespace, Subsystem: resourceSubsystem, - Name: "read_byte", - Help: "Bucketed histogram of the read byte cost for all resource groups.", - Buckets: prometheus.ExponentialBuckets(1, 8, 12), - }, []string{resourceGroupNameLabel}) - writeByteCost = prometheus.NewHistogramVec( - prometheus.HistogramOpts{ + Name: "read_byte_sum", + Help: "Counter of the read byte cost for all resource groups.", + }, []string{resourceGroupNameLabel, typeLabel}) + writeByteCost = prometheus.NewCounterVec( + prometheus.CounterOpts{ Namespace: namespace, Subsystem: resourceSubsystem, - Name: "write_byte", - Help: "Bucketed histogram of the write byte cost for all resource groups.", - Buckets: prometheus.ExponentialBuckets(1, 8, 12), - }, []string{resourceGroupNameLabel}) - kvCPUCost = prometheus.NewHistogramVec( - prometheus.HistogramOpts{ + Name: "write_byte_sum", + Help: "Counter of the write byte cost for all resource groups.", + }, []string{resourceGroupNameLabel, typeLabel}) + kvCPUCost = prometheus.NewCounterVec( + prometheus.CounterOpts{ Namespace: namespace, Subsystem: resourceSubsystem, - Name: "kv_cpu_time_ms", - Help: "Bucketed histogram of the KV CPU time cost in milliseconds for all resource groups.", - Buckets: prometheus.ExponentialBuckets(1, 10, 3), // 1 ~ 1000 - }, []string{resourceGroupNameLabel}) - sqlCPUCost = prometheus.NewHistogramVec( - prometheus.HistogramOpts{ + Name: "kv_cpu_time_ms_sum", + Help: "Counter of the KV CPU time cost in milliseconds for all resource groups.", + }, []string{resourceGroupNameLabel, typeLabel}) + sqlCPUCost = prometheus.NewCounterVec( + prometheus.CounterOpts{ Namespace: namespace, Subsystem: resourceSubsystem, - Name: "sql_cpu_time_ms", - Help: "Bucketed histogram of the SQL CPU time cost in milliseconds for all resource groups.", - Buckets: prometheus.ExponentialBuckets(1, 10, 3), // 1 ~ 1000 - }, []string{resourceGroupNameLabel}) - requestCount = prometheus.NewGaugeVec( - prometheus.GaugeOpts{ + Name: "sql_cpu_time_ms_sum", + Help: "Counter of the SQL CPU time cost in milliseconds for all resource groups.", + }, []string{resourceGroupNameLabel, typeLabel}) + requestCount = prometheus.NewCounterVec( + prometheus.CounterOpts{ Namespace: namespace, Subsystem: resourceSubsystem, Name: "request_count", Help: "The number of read/write requests for all resource groups.", }, []string{resourceGroupNameLabel, typeLabel}) + + availableRUCounter = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: ruSubsystem, + Name: "available_ru", + Help: "Counter of the available RU for all resource groups.", + }, []string{resourceGroupNameLabel}) ) func init() { @@ -115,4 +117,5 @@ func init() { prometheus.MustRegister(kvCPUCost) prometheus.MustRegister(sqlCPUCost) prometheus.MustRegister(requestCount) + prometheus.MustRegister(availableRUCounter) } diff --git a/pkg/mcs/resource_manager/server/resource_group.go b/pkg/mcs/resourcemanager/server/resource_group.go similarity index 58% rename from pkg/mcs/resource_manager/server/resource_group.go rename to pkg/mcs/resourcemanager/server/resource_group.go index 9b616e39e33..74bc463002b 100644 --- a/pkg/mcs/resource_manager/server/resource_group.go +++ b/pkg/mcs/resourcemanager/server/resource_group.go @@ -33,14 +33,15 @@ type ResourceGroup struct { Name string `json:"name"` Mode rmpb.GroupMode `json:"mode"` // RU settings - RUSettings *RequestUnitSettings `json:"r_u_settings,omitempty"` - // raw resource settings - RawResourceSettings *RawResourceSettings `json:"raw_resource_settings,omitempty"` + RUSettings *RequestUnitSettings `json:"r_u_settings,omitempty"` + Priority uint32 `json:"priority"` + Runaway *rmpb.RunawaySettings `json:"runaway_settings,omitempty"` + Background *rmpb.BackgroundSettings `json:"background_settings,omitempty"` } // RequestUnitSettings is the definition of the RU settings. type RequestUnitSettings struct { - RU GroupTokenBucket `json:"r_u,omitempty"` + RU *GroupTokenBucket `json:"r_u,omitempty"` } // NewRequestUnitSettings creates a new RequestUnitSettings with the given token bucket. @@ -50,22 +51,6 @@ func NewRequestUnitSettings(tokenBucket *rmpb.TokenBucket) *RequestUnitSettings } } -// RawResourceSettings is the definition of the native resource settings. -type RawResourceSettings struct { - CPU GroupTokenBucket `json:"cpu,omitempty"` - IOReadBandwidth GroupTokenBucket `json:"io_read_bandwidth,omitempty"` - IOWriteBandwidth GroupTokenBucket `json:"io_write_bandwidth,omitempty"` -} - -// NewRawResourceSettings creates a new RawResourceSettings with the given token buckets. -func NewRawResourceSettings(cpu, ioRead, ioWrite *rmpb.TokenBucket) *RawResourceSettings { - return &RawResourceSettings{ - CPU: NewGroupTokenBucket(cpu), - IOReadBandwidth: NewGroupTokenBucket(ioRead), - IOWriteBandwidth: NewGroupTokenBucket(ioWrite), - } -} - func (rg *ResourceGroup) String() string { res, err := json.Marshal(rg) if err != nil { @@ -92,31 +77,10 @@ func (rg *ResourceGroup) Copy() *ResourceGroup { return &newRG } -// CheckAndInit checks the validity of the resource group and initializes the default values if not setting. -// Only used to initialize the resource group when creating. -func (rg *ResourceGroup) CheckAndInit() error { - if len(rg.Name) == 0 || len(rg.Name) > 32 { - return errors.New("invalid resource group name, the length should be in [1,32]") - } - switch rg.Mode { - case rmpb.GroupMode_RUMode: - if rg.RUSettings == nil { - rg.RUSettings = NewRequestUnitSettings(nil) - } - if rg.RawResourceSettings != nil { - return errors.New("invalid resource group settings, RU mode should not set raw resource settings") - } - case rmpb.GroupMode_RawMode: - if rg.RawResourceSettings == nil { - rg.RawResourceSettings = NewRawResourceSettings(nil, nil, nil) - } - if rg.RUSettings != nil { - return errors.New("invalid resource group settings, raw mode should not set RU settings") - } - default: - return errors.New("invalid resource group mode") - } - return nil +func (rg *ResourceGroup) getRUToken() float64 { + rg.Lock() + defer rg.Unlock() + return rg.RUSettings.RU.Tokens } // PatchSettings patches the resource group settings. @@ -125,22 +89,25 @@ func (rg *ResourceGroup) CheckAndInit() error { func (rg *ResourceGroup) PatchSettings(metaGroup *rmpb.ResourceGroup) error { rg.Lock() defer rg.Unlock() + if metaGroup.GetMode() != rg.Mode { return errors.New("only support reconfigure in same mode, maybe you should delete and create a new one") } + if metaGroup.GetPriority() > 16 { + return errors.New("invalid resource group priority, the value should be in [0,16]") + } + rg.Priority = metaGroup.Priority + rg.Runaway = metaGroup.RunawaySettings + rg.Background = metaGroup.BackgroundSettings switch rg.Mode { case rmpb.GroupMode_RUMode: - if metaGroup.GetRUSettings() == nil { + settings := metaGroup.GetRUSettings() + if settings == nil { return errors.New("invalid resource group settings, RU mode should set RU settings") } - rg.RUSettings.RU.patch(metaGroup.GetRUSettings().GetRU()) + rg.RUSettings.RU.patch(settings.GetRU()) case rmpb.GroupMode_RawMode: - if metaGroup.GetRawResourceSettings() == nil { - return errors.New("invalid resource group settings, raw mode should set resource settings") - } - rg.RawResourceSettings.CPU.patch(metaGroup.GetRawResourceSettings().GetCpu()) - rg.RawResourceSettings.IOReadBandwidth.patch(metaGroup.GetRawResourceSettings().GetIoRead()) - rg.RawResourceSettings.IOWriteBandwidth.patch(metaGroup.GetRawResourceSettings().GetIoWrite()) + panic("no implementation") } log.Info("patch resource group settings", zap.String("name", rg.Name), zap.String("settings", rg.String())) return nil @@ -149,22 +116,21 @@ func (rg *ResourceGroup) PatchSettings(metaGroup *rmpb.ResourceGroup) error { // FromProtoResourceGroup converts a rmpb.ResourceGroup to a ResourceGroup. func FromProtoResourceGroup(group *rmpb.ResourceGroup) *ResourceGroup { rg := &ResourceGroup{ - Name: group.Name, - Mode: group.Mode, + Name: group.Name, + Mode: group.Mode, + Priority: group.Priority, + Runaway: group.RunawaySettings, + Background: group.BackgroundSettings, } switch group.GetMode() { case rmpb.GroupMode_RUMode: - if settings := group.GetRUSettings(); settings != nil { - rg.RUSettings = NewRequestUnitSettings(settings.GetRU()) + if group.GetRUSettings() == nil { + rg.RUSettings = NewRequestUnitSettings(nil) + } else { + rg.RUSettings = NewRequestUnitSettings(group.GetRUSettings().GetRU()) } case rmpb.GroupMode_RawMode: - if settings := group.GetRawResourceSettings(); settings != nil { - rg.RawResourceSettings = NewRawResourceSettings( - settings.GetCpu(), - settings.GetIoRead(), - settings.GetIoWrite(), - ) - } + panic("no implementation") } return rg } @@ -173,14 +139,15 @@ func FromProtoResourceGroup(group *rmpb.ResourceGroup) *ResourceGroup { func (rg *ResourceGroup) RequestRU( now time.Time, neededTokens float64, - targetPeriodMs uint64, + targetPeriodMs, clientUniqueID uint64, ) *rmpb.GrantedRUTokenBucket { rg.Lock() defer rg.Unlock() + if rg.RUSettings == nil || rg.RUSettings.RU.Settings == nil { return nil } - tb, trickleTimeMs := rg.RUSettings.RU.request(now, neededTokens, targetPeriodMs) + tb, trickleTimeMs := rg.RUSettings.RU.request(now, neededTokens, targetPeriodMs, clientUniqueID) return &rmpb.GrantedRUTokenBucket{GrantedTokens: tb, TrickleTimeMs: trickleTimeMs} } @@ -188,27 +155,22 @@ func (rg *ResourceGroup) RequestRU( func (rg *ResourceGroup) IntoProtoResourceGroup() *rmpb.ResourceGroup { rg.RLock() defer rg.RUnlock() + switch rg.Mode { case rmpb.GroupMode_RUMode: // RU mode group := &rmpb.ResourceGroup{ - Name: rg.Name, - Mode: rmpb.GroupMode_RUMode, + Name: rg.Name, + Mode: rmpb.GroupMode_RUMode, + Priority: rg.Priority, RUSettings: &rmpb.GroupRequestUnitSettings{ RU: rg.RUSettings.RU.GetTokenBucket(), }, + RunawaySettings: rg.Runaway, + BackgroundSettings: rg.Background, } return group case rmpb.GroupMode_RawMode: // Raw mode - group := &rmpb.ResourceGroup{ - Name: rg.Name, - Mode: rmpb.GroupMode_RawMode, - RawResourceSettings: &rmpb.GroupRawResourceSettings{ - Cpu: rg.RawResourceSettings.CPU.GetTokenBucket(), - IoRead: rg.RawResourceSettings.IOReadBandwidth.GetTokenBucket(), - IoWrite: rg.RawResourceSettings.IOWriteBandwidth.GetTokenBucket(), - }, - } - return group + panic("no implementation") } return nil } @@ -234,6 +196,7 @@ type GroupStates struct { func (rg *ResourceGroup) GetGroupStates() *GroupStates { rg.RLock() defer rg.RUnlock() + switch rg.Mode { case rmpb.GroupMode_RUMode: // RU mode tokens := &GroupStates{ @@ -241,12 +204,7 @@ func (rg *ResourceGroup) GetGroupStates() *GroupStates { } return tokens case rmpb.GroupMode_RawMode: // Raw mode - tokens := &GroupStates{ - CPU: rg.RawResourceSettings.CPU.GroupTokenBucketState.Clone(), - IORead: rg.RawResourceSettings.IOReadBandwidth.GroupTokenBucketState.Clone(), - IOWrite: rg.RawResourceSettings.IOWriteBandwidth.GroupTokenBucketState.Clone(), - } - return tokens + panic("no implementation") } return nil } @@ -256,18 +214,11 @@ func (rg *ResourceGroup) SetStatesIntoResourceGroup(states *GroupStates) { switch rg.Mode { case rmpb.GroupMode_RUMode: if state := states.RU; state != nil { - rg.RUSettings.RU.GroupTokenBucketState = *state + rg.RUSettings.RU.setState(state) + log.Debug("update group token bucket state", zap.String("name", rg.Name), zap.Any("state", state)) } case rmpb.GroupMode_RawMode: - if state := states.CPU; state != nil { - rg.RawResourceSettings.CPU.GroupTokenBucketState = *state - } - if state := states.IORead; state != nil { - rg.RawResourceSettings.IOReadBandwidth.GroupTokenBucketState = *state - } - if state := states.IOWrite; state != nil { - rg.RawResourceSettings.IOWriteBandwidth.GroupTokenBucketState = *state - } + panic("no implementation") } } diff --git a/pkg/mcs/resourcemanager/server/resource_group_test.go b/pkg/mcs/resourcemanager/server/resource_group_test.go new file mode 100644 index 00000000000..ee775ee989d --- /dev/null +++ b/pkg/mcs/resourcemanager/server/resource_group_test.go @@ -0,0 +1,36 @@ +package server + +import ( + "encoding/json" + "testing" + + rmpb "github.com/pingcap/kvproto/pkg/resource_manager" + "github.com/stretchr/testify/require" +) + +func TestPatchResourceGroup(t *testing.T) { + re := require.New(t) + rg := &ResourceGroup{Name: "test", Mode: rmpb.GroupMode_RUMode, RUSettings: NewRequestUnitSettings(nil)} + testCaseRU := []struct { + patchJSONString string + expectJSONString string + }{ + {`{"name":"test", "mode":1, "r_u_settings": {"r_u":{"settings":{"fill_rate": 200000}}}}`, + `{"name":"test","mode":1,"r_u_settings":{"r_u":{"settings":{"fill_rate":200000},"state":{"initialized":false}}},"priority":0}`}, + {`{"name":"test", "mode":1, "r_u_settings": {"r_u":{"settings":{"fill_rate": 200000, "burst_limit": -1}}}}`, + `{"name":"test","mode":1,"r_u_settings":{"r_u":{"settings":{"fill_rate":200000,"burst_limit":-1},"state":{"initialized":false}}},"priority":0}`}, + {`{"name":"test", "mode":1, "r_u_settings": {"r_u":{"settings":{"fill_rate": 200000, "burst_limit": -1}}}, "priority": 8, "runaway_settings": {"rule":{"exec_elapsed_time_ms":10000}, "action":1} }`, + `{"name":"test","mode":1,"r_u_settings":{"r_u":{"settings":{"fill_rate":200000,"burst_limit":-1},"state":{"initialized":false}}},"priority":8,"runaway_settings":{"rule":{"exec_elapsed_time_ms":10000},"action":1}}`}, + } + + for _, ca := range testCaseRU { + patch := &rmpb.ResourceGroup{} + err := json.Unmarshal([]byte(ca.patchJSONString), patch) + re.NoError(err) + err = rg.PatchSettings(patch) + re.NoError(err) + res, err := json.Marshal(rg.Copy()) + re.NoError(err) + re.Equal(ca.expectJSONString, string(res)) + } +} diff --git a/pkg/mcs/resource_manager/server/server.go b/pkg/mcs/resourcemanager/server/server.go similarity index 77% rename from pkg/mcs/resource_manager/server/server.go rename to pkg/mcs/resourcemanager/server/server.go index a53c23c54ab..645e118e1c9 100644 --- a/pkg/mcs/resource_manager/server/server.go +++ b/pkg/mcs/resourcemanager/server/server.go @@ -24,6 +24,7 @@ import ( "os" "os/signal" "path" + "strconv" "strings" "sync" "sync/atomic" @@ -31,14 +32,16 @@ import ( "time" grpcprometheus "github.com/grpc-ecosystem/go-grpc-prometheus" + "github.com/pingcap/kvproto/pkg/diagnosticspb" "github.com/pingcap/log" + "github.com/pingcap/sysutil" "github.com/soheilhy/cmux" "github.com/spf13/cobra" - bs "github.com/tikv/pd/pkg/basicserver" "github.com/tikv/pd/pkg/errs" "github.com/tikv/pd/pkg/mcs/discovery" "github.com/tikv/pd/pkg/mcs/utils" "github.com/tikv/pd/pkg/member" + "github.com/tikv/pd/pkg/storage/endpoint" "github.com/tikv/pd/pkg/utils/etcdutil" "github.com/tikv/pd/pkg/utils/logutil" "github.com/tikv/pd/pkg/utils/memberutil" @@ -50,17 +53,13 @@ import ( "google.golang.org/grpc" ) -const ( - // resourceManagerPrimaryPrefix defines the key prefix for keyspace group primary election. - // The entire key is in the format of "/ms//resource-manager//primary" - // in which is 5 digits integer with leading zeros. For now we use 0 as the default cluster id. - resourceManagerPrimaryPrefix = "/ms/0/resource-manager" -) - // Server is the resource manager server, and it implements bs.Server. type Server struct { - // Server state. 0 is not serving, 1 is serving. - isServing int64 + diagnosticspb.DiagnosticsServer + // Server state. 0 is not running, 1 is running. + isRunning int64 + // Server start timestamp + startTimestamp int64 ctx context.Context serverLoopCtx context.Context @@ -99,6 +98,11 @@ func (s *Server) Context() context.Context { return s.ctx } +// GetAddr returns the server address. +func (s *Server) GetAddr() string { + return s.cfg.ListenAddr +} + // Run runs the Resource Manager server. func (s *Server) Run() (err error) { if err = s.initClient(); err != nil { @@ -129,15 +133,15 @@ func (s *Server) primaryElectionLoop() { return } - primary, rev, checkAgain := s.participant.CheckLeader() + primary, checkAgain := s.participant.CheckLeader() if checkAgain { continue } if primary != nil { - log.Info("start to watch the primary/leader", zap.Stringer("resource-manager-primary", primary)) - // WatchLeader will keep looping and never return unless the primary/leader has changed. - s.participant.WatchLeader(s.serverLoopCtx, primary, rev) - log.Info("the resource manager primary/leader has changed, try to re-campaign a primary/leader") + log.Info("start to watch the primary", zap.Stringer("resource-manager-primary", primary)) + // Watch will keep looping and never return unless the primary/leader has changed. + primary.Watch(s.serverLoopCtx) + log.Info("the resource manager primary has changed, try to re-campaign a primary") } s.campaignLeader() @@ -145,14 +149,14 @@ func (s *Server) primaryElectionLoop() { } func (s *Server) campaignLeader() { - log.Info("start to campaign the primary/leader", zap.String("campaign-resource-manager-primary-name", s.participant.Member().Name)) + log.Info("start to campaign the primary/leader", zap.String("campaign-resource-manager-primary-name", s.participant.Name())) if err := s.participant.CampaignLeader(s.cfg.LeaderLease); err != nil { if err.Error() == errs.ErrEtcdTxnConflict.Error() { - log.Info("campaign resource manager primary/leader meets error due to txn conflict, another resource manager server may campaign successfully", - zap.String("campaign-resource-manager-primary-name", s.participant.Member().Name)) + log.Info("campaign resource manager primary meets error due to txn conflict, another server may campaign successfully", + zap.String("campaign-resource-manager-primary-name", s.participant.Name())) } else { - log.Error("campaign resource manager primary/leader meets error due to etcd error", - zap.String("campaign-resource-manager-primary-name", s.participant.Member().Name), + log.Error("campaign resource manager primary meets error due to etcd error", + zap.String("campaign-resource-manager-primary-name", s.participant.Name()), errs.ZapError(err)) } return @@ -168,7 +172,7 @@ func (s *Server) campaignLeader() { // maintain the leadership, after this, Resource Manager could be ready to provide service. s.participant.KeepLeader(ctx) - log.Info("campaign resource manager primary ok", zap.String("campaign-resource-manager-primary-name", s.participant.Member().Name)) + log.Info("campaign resource manager primary ok", zap.String("campaign-resource-manager-primary-name", s.participant.Name())) log.Info("triggering the primary callback functions") for _, cb := range s.primaryCallbacks { @@ -176,7 +180,7 @@ func (s *Server) campaignLeader() { } s.participant.EnableLeader() - log.Info("resource manager primary is ready to serve", zap.String("resource-manager-primary-name", s.participant.Member().Name)) + log.Info("resource manager primary is ready to serve", zap.String("resource-manager-primary-name", s.participant.Name())) leaderTicker := time.NewTicker(utils.LeaderTickInterval) defer leaderTicker.Stop() @@ -198,7 +202,7 @@ func (s *Server) campaignLeader() { // Close closes the server. func (s *Server) Close() { - if !atomic.CompareAndSwapInt64(&s.isServing, 1, 0) { + if !atomic.CompareAndSwapInt64(&s.isRunning, 1, 0) { // server is already closed return } @@ -222,9 +226,9 @@ func (s *Server) Close() { log.Info("resource manager server is closed") } -// GetRequestUnitConfig returns the RU config. -func (s *Server) GetRequestUnitConfig() *RequestUnitConfig { - return &s.cfg.RequestUnit +// GetControllerConfig returns the controller config. +func (s *Server) GetControllerConfig() *ControllerConfig { + return &s.cfg.Controller } // GetClient returns builtin etcd client. @@ -244,12 +248,12 @@ func (s *Server) AddStartCallback(callbacks ...func()) { // IsServing returns whether the server is the leader, if there is embedded etcd, or the primary otherwise. func (s *Server) IsServing() bool { - return s.participant.IsLeader() + return !s.IsClosed() && s.participant.IsLeader() } // IsClosed checks if the server loop is closed func (s *Server) IsClosed() bool { - return atomic.LoadInt64(&s.isServing) == 0 + return s != nil && atomic.LoadInt64(&s.isRunning) == 0 } // AddServiceReadyCallback adds callbacks when the server becomes the leader, if there is embedded etcd, or the primary otherwise. @@ -271,6 +275,7 @@ func (s *Server) initClient() error { } func (s *Server) startGRPCServer(l net.Listener) { + defer logutil.LogPanic() defer s.serverLoopWg.Done() gs := grpc.NewServer() @@ -282,13 +287,16 @@ func (s *Server) startGRPCServer(l net.Listener) { // it doesn't happen in a reasonable amount of time. done := make(chan struct{}) go func() { + defer logutil.LogPanic() log.Info("try to gracefully stop the server now") gs.GracefulStop() close(done) }() + timer := time.NewTimer(utils.DefaultGRPCGracefulStopTimeout) + defer timer.Stop() select { case <-done: - case <-time.After(utils.DefaultGRPCGracefulStopTimeout): + case <-timer.C: log.Info("stopping grpc gracefully is taking longer than expected and force stopping now", zap.Duration("default", utils.DefaultGRPCGracefulStopTimeout)) gs.Stop() } @@ -300,6 +308,7 @@ func (s *Server) startGRPCServer(l net.Listener) { } func (s *Server) startHTTPServer(l net.Listener) { + defer logutil.LogPanic() defer s.serverLoopWg.Done() handler, _ := SetUpRestHandler(s.service) @@ -326,6 +335,7 @@ func (s *Server) startHTTPServer(l net.Listener) { } func (s *Server) startGRPCAndHTTPServers(l net.Listener) { + defer logutil.LogPanic() defer s.serverLoopWg.Done() mux := cmux.New(l) @@ -345,13 +355,13 @@ func (s *Server) startGRPCAndHTTPServers(l net.Listener) { } } -// GetPrimary returns the primary member. -func (s *Server) GetPrimary() bs.MemberProvider { - return s.participant.GetLeader() +// GetLeaderListenUrls gets service endpoints from the leader in election group. +func (s *Server) GetLeaderListenUrls() []string { + return s.participant.GetLeaderListenUrls() } func (s *Server) startServer() (err error) { - if s.clusterID, err = etcdutil.GetClusterID(s.etcdClient, utils.ClusterIDPath); err != nil { + if s.clusterID, err = utils.InitClusterID(s.ctx, s.etcdClient); err != nil { return err } log.Info("init cluster id", zap.Uint64("cluster-id", s.clusterID)) @@ -362,11 +372,10 @@ func (s *Server) startServer() (err error) { uniqueName := s.cfg.ListenAddr uniqueID := memberutil.GenerateUniqueID(uniqueName) log.Info("joining primary election", zap.String("participant-name", uniqueName), zap.Uint64("participant-id", uniqueID)) - s.participant = member.NewParticipant(s.etcdClient, uniqueID) - s.participant.InitInfo(uniqueName, path.Join(resourceManagerPrimaryPrefix, fmt.Sprintf("%05d", 0)), "primary", "keyspace group primary election", s.cfg.ListenAddr) - s.participant.SetMemberDeployPath(s.participant.ID()) - s.participant.SetMemberBinaryVersion(s.participant.ID(), versioninfo.PDReleaseVersion) - s.participant.SetMemberGitHash(s.participant.ID(), versioninfo.PDGitHash) + resourceManagerPrimaryPrefix := endpoint.ResourceManagerSvcRootPath(s.clusterID) + s.participant = member.NewParticipant(s.etcdClient) + s.participant.InitInfo(uniqueName, uniqueID, path.Join(resourceManagerPrimaryPrefix, fmt.Sprintf("%05d", 0)), + utils.PrimaryKey, "primary election", s.cfg.AdvertiseListenAddr) s.service = &Service{ ctx: s.ctx, @@ -400,19 +409,30 @@ func (s *Server) startServer() (err error) { } // Server has started. - atomic.StoreInt64(&s.isServing, 1) - s.serviceRegister = discovery.NewServiceRegister(s.ctx, s.etcdClient, "resource_manager", s.cfg.ListenAddr, s.cfg.ListenAddr, discovery.DefaultLeaseInSeconds) - s.serviceRegister.Register() + entry := &discovery.ServiceRegistryEntry{ServiceAddr: s.cfg.AdvertiseListenAddr} + serializedEntry, err := entry.Serialize() + if err != nil { + return err + } + s.serviceRegister = discovery.NewServiceRegister(s.ctx, s.etcdClient, strconv.FormatUint(s.clusterID, 10), + utils.ResourceManagerServiceName, s.cfg.AdvertiseListenAddr, serializedEntry, discovery.DefaultLeaseInSeconds) + if err := s.serviceRegister.Register(); err != nil { + log.Error("failed to register the service", zap.String("service-name", utils.ResourceManagerServiceName), errs.ZapError(err)) + return err + } + atomic.StoreInt64(&s.isRunning, 1) return nil } -// NewServer creates a new resource manager server. -func NewServer(ctx context.Context, cfg *Config) *Server { - return &Server{ - name: cfg.Name, - ctx: ctx, - cfg: cfg, +// CreateServer creates the Server +func CreateServer(ctx context.Context, cfg *Config) *Server { + svr := &Server{ + DiagnosticsServer: sysutil.NewDiagnosticsServer(cfg.Log.File.Filename), + startTimestamp: time.Now().Unix(), + cfg: cfg, + ctx: ctx, } + return svr } // CreateServerWrapper encapsulates the configuration/log/metrics initialization and create the server @@ -446,15 +466,14 @@ func CreateServerWrapper(cmd *cobra.Command, args []string) { // Flushing any buffered log entries defer log.Sync() - versioninfo.Log("resource manager") - log.Info("resource manager config", zap.Reflect("config", cfg)) + versioninfo.Log("Resource Manager") + log.Info("Resource Manager config", zap.Reflect("config", cfg)) grpcprometheus.EnableHandlingTimeHistogram() - metricutil.Push(&cfg.Metric) ctx, cancel := context.WithCancel(context.Background()) - svr := NewServer(ctx, cfg) + svr := CreateServer(ctx, cfg) sc := make(chan os.Signal, 1) signal.Notify(sc, diff --git a/pkg/mcs/resourcemanager/server/testutil.go b/pkg/mcs/resourcemanager/server/testutil.go new file mode 100644 index 00000000000..3de0e32c0ab --- /dev/null +++ b/pkg/mcs/resourcemanager/server/testutil.go @@ -0,0 +1,77 @@ +// Copyright 2023 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "context" + "os" + + "github.com/pingcap/log" + "github.com/spf13/pflag" + "github.com/stretchr/testify/require" + "github.com/tikv/pd/pkg/utils/logutil" + "github.com/tikv/pd/pkg/utils/testutil" +) + +// NewTestServer creates a resource manager server for testing. +func NewTestServer(ctx context.Context, re *require.Assertions, cfg *Config) (*Server, testutil.CleanupFunc, error) { + // New zap logger + err := logutil.SetupLogger(cfg.Log, &cfg.Logger, &cfg.LogProps, cfg.Security.RedactInfoLog) + re.NoError(err) + log.ReplaceGlobals(cfg.Logger, cfg.LogProps) + // Flushing any buffered log entries + defer log.Sync() + + s := CreateServer(ctx, cfg) + if err = s.Run(); err != nil { + return nil, nil, err + } + + cleanup := func() { + s.Close() + os.RemoveAll(cfg.DataDir) + } + return s, cleanup, nil +} + +// GenerateConfig generates a new config with the given options. +func GenerateConfig(c *Config) (*Config, error) { + arguments := []string{ + "--listen-addr=" + c.ListenAddr, + "--advertise-listen-addr=" + c.AdvertiseListenAddr, + "--backend-endpoints=" + c.BackendEndpoints, + } + + flagSet := pflag.NewFlagSet("test", pflag.ContinueOnError) + flagSet.BoolP("version", "V", false, "print version information and exit") + flagSet.StringP("config", "", "", "config file") + flagSet.StringP("backend-endpoints", "", "", "url for etcd client") + flagSet.StringP("listen-addr", "", "", "listen address for tso service") + flagSet.StringP("advertise-listen-addr", "", "", "advertise urls for listen address (default '${listen-addr}')") + flagSet.StringP("cacert", "", "", "path of file that contains list of trusted TLS CAs") + flagSet.StringP("cert", "", "", "path of file that contains X509 certificate in PEM format") + flagSet.StringP("key", "", "", "path of file that contains X509 key in PEM format") + err := flagSet.Parse(arguments) + if err != nil { + return nil, err + } + cfg := NewConfig() + err = cfg.Parse(flagSet) + if err != nil { + return nil, err + } + + return cfg, nil +} diff --git a/pkg/mcs/resourcemanager/server/token_buckets.go b/pkg/mcs/resourcemanager/server/token_buckets.go new file mode 100644 index 00000000000..5efab52fe68 --- /dev/null +++ b/pkg/mcs/resourcemanager/server/token_buckets.go @@ -0,0 +1,420 @@ +// Copyright 2022 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS,g +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "math" + "time" + + "github.com/gogo/protobuf/proto" + rmpb "github.com/pingcap/kvproto/pkg/resource_manager" +) + +const ( + defaultRefillRate = 10000 + defaultInitialTokens = 10 * 10000 +) + +const ( + defaultReserveRatio = 0.5 + defaultLoanCoefficient = 2 + maxAssignTokens = math.MaxFloat64 / 1024 // assume max client connect is 1024 +) + +// GroupTokenBucket is a token bucket for a resource group. +// Now we don't save consumption in `GroupTokenBucket`, only statistics it in prometheus. +type GroupTokenBucket struct { + // Settings is the setting of TokenBucket. + // BurstLimit is used as below: + // - If b == 0, that means the limiter is unlimited capacity. default use in resource controller (burst with a rate within an unlimited capacity). + // - If b < 0, that means the limiter is unlimited capacity and fillrate(r) is ignored, can be seen as r == Inf (burst within an unlimited capacity). + // - If b > 0, that means the limiter is limited capacity. + // MaxTokens limits the number of tokens that can be accumulated + Settings *rmpb.TokenLimitSettings `json:"settings,omitempty"` + GroupTokenBucketState `json:"state,omitempty"` +} + +func (gtb *GroupTokenBucket) setState(state *GroupTokenBucketState) { + gtb.Tokens = state.Tokens + gtb.LastUpdate = state.LastUpdate + gtb.Initialized = state.Initialized +} + +// TokenSlot is used to split a token bucket into multiple slots to +// server different clients within the same resource group. +type TokenSlot struct { + // settings is the token limit settings for the slot. + settings *rmpb.TokenLimitSettings + // requireTokensSum is the number of tokens required. + requireTokensSum float64 + // tokenCapacity is the number of tokens in the slot. + tokenCapacity float64 + lastTokenCapacity float64 +} + +// GroupTokenBucketState is the running state of TokenBucket. +type GroupTokenBucketState struct { + Tokens float64 `json:"tokens,omitempty"` + // ClientUniqueID -> TokenSlot + tokenSlots map[uint64]*TokenSlot + clientConsumptionTokensSum float64 + lastBurstTokens float64 + + LastUpdate *time.Time `json:"last_update,omitempty"` + Initialized bool `json:"initialized"` + // settingChanged is used to avoid that the number of tokens returned is jitter because of changing fill rate. + settingChanged bool +} + +// Clone returns the copy of GroupTokenBucketState +func (gts *GroupTokenBucketState) Clone() *GroupTokenBucketState { + tokenSlots := make(map[uint64]*TokenSlot) + for id, tokens := range gts.tokenSlots { + tokenSlots[id] = tokens + } + var lastUpdate *time.Time + if gts.LastUpdate != nil { + newLastUpdate := *gts.LastUpdate + lastUpdate = &newLastUpdate + } + return &GroupTokenBucketState{ + Tokens: gts.Tokens, + LastUpdate: lastUpdate, + Initialized: gts.Initialized, + tokenSlots: tokenSlots, + clientConsumptionTokensSum: gts.clientConsumptionTokensSum, + } +} + +func (gts *GroupTokenBucketState) resetLoan() { + gts.settingChanged = false + gts.Tokens = 0 + gts.clientConsumptionTokensSum = 0 + evenRatio := 1.0 + if l := len(gts.tokenSlots); l > 0 { + evenRatio = 1 / float64(l) + } + + evenTokens := gts.Tokens * evenRatio + for _, slot := range gts.tokenSlots { + slot.requireTokensSum = 0 + slot.tokenCapacity = evenTokens + slot.lastTokenCapacity = evenTokens + } +} + +func (gts *GroupTokenBucketState) balanceSlotTokens( + clientUniqueID uint64, + settings *rmpb.TokenLimitSettings, + requiredToken, elapseTokens float64) { + slot, exist := gts.tokenSlots[clientUniqueID] + if !exist { + // Only slots that require a positive number will be considered alive, + // but still need to allocate the elapsed tokens as well. + if requiredToken != 0 { + slot = &TokenSlot{} + gts.tokenSlots[clientUniqueID] = slot + gts.clientConsumptionTokensSum = 0 + } + } else { + if gts.clientConsumptionTokensSum >= maxAssignTokens { + gts.clientConsumptionTokensSum = 0 + } + // Clean up slot that required 0. + if requiredToken == 0 { + delete(gts.tokenSlots, clientUniqueID) + gts.clientConsumptionTokensSum = 0 + } + } + + if len(gts.tokenSlots) == 0 { + return + } + evenRatio := 1 / float64(len(gts.tokenSlots)) + if settings.GetBurstLimit() <= 0 { + for _, slot := range gts.tokenSlots { + slot.settings = &rmpb.TokenLimitSettings{ + FillRate: uint64(float64(settings.GetFillRate()) * evenRatio), + BurstLimit: settings.GetBurstLimit(), + } + } + return + } + + for _, slot := range gts.tokenSlots { + if gts.clientConsumptionTokensSum == 0 || len(gts.tokenSlots) == 1 { + // Need to make each slot even. + slot.tokenCapacity = evenRatio * gts.Tokens + slot.lastTokenCapacity = evenRatio * gts.Tokens + slot.requireTokensSum = 0 + gts.clientConsumptionTokensSum = 0 + + var ( + fillRate = float64(settings.GetFillRate()) * evenRatio + burstLimit = float64(settings.GetBurstLimit()) * evenRatio + ) + + slot.settings = &rmpb.TokenLimitSettings{ + FillRate: uint64(fillRate), + BurstLimit: int64(burstLimit), + } + } else { + // In order to have fewer tokens available to clients that are currently consuming more. + // We have the following formula: + // client1: (1 - a/N + 1/N) * 1/N + // client2: (1 - b/N + 1/N) * 1/N + // ... + // clientN: (1 - n/N + 1/N) * 1/N + // Sum is: + // (N - (a+b+...+n)/N +1) * 1/N => (N - 1 + 1) * 1/N => 1 + ratio := (1 - slot.requireTokensSum/gts.clientConsumptionTokensSum + evenRatio) * evenRatio + + var ( + fillRate = float64(settings.GetFillRate()) * ratio + burstLimit = float64(settings.GetBurstLimit()) * ratio + assignToken = elapseTokens * ratio + ) + + // Need to reserve burst limit to next balance. + if burstLimit > 0 && slot.tokenCapacity > burstLimit { + reservedTokens := slot.tokenCapacity - burstLimit + gts.lastBurstTokens += reservedTokens + gts.Tokens -= reservedTokens + assignToken -= reservedTokens + } + + slot.tokenCapacity += assignToken + slot.lastTokenCapacity += assignToken + slot.settings = &rmpb.TokenLimitSettings{ + FillRate: uint64(fillRate), + BurstLimit: int64(burstLimit), + } + } + } + if requiredToken != 0 { + // Only slots that require a positive number will be considered alive. + slot.requireTokensSum += requiredToken + gts.clientConsumptionTokensSum += requiredToken + } +} + +// NewGroupTokenBucket returns a new GroupTokenBucket +func NewGroupTokenBucket(tokenBucket *rmpb.TokenBucket) *GroupTokenBucket { + if tokenBucket == nil || tokenBucket.Settings == nil { + return &GroupTokenBucket{} + } + return &GroupTokenBucket{ + Settings: tokenBucket.GetSettings(), + GroupTokenBucketState: GroupTokenBucketState{ + Tokens: tokenBucket.GetTokens(), + tokenSlots: make(map[uint64]*TokenSlot), + }, + } +} + +// GetTokenBucket returns the grpc protoc struct of GroupTokenBucket. +func (gtb *GroupTokenBucket) GetTokenBucket() *rmpb.TokenBucket { + if gtb.Settings == nil { + return nil + } + return &rmpb.TokenBucket{ + Settings: gtb.Settings, + Tokens: gtb.Tokens, + } +} + +// patch patches the token bucket settings. +func (gtb *GroupTokenBucket) patch(tb *rmpb.TokenBucket) { + if tb == nil { + return + } + if setting := proto.Clone(tb.GetSettings()).(*rmpb.TokenLimitSettings); setting != nil { + gtb.Settings = setting + gtb.settingChanged = true + } + + // The settings in token is delta of the last update and now. + gtb.Tokens += tb.GetTokens() +} + +// init initializes the group token bucket. +func (gtb *GroupTokenBucket) init(now time.Time, clientID uint64) { + if gtb.Settings.FillRate == 0 { + gtb.Settings.FillRate = defaultRefillRate + } + if gtb.Tokens < defaultInitialTokens { + gtb.Tokens = defaultInitialTokens + } + // init slot + gtb.tokenSlots[clientID] = &TokenSlot{ + settings: gtb.Settings, + tokenCapacity: gtb.Tokens, + lastTokenCapacity: gtb.Tokens, + } + gtb.LastUpdate = &now + gtb.Initialized = true +} + +// updateTokens updates the tokens and settings. +func (gtb *GroupTokenBucket) updateTokens(now time.Time, burstLimit int64, clientUniqueID uint64, consumptionToken float64) { + var elapseTokens float64 + if !gtb.Initialized { + gtb.init(now, clientUniqueID) + } else if delta := now.Sub(*gtb.LastUpdate); delta > 0 { + elapseTokens = float64(gtb.Settings.GetFillRate())*delta.Seconds() + gtb.lastBurstTokens + gtb.lastBurstTokens = 0 + gtb.Tokens += elapseTokens + gtb.LastUpdate = &now + } + // Reloan when setting changed + if gtb.settingChanged && gtb.Tokens <= 0 { + elapseTokens = 0 + gtb.resetLoan() + } + if burst := float64(burstLimit); burst > 0 && gtb.Tokens > burst { + elapseTokens -= gtb.Tokens - burst + gtb.Tokens = burst + } + // Balance each slots. + gtb.balanceSlotTokens(clientUniqueID, gtb.Settings, consumptionToken, elapseTokens) +} + +// request requests tokens from the corresponding slot. +func (gtb *GroupTokenBucket) request(now time.Time, + neededTokens float64, + targetPeriodMs, clientUniqueID uint64, +) (*rmpb.TokenBucket, int64) { + burstLimit := gtb.Settings.GetBurstLimit() + gtb.updateTokens(now, burstLimit, clientUniqueID, neededTokens) + slot, ok := gtb.tokenSlots[clientUniqueID] + if !ok { + return &rmpb.TokenBucket{Settings: &rmpb.TokenLimitSettings{BurstLimit: burstLimit}}, 0 + } + res, trickleDuration := slot.assignSlotTokens(neededTokens, targetPeriodMs) + // Update bucket to record all tokens. + gtb.Tokens -= slot.lastTokenCapacity - slot.tokenCapacity + slot.lastTokenCapacity = slot.tokenCapacity + + return res, trickleDuration +} + +func (ts *TokenSlot) assignSlotTokens(neededTokens float64, targetPeriodMs uint64) (*rmpb.TokenBucket, int64) { + var res rmpb.TokenBucket + burstLimit := ts.settings.GetBurstLimit() + res.Settings = &rmpb.TokenLimitSettings{BurstLimit: burstLimit} + // If BurstLimit < 0, just return. + if burstLimit < 0 { + res.Tokens = neededTokens + return &res, 0 + } + // FillRate is used for the token server unavailable in abnormal situation. + if neededTokens <= 0 { + return &res, 0 + } + // If the current tokens can directly meet the requirement, returns the need token. + if ts.tokenCapacity >= neededTokens { + ts.tokenCapacity -= neededTokens + // granted the total request tokens + res.Tokens = neededTokens + return &res, 0 + } + + // Firstly allocate the remaining tokens + var grantedTokens float64 + hasRemaining := false + if ts.tokenCapacity > 0 { + grantedTokens = ts.tokenCapacity + neededTokens -= grantedTokens + ts.tokenCapacity = 0 + hasRemaining = true + } + + var ( + targetPeriodTime = time.Duration(targetPeriodMs) * time.Millisecond + targetPeriodTimeSec = targetPeriodTime.Seconds() + trickleTime = 0. + fillRate = ts.settings.GetFillRate() + ) + + loanCoefficient := defaultLoanCoefficient + // When BurstLimit less or equal FillRate, the server does not accumulate a significant number of tokens. + // So we don't need to smooth the token allocation speed. + if burstLimit > 0 && burstLimit <= int64(fillRate) { + loanCoefficient = 1 + } + // When there are loan, the allotment will match the fill rate. + // We will have k threshold, beyond which the token allocation will be a minimum. + // The threshold unit is `fill rate * target period`. + // | + // k*fill_rate |* * * * * * * + // | * + // *** | * + // | * + // | * + // fill_rate | * + // reserve_rate | * + // | + // grant_rate 0 ------------------------------------------------------------------------------------ + // loan *** k*period_token (k+k-1)*period_token *** (k+k+1...+1)*period_token + p := make([]float64, loanCoefficient) + p[0] = float64(loanCoefficient) * float64(fillRate) * targetPeriodTimeSec + for i := 1; i < loanCoefficient; i++ { + p[i] = float64(loanCoefficient-i)*float64(fillRate)*targetPeriodTimeSec + p[i-1] + } + for i := 0; i < loanCoefficient && neededTokens > 0 && trickleTime < targetPeriodTimeSec; i++ { + loan := -ts.tokenCapacity + if loan >= p[i] { + continue + } + roundReserveTokens := p[i] - loan + fillRate := float64(loanCoefficient-i) * float64(fillRate) + if roundReserveTokens > neededTokens { + ts.tokenCapacity -= neededTokens + grantedTokens += neededTokens + trickleTime += grantedTokens / fillRate + neededTokens = 0 + } else { + roundReserveTime := roundReserveTokens / fillRate + if roundReserveTime+trickleTime >= targetPeriodTimeSec { + roundTokens := (targetPeriodTimeSec - trickleTime) * fillRate + neededTokens -= roundTokens + ts.tokenCapacity -= roundTokens + grantedTokens += roundTokens + trickleTime = targetPeriodTimeSec + } else { + grantedTokens += roundReserveTokens + neededTokens -= roundReserveTokens + ts.tokenCapacity -= roundReserveTokens + trickleTime += roundReserveTime + } + } + } + if neededTokens > 0 && grantedTokens < defaultReserveRatio*float64(fillRate)*targetPeriodTimeSec { + reservedTokens := math.Min(neededTokens+grantedTokens, defaultReserveRatio*float64(fillRate)*targetPeriodTimeSec) + ts.tokenCapacity -= reservedTokens - grantedTokens + grantedTokens = reservedTokens + } + res.Tokens = grantedTokens + + var trickleDuration time.Duration + // Can't directly treat targetPeriodTime as trickleTime when there is a token remaining. + // If treated, client consumption will be slowed down (actually could be increased). + if hasRemaining { + trickleDuration = time.Duration(math.Min(trickleTime, targetPeriodTime.Seconds()) * float64(time.Second)) + } else { + trickleDuration = targetPeriodTime + } + return &res, trickleDuration.Milliseconds() +} diff --git a/pkg/mcs/resource_manager/server/token_buckets_test.go b/pkg/mcs/resourcemanager/server/token_buckets_test.go similarity index 78% rename from pkg/mcs/resource_manager/server/token_buckets_test.go rename to pkg/mcs/resourcemanager/server/token_buckets_test.go index e064a6504fe..a7d3b9e3bad 100644 --- a/pkg/mcs/resource_manager/server/token_buckets_test.go +++ b/pkg/mcs/resourcemanager/server/token_buckets_test.go @@ -33,9 +33,10 @@ func TestGroupTokenBucketUpdateAndPatch(t *testing.T) { }, } + clientUniqueID := uint64(0) tb := NewGroupTokenBucket(tbSetting) time1 := time.Now() - tb.request(time1, 0, 0) + tb.request(time1, 0, 0, clientUniqueID) re.LessOrEqual(math.Abs(tbSetting.Tokens-tb.Tokens), 1e-7) re.Equal(tbSetting.Settings.FillRate, tb.Settings.FillRate) @@ -49,7 +50,7 @@ func TestGroupTokenBucketUpdateAndPatch(t *testing.T) { tb.patch(tbSetting) time2 := time.Now() - tb.request(time2, 0, 0) + tb.request(time2, 0, 0, clientUniqueID) re.LessOrEqual(math.Abs(100000-tb.Tokens), time2.Sub(time1).Seconds()*float64(tbSetting.Settings.FillRate)+1e7) re.Equal(tbSetting.Settings.FillRate, tb.Settings.FillRate) } @@ -66,27 +67,30 @@ func TestGroupTokenBucketRequest(t *testing.T) { gtb := NewGroupTokenBucket(tbSetting) time1 := time.Now() - tb, trickle := gtb.request(time1, 190000, uint64(time.Second)*10/uint64(time.Millisecond)) + clientUniqueID := uint64(0) + tb, trickle := gtb.request(time1, 190000, uint64(time.Second)*10/uint64(time.Millisecond), clientUniqueID) re.LessOrEqual(math.Abs(tb.Tokens-190000), 1e-7) re.Equal(trickle, int64(0)) // need to lend token - tb, trickle = gtb.request(time1, 11000, uint64(time.Second)*10/uint64(time.Millisecond)) + tb, trickle = gtb.request(time1, 11000, uint64(time.Second)*10/uint64(time.Millisecond), clientUniqueID) re.LessOrEqual(math.Abs(tb.Tokens-11000), 1e-7) re.Equal(trickle, int64(time.Second)*11000./4000./int64(time.Millisecond)) - tb, trickle = gtb.request(time1, 35000, uint64(time.Second)*10/uint64(time.Millisecond)) + tb, trickle = gtb.request(time1, 35000, uint64(time.Second)*10/uint64(time.Millisecond), clientUniqueID) re.LessOrEqual(math.Abs(tb.Tokens-35000), 1e-7) re.Equal(trickle, int64(time.Second)*10/int64(time.Millisecond)) - tb, trickle = gtb.request(time1, 60000, uint64(time.Second)*10/uint64(time.Millisecond)) + tb, trickle = gtb.request(time1, 60000, uint64(time.Second)*10/uint64(time.Millisecond), clientUniqueID) re.LessOrEqual(math.Abs(tb.Tokens-22000), 1e-7) re.Equal(trickle, int64(time.Second)*10/int64(time.Millisecond)) - tb, trickle = gtb.request(time1, 3000, uint64(time.Second)*10/uint64(time.Millisecond)) - re.LessOrEqual(math.Abs(tb.Tokens-2000), 1e-7) + // Get reserved 10000 tokens = fillrate(2000) * 10 * defaultReserveRatio(0.5) + // Max loan tokens is 60000. + tb, trickle = gtb.request(time1, 3000, uint64(time.Second)*10/uint64(time.Millisecond), clientUniqueID) + re.LessOrEqual(math.Abs(tb.Tokens-3000), 1e-7) re.Equal(trickle, int64(time.Second)*10/int64(time.Millisecond)) - tb, trickle = gtb.request(time1, 3000, uint64(time.Second)*10/uint64(time.Millisecond)) - re.LessOrEqual(math.Abs(tb.Tokens-1000), 1e-7) + tb, trickle = gtb.request(time1, 12000, uint64(time.Second)*10/uint64(time.Millisecond), clientUniqueID) + re.LessOrEqual(math.Abs(tb.Tokens-10000), 1e-7) re.Equal(trickle, int64(time.Second)*10/int64(time.Millisecond)) - time2 := time1.Add(10 * time.Second) - tb, trickle = gtb.request(time2, 20000, uint64(time.Second)*10/uint64(time.Millisecond)) - re.LessOrEqual(math.Abs(tb.Tokens-19000), 1e-7) + time2 := time1.Add(20 * time.Second) + tb, trickle = gtb.request(time2, 20000, uint64(time.Second)*10/uint64(time.Millisecond), clientUniqueID) + re.LessOrEqual(math.Abs(tb.Tokens-20000), 1e-7) re.Equal(trickle, int64(time.Second)*10/int64(time.Millisecond)) } diff --git a/pkg/mcs/scheduling/server/apis/v1/api.go b/pkg/mcs/scheduling/server/apis/v1/api.go new file mode 100644 index 00000000000..26eab6d9424 --- /dev/null +++ b/pkg/mcs/scheduling/server/apis/v1/api.go @@ -0,0 +1,94 @@ +// Copyright 2023 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package apis + +import ( + "net/http" + "sync" + + "github.com/gin-contrib/cors" + "github.com/gin-contrib/gzip" + "github.com/gin-contrib/pprof" + "github.com/gin-gonic/gin" + "github.com/joho/godotenv" + scheserver "github.com/tikv/pd/pkg/mcs/scheduling/server" + "github.com/tikv/pd/pkg/mcs/utils" + "github.com/tikv/pd/pkg/utils/apiutil" + "github.com/tikv/pd/pkg/utils/apiutil/multiservicesapi" + "github.com/unrolled/render" +) + +// APIPathPrefix is the prefix of the API path. +const APIPathPrefix = "/scheduling/api/v1/" + +var ( + once sync.Once + apiServiceGroup = apiutil.APIServiceGroup{ + Name: "scheduling", + Version: "v1", + IsCore: false, + PathPrefix: APIPathPrefix, + } +) + +func init() { + scheserver.SetUpRestHandler = func(srv *scheserver.Service) (http.Handler, apiutil.APIServiceGroup) { + s := NewService(srv) + return s.apiHandlerEngine, apiServiceGroup + } +} + +// Service is the tso service. +type Service struct { + apiHandlerEngine *gin.Engine + root *gin.RouterGroup + + srv *scheserver.Service + rd *render.Render +} + +func createIndentRender() *render.Render { + return render.New(render.Options{ + IndentJSON: true, + }) +} + +// NewService returns a new Service. +func NewService(srv *scheserver.Service) *Service { + once.Do(func() { + // These global modification will be effective only for the first invoke. + _ = godotenv.Load() + gin.SetMode(gin.ReleaseMode) + }) + apiHandlerEngine := gin.New() + apiHandlerEngine.Use(gin.Recovery()) + apiHandlerEngine.Use(cors.Default()) + apiHandlerEngine.Use(gzip.Gzip(gzip.DefaultCompression)) + apiHandlerEngine.Use(func(c *gin.Context) { + c.Set(multiservicesapi.ServiceContextKey, srv) + c.Next() + }) + apiHandlerEngine.Use(multiservicesapi.ServiceRedirector()) + apiHandlerEngine.GET("metrics", utils.PromHandler()) + pprof.Register(apiHandlerEngine) + root := apiHandlerEngine.Group(APIPathPrefix) + s := &Service{ + srv: srv, + apiHandlerEngine: apiHandlerEngine, + root: root, + rd: createIndentRender(), + } + return s +} diff --git a/pkg/mcs/scheduling/server/cluster.go b/pkg/mcs/scheduling/server/cluster.go new file mode 100644 index 00000000000..88b79ad9669 --- /dev/null +++ b/pkg/mcs/scheduling/server/cluster.go @@ -0,0 +1,38 @@ +package server + +import ( + "context" + "time" + + "github.com/tikv/pd/pkg/core" + "github.com/tikv/pd/pkg/mcs/scheduling/server/config" + "github.com/tikv/pd/pkg/schedule/labeler" + "github.com/tikv/pd/pkg/schedule/placement" + "github.com/tikv/pd/pkg/storage/endpoint" +) + +// Cluster is used to manage all information for scheduling purpose. +type Cluster struct { + basicCluster *core.BasicCluster + ruleManager *placement.RuleManager + labelerManager *labeler.RegionLabeler + persistConfig *config.PersistConfig +} + +const regionLabelGCInterval = time.Hour + +// NewCluster creates a new cluster. +func NewCluster(ctx context.Context, storage endpoint.RuleStorage, cfg *config.Config) (*Cluster, error) { + basicCluster := core.NewBasicCluster() + persistConfig := config.NewPersistConfig(cfg) + labelerManager, err := labeler.NewRegionLabeler(ctx, storage, regionLabelGCInterval) + if err != nil { + return nil, err + } + return &Cluster{ + basicCluster: basicCluster, + ruleManager: placement.NewRuleManager(storage, basicCluster, persistConfig), + labelerManager: labelerManager, + persistConfig: persistConfig, + }, nil +} diff --git a/pkg/mcs/scheduling/server/config/config.go b/pkg/mcs/scheduling/server/config/config.go new file mode 100644 index 00000000000..48ec0917e25 --- /dev/null +++ b/pkg/mcs/scheduling/server/config/config.go @@ -0,0 +1,363 @@ +// Copyright 2023 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import ( + "fmt" + "os" + "path/filepath" + "strings" + "sync/atomic" + "time" + "unsafe" + + "github.com/BurntSushi/toml" + "github.com/coreos/go-semver/semver" + "github.com/pingcap/errors" + "github.com/pingcap/kvproto/pkg/metapb" + "github.com/pingcap/log" + "github.com/spf13/pflag" + "github.com/tikv/pd/pkg/core/constant" + "github.com/tikv/pd/pkg/core/storelimit" + "github.com/tikv/pd/pkg/mcs/utils" + sc "github.com/tikv/pd/pkg/schedule/config" + "github.com/tikv/pd/pkg/utils/configutil" + "github.com/tikv/pd/pkg/utils/grpcutil" + "github.com/tikv/pd/pkg/utils/metricutil" + "go.uber.org/zap" +) + +const ( + defaultName = "Scheduling" + defaultBackendEndpoints = "http://127.0.0.1:2379" + defaultListenAddr = "http://127.0.0.1:3379" +) + +// Config is the configuration for the scheduling. +type Config struct { + BackendEndpoints string `toml:"backend-endpoints" json:"backend-endpoints"` + ListenAddr string `toml:"listen-addr" json:"listen-addr"` + AdvertiseListenAddr string `toml:"advertise-listen-addr" json:"advertise-listen-addr"` + Name string `toml:"name" json:"name"` + DataDir string `toml:"data-dir" json:"data-dir"` // TODO: remove this after refactoring + EnableGRPCGateway bool `json:"enable-grpc-gateway"` // TODO: use it + + Metric metricutil.MetricConfig `toml:"metric" json:"metric"` + + // Log related config. + Log log.Config `toml:"log" json:"log"` + Logger *zap.Logger + LogProps *log.ZapProperties + + Security configutil.SecurityConfig `toml:"security" json:"security"` + + // WarningMsgs contains all warnings during parsing. + WarningMsgs []string + + // LeaderLease defines the time within which a Scheduling primary/leader must + // update its TTL in etcd, otherwise etcd will expire the leader key and other servers + // can campaign the primary/leader again. Etcd only supports seconds TTL, so here is + // second too. + LeaderLease int64 `toml:"lease" json:"lease"` + + ClusterVersion semver.Version `toml:"cluster-version" json:"cluster-version"` + + Schedule sc.ScheduleConfig `toml:"schedule" json:"schedule"` + Replication sc.ReplicationConfig `toml:"replication" json:"replication"` +} + +// NewConfig creates a new config. +func NewConfig() *Config { + return &Config{} +} + +// Parse parses flag definitions from the argument list. +func (c *Config) Parse(flagSet *pflag.FlagSet) error { + // Load config file if specified. + var ( + meta *toml.MetaData + err error + ) + if configFile, _ := flagSet.GetString("config"); configFile != "" { + meta, err = configutil.ConfigFromFile(c, configFile) + if err != nil { + return err + } + } + + // Ignore the error check here + configutil.AdjustCommandLineString(flagSet, &c.Log.Level, "log-level") + configutil.AdjustCommandLineString(flagSet, &c.Log.File.Filename, "log-file") + configutil.AdjustCommandLineString(flagSet, &c.Metric.PushAddress, "metrics-addr") + configutil.AdjustCommandLineString(flagSet, &c.Security.CAPath, "cacert") + configutil.AdjustCommandLineString(flagSet, &c.Security.CertPath, "cert") + configutil.AdjustCommandLineString(flagSet, &c.Security.KeyPath, "key") + configutil.AdjustCommandLineString(flagSet, &c.BackendEndpoints, "backend-endpoints") + configutil.AdjustCommandLineString(flagSet, &c.ListenAddr, "listen-addr") + configutil.AdjustCommandLineString(flagSet, &c.AdvertiseListenAddr, "advertise-listen-addr") + + return c.adjust(meta) +} + +// adjust is used to adjust the scheduling configurations. +func (c *Config) adjust(meta *toml.MetaData) error { + configMetaData := configutil.NewConfigMetadata(meta) + if err := configMetaData.CheckUndecoded(); err != nil { + c.WarningMsgs = append(c.WarningMsgs, err.Error()) + } + + if c.Name == "" { + hostname, err := os.Hostname() + if err != nil { + return err + } + configutil.AdjustString(&c.Name, fmt.Sprintf("%s-%s", defaultName, hostname)) + } + configutil.AdjustString(&c.DataDir, fmt.Sprintf("default.%s", c.Name)) + configutil.AdjustPath(&c.DataDir) + + if err := c.validate(); err != nil { + return err + } + + configutil.AdjustString(&c.BackendEndpoints, defaultBackendEndpoints) + configutil.AdjustString(&c.ListenAddr, defaultListenAddr) + configutil.AdjustString(&c.AdvertiseListenAddr, c.ListenAddr) + + if !configMetaData.IsDefined("enable-grpc-gateway") { + c.EnableGRPCGateway = utils.DefaultEnableGRPCGateway + } + + c.adjustLog(configMetaData.Child("log")) + c.Security.Encryption.Adjust() + + if len(c.Log.Format) == 0 { + c.Log.Format = utils.DefaultLogFormat + } + + configutil.AdjustInt64(&c.LeaderLease, utils.DefaultLeaderLease) + + if err := c.Schedule.Adjust(configMetaData.Child("schedule"), false); err != nil { + return err + } + return c.Replication.Adjust(configMetaData.Child("replication")) +} + +func (c *Config) adjustLog(meta *configutil.ConfigMetaData) { + if !meta.IsDefined("disable-error-verbose") { + c.Log.DisableErrorVerbose = utils.DefaultDisableErrorVerbose + } +} + +// GetTLSConfig returns the TLS config. +func (c *Config) GetTLSConfig() *grpcutil.TLSConfig { + return &c.Security.TLSConfig +} + +// validate is used to validate if some configurations are right. +func (c *Config) validate() error { + dataDir, err := filepath.Abs(c.DataDir) + if err != nil { + return errors.WithStack(err) + } + logFile, err := filepath.Abs(c.Log.File.Filename) + if err != nil { + return errors.WithStack(err) + } + rel, err := filepath.Rel(dataDir, filepath.Dir(logFile)) + if err != nil { + return errors.WithStack(err) + } + if !strings.HasPrefix(rel, "..") { + return errors.New("log directory shouldn't be the subdirectory of data directory") + } + + return nil +} + +// PersistConfig wraps all configurations that need to persist to storage and +// allows to access them safely. +type PersistConfig struct { + clusterVersion unsafe.Pointer + schedule atomic.Value + replication atomic.Value +} + +// NewPersistConfig creates a new PersistConfig instance. +func NewPersistConfig(cfg *Config) *PersistConfig { + o := &PersistConfig{} + o.SetClusterVersion(&cfg.ClusterVersion) + o.schedule.Store(&cfg.Schedule) + o.replication.Store(&cfg.Replication) + return o +} + +// GetClusterVersion returns the cluster version. +func (o *PersistConfig) GetClusterVersion() *semver.Version { + return (*semver.Version)(atomic.LoadPointer(&o.clusterVersion)) +} + +// SetClusterVersion sets the cluster version. +func (o *PersistConfig) SetClusterVersion(v *semver.Version) { + atomic.StorePointer(&o.clusterVersion, unsafe.Pointer(v)) +} + +// GetScheduleConfig returns the scheduling configurations. +func (o *PersistConfig) GetScheduleConfig() *sc.ScheduleConfig { + return o.schedule.Load().(*sc.ScheduleConfig) +} + +// SetScheduleConfig sets the scheduling configuration. +func (o *PersistConfig) SetScheduleConfig(cfg *sc.ScheduleConfig) { + o.schedule.Store(cfg) +} + +// GetReplicationConfig returns replication configurations. +func (o *PersistConfig) GetReplicationConfig() *sc.ReplicationConfig { + return o.replication.Load().(*sc.ReplicationConfig) +} + +// SetReplicationConfig sets the PD replication configuration. +func (o *PersistConfig) SetReplicationConfig(cfg *sc.ReplicationConfig) { + o.replication.Store(cfg) +} + +// GetMaxReplicas returns the max replicas. +func (o *PersistConfig) GetMaxReplicas() int { + return int(o.GetReplicationConfig().MaxReplicas) +} + +// GetMaxSnapshotCount returns the max snapshot count. +func (o *PersistConfig) GetMaxSnapshotCount() uint64 { + return o.GetScheduleConfig().MaxSnapshotCount +} + +// GetMaxPendingPeerCount returns the max pending peer count. +func (o *PersistConfig) GetMaxPendingPeerCount() uint64 { + return o.GetScheduleConfig().MaxPendingPeerCount +} + +// IsPlacementRulesEnabled returns if the placement rules is enabled. +func (o *PersistConfig) IsPlacementRulesEnabled() bool { + return o.GetReplicationConfig().EnablePlacementRules +} + +// GetLowSpaceRatio returns the low space ratio. +func (o *PersistConfig) GetLowSpaceRatio() float64 { + return o.GetScheduleConfig().LowSpaceRatio +} + +// GetHighSpaceRatio returns the high space ratio. +func (o *PersistConfig) GetHighSpaceRatio() float64 { + return o.GetScheduleConfig().HighSpaceRatio +} + +// GetMaxStoreDownTime returns the max store downtime. +func (o *PersistConfig) GetMaxStoreDownTime() time.Duration { + return o.GetScheduleConfig().MaxStoreDownTime.Duration +} + +// GetLocationLabels returns the location labels. +func (o *PersistConfig) GetLocationLabels() []string { + return o.GetReplicationConfig().LocationLabels +} + +// CheckLabelProperty checks if the label property is satisfied. +func (o *PersistConfig) CheckLabelProperty(typ string, labels []*metapb.StoreLabel) bool { + return false +} + +// IsUseJointConsensus returns if the joint consensus is enabled. +func (o *PersistConfig) IsUseJointConsensus() bool { + return true +} + +// GetKeyType returns the key type. +func (o *PersistConfig) GetKeyType() constant.KeyType { + return constant.StringToKeyType("table") +} + +// IsCrossTableMergeEnabled returns if the cross table merge is enabled. +func (o *PersistConfig) IsCrossTableMergeEnabled() bool { + return o.GetScheduleConfig().EnableCrossTableMerge +} + +// IsOneWayMergeEnabled returns if the one way merge is enabled. +func (o *PersistConfig) IsOneWayMergeEnabled() bool { + return o.GetScheduleConfig().EnableOneWayMerge +} + +// GetMergeScheduleLimit returns the merge schedule limit. +func (o *PersistConfig) GetMergeScheduleLimit() uint64 { + return o.GetScheduleConfig().MergeScheduleLimit +} + +// GetRegionScoreFormulaVersion returns the region score formula version. +func (o *PersistConfig) GetRegionScoreFormulaVersion() string { + return o.GetScheduleConfig().RegionScoreFormulaVersion +} + +// GetSchedulerMaxWaitingOperator returns the scheduler max waiting operator. +func (o *PersistConfig) GetSchedulerMaxWaitingOperator() uint64 { + return o.GetScheduleConfig().SchedulerMaxWaitingOperator +} + +// GetStoreLimitByType returns the limit of a store with a given type. +func (o *PersistConfig) GetStoreLimitByType(storeID uint64, typ storelimit.Type) (returned float64) { + limit := o.GetStoreLimit(storeID) + switch typ { + case storelimit.AddPeer: + return limit.AddPeer + case storelimit.RemovePeer: + return limit.RemovePeer + // todo: impl it in store limit v2. + case storelimit.SendSnapshot: + return 0.0 + default: + panic("no such limit type") + } +} + +// GetStoreLimit returns the limit of a store. +func (o *PersistConfig) GetStoreLimit(storeID uint64) (returnSC sc.StoreLimitConfig) { + if limit, ok := o.GetScheduleConfig().StoreLimit[storeID]; ok { + return limit + } + cfg := o.GetScheduleConfig().Clone() + sc := sc.StoreLimitConfig{ + AddPeer: sc.DefaultStoreLimit.GetDefaultStoreLimit(storelimit.AddPeer), + RemovePeer: sc.DefaultStoreLimit.GetDefaultStoreLimit(storelimit.RemovePeer), + } + + cfg.StoreLimit[storeID] = sc + o.SetScheduleConfig(cfg) + return o.GetScheduleConfig().StoreLimit[storeID] +} + +// IsWitnessAllowed returns if the witness is allowed. +func (o *PersistConfig) IsWitnessAllowed() bool { + return false +} + +// IsPlacementRulesCacheEnabled returns if the placement rules cache is enabled. +func (o *PersistConfig) IsPlacementRulesCacheEnabled() bool { + return false +} + +// SetPlacementRulesCacheEnabled sets if the placement rules cache is enabled. +func (o *PersistConfig) SetPlacementRulesCacheEnabled(b bool) {} + +// SetEnableWitness sets if the witness is enabled. +func (o *PersistConfig) SetEnableWitness(b bool) {} diff --git a/pkg/mcs/scheduling/server/config/config_watcher.go b/pkg/mcs/scheduling/server/config/config_watcher.go new file mode 100644 index 00000000000..fdf957b72c5 --- /dev/null +++ b/pkg/mcs/scheduling/server/config/config_watcher.go @@ -0,0 +1,106 @@ +// Copyright 2023 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import ( + "context" + "encoding/json" + "sync" + + "github.com/coreos/go-semver/semver" + "github.com/pingcap/log" + sc "github.com/tikv/pd/pkg/schedule/config" + "github.com/tikv/pd/pkg/utils/etcdutil" + "go.etcd.io/etcd/clientv3" + "go.etcd.io/etcd/mvcc/mvccpb" + "go.uber.org/zap" +) + +// Watcher is used to watch the PD API server for any configuration changes. +type Watcher struct { + wg sync.WaitGroup + ctx context.Context + cancel context.CancelFunc + + etcdClient *clientv3.Client + watcher *etcdutil.LoopWatcher + + *PersistConfig + // TODO: watch the scheduler config change. +} + +type persistedConfig struct { + Schedule sc.ScheduleConfig `json:"schedule"` + Replication sc.ReplicationConfig `json:"replication"` + ClusterVersion semver.Version `json:"cluster-version"` +} + +// NewWatcher creates a new watcher to watch the config meta change from PD API server. +func NewWatcher( + ctx context.Context, + etcdClient *clientv3.Client, + // configPath is the path of the configuration in etcd: + // - Key: /pd/{cluster_id}/config + // - Value: configuration JSON. + configPath string, + persistConfig *PersistConfig, +) (*Watcher, error) { + ctx, cancel := context.WithCancel(ctx) + cw := &Watcher{ + ctx: ctx, + cancel: cancel, + etcdClient: etcdClient, + PersistConfig: persistConfig, + } + putFn := func(kv *mvccpb.KeyValue) error { + cfg := &persistedConfig{} + if err := json.Unmarshal(kv.Value, cfg); err != nil { + log.Warn("failed to unmarshal scheduling config entry", + zap.String("event-kv-key", string(kv.Key)), zap.Error(err)) + return err + } + cw.SetScheduleConfig(&cfg.Schedule) + cw.SetReplicationConfig(&cfg.Replication) + cw.SetClusterVersion(&cfg.ClusterVersion) + return nil + } + deleteFn := func(kv *mvccpb.KeyValue) error { + return nil + } + postEventFn := func() error { + return nil + } + cw.watcher = etcdutil.NewLoopWatcher( + ctx, + &cw.wg, + etcdClient, + "scheduling-config-watcher", + configPath, + putFn, + deleteFn, + postEventFn, + ) + cw.watcher.StartWatchLoop() + if err := cw.watcher.WaitLoad(); err != nil { + return nil, err + } + return cw, nil +} + +// Close closes the watcher. +func (cw *Watcher) Close() { + cw.cancel() + cw.wg.Wait() +} diff --git a/pkg/mcs/scheduling/server/grpc_service.go b/pkg/mcs/scheduling/server/grpc_service.go new file mode 100644 index 00000000000..3b0e51f1f66 --- /dev/null +++ b/pkg/mcs/scheduling/server/grpc_service.go @@ -0,0 +1,71 @@ +// Copyright 2023 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "net/http" + + "github.com/pingcap/log" + bs "github.com/tikv/pd/pkg/basicserver" + "github.com/tikv/pd/pkg/mcs/registry" + "github.com/tikv/pd/pkg/utils/apiutil" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// gRPC errors +var ( + ErrNotStarted = status.Errorf(codes.Unavailable, "server not started") + ErrClusterMismatched = status.Errorf(codes.Unavailable, "cluster mismatched") +) + +// SetUpRestHandler is a hook to sets up the REST service. +var SetUpRestHandler = func(srv *Service) (http.Handler, apiutil.APIServiceGroup) { + return dummyRestService{}, apiutil.APIServiceGroup{} +} + +type dummyRestService struct{} + +func (d dummyRestService) ServeHTTP(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotImplemented) + w.Write([]byte("not implemented")) +} + +// Service is the scheduling grpc service. +type Service struct { + *Server +} + +// NewService creates a new TSO service. +func NewService(svr bs.Server) registry.RegistrableService { + server, ok := svr.(*Server) + if !ok { + log.Fatal("create scheduling server failed") + } + return &Service{ + Server: server, + } +} + +// RegisterGRPCService registers the service to gRPC server. +func (s *Service) RegisterGRPCService(g *grpc.Server) { +} + +// RegisterRESTHandler registers the service to REST server. +func (s *Service) RegisterRESTHandler(userDefineHandlers map[string]http.Handler) { + handler, group := SetUpRestHandler(s) + apiutil.RegisterUserDefinedHandlers(userDefineHandlers, &group, handler) +} diff --git a/pkg/mcs/scheduling/server/metrics.go b/pkg/mcs/scheduling/server/metrics.go new file mode 100644 index 00000000000..b3f5b7b41de --- /dev/null +++ b/pkg/mcs/scheduling/server/metrics.go @@ -0,0 +1,37 @@ +// Copyright 2023 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import "github.com/prometheus/client_golang/prometheus" + +const ( + namespace = "scheduling" + serverSubsystem = "server" +) + +var ( + // Meta & Server info. + serverInfo = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: serverSubsystem, + Name: "info", + Help: "Indicate the scheduling server info, and the value is the start timestamp (s).", + }, []string{"version", "hash"}) +) + +func init() { + prometheus.MustRegister(serverInfo) +} diff --git a/pkg/mcs/scheduling/server/server.go b/pkg/mcs/scheduling/server/server.go new file mode 100644 index 00000000000..a1c74ab0c99 --- /dev/null +++ b/pkg/mcs/scheduling/server/server.go @@ -0,0 +1,610 @@ +// Copyright 2023 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "context" + "crypto/tls" + "fmt" + "net" + "net/http" + "net/url" + "os" + "os/signal" + "path" + "strconv" + "strings" + "sync" + "sync/atomic" + "syscall" + "time" + + grpcprometheus "github.com/grpc-ecosystem/go-grpc-prometheus" + "github.com/pingcap/failpoint" + "github.com/pingcap/kvproto/pkg/diagnosticspb" + "github.com/pingcap/log" + "github.com/pingcap/sysutil" + "github.com/soheilhy/cmux" + "github.com/spf13/cobra" + "github.com/tikv/pd/pkg/errs" + "github.com/tikv/pd/pkg/mcs/discovery" + "github.com/tikv/pd/pkg/mcs/scheduling/server/config" + "github.com/tikv/pd/pkg/mcs/utils" + "github.com/tikv/pd/pkg/member" + "github.com/tikv/pd/pkg/storage/endpoint" + "github.com/tikv/pd/pkg/storage/kv" + "github.com/tikv/pd/pkg/utils/etcdutil" + "github.com/tikv/pd/pkg/utils/grpcutil" + "github.com/tikv/pd/pkg/utils/logutil" + "github.com/tikv/pd/pkg/utils/memberutil" + "github.com/tikv/pd/pkg/utils/metricutil" + "github.com/tikv/pd/pkg/versioninfo" + "go.etcd.io/etcd/clientv3" + "go.etcd.io/etcd/pkg/types" + "go.uber.org/zap" + "google.golang.org/grpc" +) + +// Server is the scheduling server, and it implements bs.Server. +type Server struct { + diagnosticspb.DiagnosticsServer + + // Server state. 0 is not running, 1 is running. + isRunning int64 + // Server start timestamp + startTimestamp int64 + + ctx context.Context + serverLoopCtx context.Context + serverLoopCancel func() + serverLoopWg sync.WaitGroup + + cfg *config.Config + name string + clusterID uint64 + listenURL *url.URL + backendUrls []url.URL + + // etcd client + etcdClient *clientv3.Client + // http client + httpClient *http.Client + + // Store as map[string]*grpc.ClientConn + clientConns sync.Map + + // for the primary election of scheduling + participant *member.Participant + + secure bool + muxListener net.Listener + httpListener net.Listener + grpcServer *grpc.Server + httpServer *http.Server + service *Service + + // Callback functions for different stages + // startCallbacks will be called after the server is started. + startCallbacks []func() + // primaryCallbacks will be called after the server becomes leader. + primaryCallbacks []func(context.Context) + + // for service registry + serviceID *discovery.ServiceRegistryEntry + serviceRegister *discovery.ServiceRegister + + cluster *Cluster + storage *endpoint.StorageEndpoint +} + +// Name returns the unique etcd name for this server in etcd cluster. +func (s *Server) Name() string { + return s.name +} + +// Context returns the context. +func (s *Server) Context() context.Context { + return s.ctx +} + +// GetAddr returns the server address. +func (s *Server) GetAddr() string { + return s.cfg.ListenAddr +} + +// GetBackendEndpoints returns the backend endpoints. +func (s *Server) GetBackendEndpoints() string { + return s.cfg.BackendEndpoints +} + +// GetClientConns returns the client connections. +func (s *Server) GetClientConns() *sync.Map { + return &s.clientConns +} + +// Run runs the scheduling server. +func (s *Server) Run() error { + skipWaitAPIServiceReady := false + failpoint.Inject("skipWaitAPIServiceReady", func() { + skipWaitAPIServiceReady = true + }) + if !skipWaitAPIServiceReady { + if err := utils.WaitAPIServiceReady(s); err != nil { + return err + } + } + + if err := s.initClient(); err != nil { + return err + } + return s.startServer() +} + +func (s *Server) startServerLoop() { + s.serverLoopCtx, s.serverLoopCancel = context.WithCancel(s.ctx) + s.serverLoopWg.Add(1) + go s.primaryElectionLoop() +} + +func (s *Server) primaryElectionLoop() { + defer logutil.LogPanic() + defer s.serverLoopWg.Done() + + for { + if s.IsClosed() { + log.Info("server is closed, exit scheduling primary election loop") + return + } + + primary, checkAgain := s.participant.CheckLeader() + if checkAgain { + continue + } + if primary != nil { + log.Info("start to watch the primary", zap.Stringer("scheduling-primary", primary)) + // Watch will keep looping and never return unless the primary/leader has changed. + primary.Watch(s.serverLoopCtx) + log.Info("the scheduling primary has changed, try to re-campaign a primary") + } + + s.campaignLeader() + } +} + +func (s *Server) campaignLeader() { + log.Info("start to campaign the primary/leader", zap.String("campaign-scheduling-primary-name", s.participant.Name())) + if err := s.participant.CampaignLeader(s.cfg.LeaderLease); err != nil { + if err.Error() == errs.ErrEtcdTxnConflict.Error() { + log.Info("campaign scheduling primary meets error due to txn conflict, another server may campaign successfully", + zap.String("campaign-scheduling-primary-name", s.participant.Name())) + } else { + log.Error("campaign scheduling primary meets error due to etcd error", + zap.String("campaign-scheduling-primary-name", s.participant.Name()), + errs.ZapError(err)) + } + return + } + + // Start keepalive the leadership and enable Scheduling service. + ctx, cancel := context.WithCancel(s.serverLoopCtx) + var resetLeaderOnce sync.Once + defer resetLeaderOnce.Do(func() { + cancel() + s.participant.ResetLeader() + }) + + // maintain the leadership, after this, Scheduling could be ready to provide service. + s.participant.KeepLeader(ctx) + log.Info("campaign scheduling primary ok", zap.String("campaign-scheduling-primary-name", s.participant.Name())) + + log.Info("triggering the primary callback functions") + for _, cb := range s.primaryCallbacks { + cb(ctx) + } + + s.participant.EnableLeader() + log.Info("scheduling primary is ready to serve", zap.String("scheduling-primary-name", s.participant.Name())) + + leaderTicker := time.NewTicker(utils.LeaderTickInterval) + defer leaderTicker.Stop() + + for { + select { + case <-leaderTicker.C: + if !s.participant.IsLeader() { + log.Info("no longer a primary/leader because lease has expired, the scheduling primary/leader will step down") + return + } + case <-ctx.Done(): + // Server is closed and it should return nil. + log.Info("server is closed") + return + } + } +} + +// Close closes the server. +func (s *Server) Close() { + if !atomic.CompareAndSwapInt64(&s.isRunning, 1, 0) { + // server is already closed + return + } + + log.Info("closing scheduling server ...") + s.serviceRegister.Deregister() + s.stopHTTPServer() + s.stopGRPCServer() + s.muxListener.Close() + s.serverLoopCancel() + s.serverLoopWg.Wait() + + if s.etcdClient != nil { + if err := s.etcdClient.Close(); err != nil { + log.Error("close etcd client meet error", errs.ZapError(errs.ErrCloseEtcdClient, err)) + } + } + + if s.httpClient != nil { + s.httpClient.CloseIdleConnections() + } + log.Info("scheduling server is closed") +} + +// GetClient returns builtin etcd client. +func (s *Server) GetClient() *clientv3.Client { + return s.etcdClient +} + +// GetHTTPClient returns builtin http client. +func (s *Server) GetHTTPClient() *http.Client { + return s.httpClient +} + +// AddStartCallback adds a callback in the startServer phase. +func (s *Server) AddStartCallback(callbacks ...func()) { + s.startCallbacks = append(s.startCallbacks, callbacks...) +} + +// IsServing returns whether the server is the leader, if there is embedded etcd, or the primary otherwise. +func (s *Server) IsServing() bool { + return !s.IsClosed() && s.participant.IsLeader() +} + +// IsClosed checks if the server loop is closed +func (s *Server) IsClosed() bool { + return s != nil && atomic.LoadInt64(&s.isRunning) == 0 +} + +// AddServiceReadyCallback adds callbacks when the server becomes the leader, if there is embedded etcd, or the primary otherwise. +func (s *Server) AddServiceReadyCallback(callbacks ...func(context.Context)) { + s.primaryCallbacks = append(s.primaryCallbacks, callbacks...) +} + +// GetDelegateClient returns grpc client connection talking to the forwarded host +func (s *Server) GetDelegateClient(ctx context.Context, forwardedHost string) (*grpc.ClientConn, error) { + client, ok := s.clientConns.Load(forwardedHost) + if !ok { + tlsConfig, err := s.GetTLSConfig().ToTLSConfig() + if err != nil { + return nil, err + } + cc, err := grpcutil.GetClientConn(ctx, forwardedHost, tlsConfig) + if err != nil { + return nil, err + } + client = cc + s.clientConns.Store(forwardedHost, cc) + } + return client.(*grpc.ClientConn), nil +} + +// GetTLSConfig gets the security config. +func (s *Server) GetTLSConfig() *grpcutil.TLSConfig { + return &s.cfg.Security.TLSConfig +} + +func (s *Server) initClient() error { + tlsConfig, err := s.cfg.Security.ToTLSConfig() + if err != nil { + return err + } + s.backendUrls, err = types.NewURLs(strings.Split(s.cfg.BackendEndpoints, ",")) + if err != nil { + return err + } + s.etcdClient, s.httpClient, err = etcdutil.CreateClients(tlsConfig, s.backendUrls) + return err +} + +func (s *Server) startGRPCServer(l net.Listener) { + defer logutil.LogPanic() + defer s.serverLoopWg.Done() + + log.Info("grpc server starts serving", zap.String("address", l.Addr().String())) + err := s.grpcServer.Serve(l) + if s.IsClosed() { + log.Info("grpc server stopped") + } else { + log.Fatal("grpc server stopped unexpectedly", errs.ZapError(err)) + } +} + +func (s *Server) startHTTPServer(l net.Listener) { + defer logutil.LogPanic() + defer s.serverLoopWg.Done() + + log.Info("http server starts serving", zap.String("address", l.Addr().String())) + err := s.httpServer.Serve(l) + if s.IsClosed() { + log.Info("http server stopped") + } else { + log.Fatal("http server stopped unexpectedly", errs.ZapError(err)) + } +} + +func (s *Server) startGRPCAndHTTPServers(serverReadyChan chan<- struct{}, l net.Listener) { + defer logutil.LogPanic() + defer s.serverLoopWg.Done() + + mux := cmux.New(l) + // Don't hang on matcher after closing listener + mux.SetReadTimeout(3 * time.Second) + grpcL := mux.MatchWithWriters(cmux.HTTP2MatchHeaderFieldSendSettings("content-type", "application/grpc")) + if s.secure { + s.httpListener = mux.Match(cmux.Any()) + } else { + s.httpListener = mux.Match(cmux.HTTP1()) + } + + s.grpcServer = grpc.NewServer() + s.service.RegisterGRPCService(s.grpcServer) + diagnosticspb.RegisterDiagnosticsServer(s.grpcServer, s) + s.serverLoopWg.Add(1) + go s.startGRPCServer(grpcL) + + handler, _ := SetUpRestHandler(s.service) + s.httpServer = &http.Server{ + Handler: handler, + ReadTimeout: 3 * time.Second, + } + s.serverLoopWg.Add(1) + go s.startHTTPServer(s.httpListener) + + serverReadyChan <- struct{}{} + if err := mux.Serve(); err != nil { + if s.IsClosed() { + log.Info("mux stopped serving", errs.ZapError(err)) + } else { + log.Fatal("mux stopped serving unexpectedly", errs.ZapError(err)) + } + } +} + +// GetLeaderListenUrls gets service endpoints from the leader in election group. +func (s *Server) GetLeaderListenUrls() []string { + return s.participant.GetLeaderListenUrls() +} + +func (s *Server) stopHTTPServer() { + log.Info("stopping http server") + defer log.Info("http server stopped") + + ctx, cancel := context.WithTimeout(context.Background(), utils.DefaultHTTPGracefulShutdownTimeout) + defer cancel() + + // First, try to gracefully shutdown the http server + ch := make(chan struct{}) + go func() { + defer close(ch) + s.httpServer.Shutdown(ctx) + }() + + select { + case <-ch: + case <-ctx.Done(): + // Took too long, manually close open transports + log.Warn("http server graceful shutdown timeout, forcing close") + s.httpServer.Close() + // concurrent Graceful Shutdown should be interrupted + <-ch + } +} + +func (s *Server) stopGRPCServer() { + log.Info("stopping grpc server") + defer log.Info("grpc server stopped") + + // Do not grpc.Server.GracefulStop with TLS enabled etcd server + // See https://github.com/grpc/grpc-go/issues/1384#issuecomment-317124531 + // and https://github.com/etcd-io/etcd/issues/8916 + if s.secure { + s.grpcServer.Stop() + return + } + + ctx, cancel := context.WithTimeout(context.Background(), utils.DefaultGRPCGracefulStopTimeout) + defer cancel() + + // First, try to gracefully shutdown the grpc server + ch := make(chan struct{}) + go func() { + defer close(ch) + // Close listeners to stop accepting new connections, + // will block on any existing transports + s.grpcServer.GracefulStop() + }() + + // Wait until all pending RPCs are finished + select { + case <-ch: + case <-ctx.Done(): + // Took too long, manually close open transports + // e.g. watch streams + log.Warn("grpc server graceful shutdown timeout, forcing close") + s.grpcServer.Stop() + // concurrent GracefulStop should be interrupted + <-ch + } +} + +func (s *Server) startServer() (err error) { + if s.clusterID, err = utils.InitClusterID(s.ctx, s.etcdClient); err != nil { + return err + } + log.Info("init cluster id", zap.Uint64("cluster-id", s.clusterID)) + // The independent Scheduling service still reuses PD version info since PD and Scheduling are just + // different service modes provided by the same pd-server binary + serverInfo.WithLabelValues(versioninfo.PDReleaseVersion, versioninfo.PDGitHash).Set(float64(time.Now().Unix())) + + uniqueName := s.cfg.ListenAddr + uniqueID := memberutil.GenerateUniqueID(uniqueName) + log.Info("joining primary election", zap.String("participant-name", uniqueName), zap.Uint64("participant-id", uniqueID)) + schedulingPrimaryPrefix := endpoint.SchedulingSvcRootPath(s.clusterID) + s.participant = member.NewParticipant(s.etcdClient) + s.participant.InitInfo(uniqueName, uniqueID, path.Join(schedulingPrimaryPrefix, fmt.Sprintf("%05d", 0)), + utils.PrimaryKey, "primary election", s.cfg.AdvertiseListenAddr) + s.storage = endpoint.NewStorageEndpoint( + kv.NewEtcdKVBase(s.etcdClient, endpoint.SchedulingSvcRootPath(s.clusterID)), nil) + s.cluster, err = NewCluster(s.ctx, s.storage, s.cfg) + if err != nil { + return err + } + tlsConfig, err := s.cfg.Security.ToTLSConfig() + if err != nil { + return err + } + if tlsConfig != nil { + s.secure = true + s.muxListener, err = tls.Listen(utils.TCPNetworkStr, s.listenURL.Host, tlsConfig) + } else { + s.muxListener, err = net.Listen(utils.TCPNetworkStr, s.listenURL.Host) + } + if err != nil { + return err + } + + serverReadyChan := make(chan struct{}) + defer close(serverReadyChan) + s.serverLoopWg.Add(1) + go s.startGRPCAndHTTPServers(serverReadyChan, s.muxListener) + <-serverReadyChan + s.startServerLoop() + + // Run callbacks + log.Info("triggering the start callback functions") + for _, cb := range s.startCallbacks { + cb() + } + + // Server has started. + serializedEntry, err := s.serviceID.Serialize() + if err != nil { + return err + } + s.serviceRegister = discovery.NewServiceRegister(s.ctx, s.etcdClient, strconv.FormatUint(s.clusterID, 10), + utils.SchedulingServiceName, s.cfg.AdvertiseListenAddr, serializedEntry, discovery.DefaultLeaseInSeconds) + if err := s.serviceRegister.Register(); err != nil { + log.Error("failed to register the service", zap.String("service-name", utils.SchedulingServiceName), errs.ZapError(err)) + return err + } + + atomic.StoreInt64(&s.isRunning, 1) + return nil +} + +// CreateServer creates the Server +func CreateServer(ctx context.Context, cfg *config.Config) *Server { + svr := &Server{ + DiagnosticsServer: sysutil.NewDiagnosticsServer(cfg.Log.File.Filename), + startTimestamp: time.Now().Unix(), + cfg: cfg, + ctx: ctx, + } + return svr +} + +// CreateServerWrapper encapsulates the configuration/log/metrics initialization and create the server +func CreateServerWrapper(cmd *cobra.Command, args []string) { + cmd.Flags().Parse(args) + cfg := config.NewConfig() + flagSet := cmd.Flags() + err := cfg.Parse(flagSet) + defer logutil.LogPanic() + + if err != nil { + cmd.Println(err) + return + } + + if printVersion, err := flagSet.GetBool("version"); err != nil { + cmd.Println(err) + return + } else if printVersion { + versioninfo.Print() + exit(0) + } + + // New zap logger + err = logutil.SetupLogger(cfg.Log, &cfg.Logger, &cfg.LogProps, cfg.Security.RedactInfoLog) + if err == nil { + log.ReplaceGlobals(cfg.Logger, cfg.LogProps) + } else { + log.Fatal("initialize logger error", errs.ZapError(err)) + } + // Flushing any buffered log entries + defer log.Sync() + + versioninfo.Log("Scheduling") + log.Info("Scheduling config", zap.Reflect("config", cfg)) + + grpcprometheus.EnableHandlingTimeHistogram() + metricutil.Push(&cfg.Metric) + + ctx, cancel := context.WithCancel(context.Background()) + svr := CreateServer(ctx, cfg) + + sc := make(chan os.Signal, 1) + signal.Notify(sc, + syscall.SIGHUP, + syscall.SIGINT, + syscall.SIGTERM, + syscall.SIGQUIT) + + var sig os.Signal + go func() { + sig = <-sc + cancel() + }() + + if err := svr.Run(); err != nil { + log.Fatal("run server failed", errs.ZapError(err)) + } + + <-ctx.Done() + log.Info("got signal to exit", zap.String("signal", sig.String())) + + svr.Close() + switch sig { + case syscall.SIGTERM: + exit(0) + default: + exit(1) + } +} + +func exit(code int) { + log.Sync() + os.Exit(code) +} diff --git a/pkg/mcs/tso/server/apis/v1/api.go b/pkg/mcs/tso/server/apis/v1/api.go index 59905e8e34e..f33be3bade9 100644 --- a/pkg/mcs/tso/server/apis/v1/api.go +++ b/pkg/mcs/tso/server/apis/v1/api.go @@ -16,21 +16,32 @@ package apis import ( "net/http" + "sync" "github.com/gin-contrib/cors" "github.com/gin-contrib/gzip" + "github.com/gin-contrib/pprof" "github.com/gin-gonic/gin" + "github.com/joho/godotenv" + "github.com/pingcap/kvproto/pkg/tsopb" + "github.com/pingcap/log" tsoserver "github.com/tikv/pd/pkg/mcs/tso/server" + "github.com/tikv/pd/pkg/mcs/utils" + "github.com/tikv/pd/pkg/storage/endpoint" "github.com/tikv/pd/pkg/tso" "github.com/tikv/pd/pkg/utils/apiutil" "github.com/tikv/pd/pkg/utils/apiutil/multiservicesapi" "github.com/unrolled/render" + "go.uber.org/zap" ) -// APIPathPrefix is the prefix of the API path. -const APIPathPrefix = "/tso/api/v1" +const ( + // APIPathPrefix is the prefix of the API path. + APIPathPrefix = "/tso/api/v1" +) var ( + once sync.Once apiServiceGroup = apiutil.APIServiceGroup{ Name: "tso", Version: "v1", @@ -42,14 +53,14 @@ var ( func init() { tsoserver.SetUpRestHandler = func(srv *tsoserver.Service) (http.Handler, apiutil.APIServiceGroup) { s := NewService(srv) - return s.handler(), apiServiceGroup + return s.apiHandlerEngine, apiServiceGroup } } // Service is the tso service. type Service struct { apiHandlerEngine *gin.Engine - baseEndpoint *gin.RouterGroup + root *gin.RouterGroup srv *tsoserver.Service rd *render.Render @@ -63,34 +74,75 @@ func createIndentRender() *render.Render { // NewService returns a new Service. func NewService(srv *tsoserver.Service) *Service { + once.Do(func() { + // These global modification will be effective only for the first invoke. + _ = godotenv.Load() + gin.SetMode(gin.ReleaseMode) + }) apiHandlerEngine := gin.New() apiHandlerEngine.Use(gin.Recovery()) apiHandlerEngine.Use(cors.Default()) apiHandlerEngine.Use(gzip.Gzip(gzip.DefaultCompression)) apiHandlerEngine.Use(func(c *gin.Context) { - c.Set("service", srv) + c.Set(multiservicesapi.ServiceContextKey, srv) c.Next() }) apiHandlerEngine.Use(multiservicesapi.ServiceRedirector()) - endpoint := apiHandlerEngine.Group(APIPathPrefix) + apiHandlerEngine.GET("metrics", utils.PromHandler()) + pprof.Register(apiHandlerEngine) + root := apiHandlerEngine.Group(APIPathPrefix) s := &Service{ srv: srv, apiHandlerEngine: apiHandlerEngine, - baseEndpoint: endpoint, + root: root, rd: createIndentRender(), } - s.RegisterRouter() + s.RegisterAdminRouter() + s.RegisterKeyspaceGroupRouter() return s } -// RegisterRouter registers the router of the service. -func (s *Service) RegisterRouter() { +// RegisterAdminRouter registers the router of the TSO admin handler. +func (s *Service) RegisterAdminRouter() { + router := s.root.Group("admin") tsoAdminHandler := tso.NewAdminHandler(s.srv.GetHandler(), s.rd) - s.baseEndpoint.POST("/admin/reset-ts", gin.WrapF(tsoAdminHandler.ResetTS)) + router.POST("/reset-ts", gin.WrapF(tsoAdminHandler.ResetTS)) } -func (s *Service) handler() http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - s.apiHandlerEngine.ServeHTTP(w, r) - }) +// RegisterKeyspaceGroupRouter registers the router of the TSO keyspace group handler. +func (s *Service) RegisterKeyspaceGroupRouter() { + router := s.root.Group("keyspace-groups") + router.GET("/members", GetKeyspaceGroupMembers) +} + +// KeyspaceGroupMember contains the keyspace group and its member information. +type KeyspaceGroupMember struct { + Group *endpoint.KeyspaceGroup + Member *tsopb.Participant + IsPrimary bool `json:"is_primary"` + PrimaryID uint64 `json:"primary_id"` +} + +// GetKeyspaceGroupMembers gets the keyspace group members that the TSO service is serving. +func GetKeyspaceGroupMembers(c *gin.Context) { + svr := c.MustGet(multiservicesapi.ServiceContextKey).(*tsoserver.Service) + kgm := svr.GetKeyspaceGroupManager() + keyspaceGroups := kgm.GetKeyspaceGroups() + members := make(map[uint32]*KeyspaceGroupMember, len(keyspaceGroups)) + for id, group := range keyspaceGroups { + am, err := kgm.GetAllocatorManager(id) + if err != nil { + log.Error("failed to get allocator manager", + zap.Uint32("keyspace-group-id", id), zap.Error(err)) + continue + } + member := am.GetMember() + members[id] = &KeyspaceGroupMember{ + Group: group, + Member: member.GetMember().(*tsopb.Participant), + IsPrimary: member.IsLeader(), + PrimaryID: member.GetLeaderID(), + } + } + c.IndentedJSON(http.StatusOK, members) } diff --git a/pkg/mcs/tso/server/config.go b/pkg/mcs/tso/server/config.go index 14c13c601c9..23bff558239 100644 --- a/pkg/mcs/tso/server/config.go +++ b/pkg/mcs/tso/server/config.go @@ -26,6 +26,7 @@ import ( "github.com/pkg/errors" "github.com/spf13/pflag" "github.com/tikv/pd/pkg/mcs/utils" + "github.com/tikv/pd/pkg/tso" "github.com/tikv/pd/pkg/utils/configutil" "github.com/tikv/pd/pkg/utils/grpcutil" "github.com/tikv/pd/pkg/utils/metricutil" @@ -46,10 +47,13 @@ const ( minTSOUpdatePhysicalInterval = 1 * time.Millisecond ) +var _ tso.ServiceConfig = (*Config)(nil) + // Config is the configuration for the TSO. type Config struct { - BackendEndpoints string `toml:"backend-endpoints" json:"backend-endpoints"` - ListenAddr string `toml:"listen-addr" json:"listen-addr"` + BackendEndpoints string `toml:"backend-endpoints" json:"backend-endpoints"` + ListenAddr string `toml:"listen-addr" json:"listen-addr"` + AdvertiseListenAddr string `toml:"advertise-listen-addr" json:"advertise-listen-addr"` Name string `toml:"name" json:"name"` DataDir string `toml:"data-dir" json:"data-dir"` @@ -98,6 +102,31 @@ func NewConfig() *Config { return &Config{} } +// GetName returns the Name +func (c *Config) GetName() string { + return c.Name +} + +// GeBackendEndpoints returns the BackendEndpoints +func (c *Config) GeBackendEndpoints() string { + return c.BackendEndpoints +} + +// GetListenAddr returns the ListenAddr +func (c *Config) GetListenAddr() string { + return c.ListenAddr +} + +// GetAdvertiseListenAddr returns the AdvertiseListenAddr +func (c *Config) GetAdvertiseListenAddr() string { + return c.AdvertiseListenAddr +} + +// GetLeaderLease returns the leader lease. +func (c *Config) GetLeaderLease() int64 { + return c.LeaderLease +} + // IsLocalTSOEnabled returns if the local TSO is enabled. func (c *Config) IsLocalTSOEnabled() bool { return c.EnableLocalTSO @@ -113,6 +142,11 @@ func (c *Config) GetTSOSaveInterval() time.Duration { return c.TSOSaveInterval.Duration } +// GetMaxResetTSGap returns the MaxResetTSGap. +func (c *Config) GetMaxResetTSGap() time.Duration { + return c.MaxResetTSGap.Duration +} + // GetTLSConfig returns the TLS config. func (c *Config) GetTLSConfig() *grpcutil.TLSConfig { return &c.Security.TLSConfig @@ -133,14 +167,15 @@ func (c *Config) Parse(flagSet *pflag.FlagSet) error { } // Ignore the error check here - configutil.AdjustCommandlineString(flagSet, &c.Log.Level, "log-level") - configutil.AdjustCommandlineString(flagSet, &c.Log.File.Filename, "log-file") - configutil.AdjustCommandlineString(flagSet, &c.Metric.PushAddress, "metrics-addr") - configutil.AdjustCommandlineString(flagSet, &c.Security.CAPath, "cacert") - configutil.AdjustCommandlineString(flagSet, &c.Security.CertPath, "cert") - configutil.AdjustCommandlineString(flagSet, &c.Security.KeyPath, "key") - configutil.AdjustCommandlineString(flagSet, &c.BackendEndpoints, "backend-endpoints") - configutil.AdjustCommandlineString(flagSet, &c.ListenAddr, "listen-addr") + configutil.AdjustCommandLineString(flagSet, &c.Log.Level, "log-level") + configutil.AdjustCommandLineString(flagSet, &c.Log.File.Filename, "log-file") + configutil.AdjustCommandLineString(flagSet, &c.Metric.PushAddress, "metrics-addr") + configutil.AdjustCommandLineString(flagSet, &c.Security.CAPath, "cacert") + configutil.AdjustCommandLineString(flagSet, &c.Security.CertPath, "cert") + configutil.AdjustCommandLineString(flagSet, &c.Security.KeyPath, "key") + configutil.AdjustCommandLineString(flagSet, &c.BackendEndpoints, "backend-endpoints") + configutil.AdjustCommandLineString(flagSet, &c.ListenAddr, "listen-addr") + configutil.AdjustCommandLineString(flagSet, &c.AdvertiseListenAddr, "advertise-listen-addr") return c.Adjust(meta, false) } @@ -167,6 +202,7 @@ func (c *Config) Adjust(meta *toml.MetaData, reloading bool) error { configutil.AdjustString(&c.BackendEndpoints, defaultBackendEndpoints) configutil.AdjustString(&c.ListenAddr, defaultListenAddr) + configutil.AdjustString(&c.AdvertiseListenAddr, c.ListenAddr) configutil.AdjustDuration(&c.MaxResetTSGap, defaultMaxResetTSGap) configutil.AdjustInt64(&c.LeaderLease, utils.DefaultLeaderLease) diff --git a/pkg/mcs/tso/server/config_test.go b/pkg/mcs/tso/server/config_test.go new file mode 100644 index 00000000000..9f5bc298964 --- /dev/null +++ b/pkg/mcs/tso/server/config_test.go @@ -0,0 +1,99 @@ +// Copyright 2023 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "strings" + "testing" + "time" + + "github.com/BurntSushi/toml" + "github.com/stretchr/testify/require" + "github.com/tikv/pd/pkg/mcs/utils" +) + +func TestConfigBasic(t *testing.T) { + re := require.New(t) + + cfg := NewConfig() + cfg, err := GenerateConfig(cfg) + re.NoError(err) + + // Test default values. + re.True(strings.HasPrefix(cfg.GetName(), defaultName)) + re.Equal(defaultBackendEndpoints, cfg.BackendEndpoints) + re.Equal(defaultListenAddr, cfg.ListenAddr) + re.Equal(utils.DefaultLeaderLease, cfg.LeaderLease) + re.False(cfg.EnableLocalTSO) + re.True(cfg.EnableGRPCGateway) + re.Equal(defaultTSOSaveInterval, cfg.TSOSaveInterval.Duration) + re.Equal(defaultTSOUpdatePhysicalInterval, cfg.TSOUpdatePhysicalInterval.Duration) + re.Equal(defaultMaxResetTSGap, cfg.MaxResetTSGap.Duration) + + // Test setting values. + cfg.Name = "test-name" + cfg.BackendEndpoints = "test-endpoints" + cfg.ListenAddr = "test-listen-addr" + cfg.AdvertiseListenAddr = "test-advertise-listen-addr" + cfg.LeaderLease = 123 + cfg.EnableLocalTSO = true + cfg.TSOSaveInterval.Duration = time.Duration(10) * time.Second + cfg.TSOUpdatePhysicalInterval.Duration = time.Duration(100) * time.Millisecond + cfg.MaxResetTSGap.Duration = time.Duration(1) * time.Hour + + re.Equal("test-name", cfg.GetName()) + re.Equal("test-endpoints", cfg.GeBackendEndpoints()) + re.Equal("test-listen-addr", cfg.GetListenAddr()) + re.Equal("test-advertise-listen-addr", cfg.GetAdvertiseListenAddr()) + re.Equal(int64(123), cfg.GetLeaderLease()) + re.True(cfg.EnableLocalTSO) + re.Equal(time.Duration(10)*time.Second, cfg.TSOSaveInterval.Duration) + re.Equal(time.Duration(100)*time.Millisecond, cfg.TSOUpdatePhysicalInterval.Duration) + re.Equal(time.Duration(1)*time.Hour, cfg.MaxResetTSGap.Duration) +} + +func TestLoadFromConfig(t *testing.T) { + re := require.New(t) + cfgData := ` +backend-endpoints = "test-endpoints" +listen-addr = "test-listen-addr" +advertise-listen-addr = "test-advertise-listen-addr" +name = "tso-test-name" +data-dir = "/var/lib/tso" +enable-grpc-gateway = false +lease = 123 +enable-local-tso = true +tso-save-interval = "10s" +tso-update-physical-interval = "100ms" +max-gap-reset-ts = "1h" +` + + cfg := NewConfig() + meta, err := toml.Decode(cfgData, &cfg) + re.NoError(err) + err = cfg.Adjust(&meta, false) + re.NoError(err) + + re.Equal("tso-test-name", cfg.GetName()) + re.Equal("test-endpoints", cfg.GeBackendEndpoints()) + re.Equal("test-listen-addr", cfg.GetListenAddr()) + re.Equal("test-advertise-listen-addr", cfg.GetAdvertiseListenAddr()) + re.Equal("/var/lib/tso", cfg.DataDir) + re.Equal(int64(123), cfg.GetLeaderLease()) + re.True(cfg.EnableLocalTSO) + re.Equal(time.Duration(10)*time.Second, cfg.TSOSaveInterval.Duration) + re.Equal(time.Duration(100)*time.Millisecond, cfg.TSOUpdatePhysicalInterval.Duration) + re.Equal(time.Duration(1)*time.Hour, cfg.MaxResetTSGap.Duration) +} diff --git a/pkg/mcs/tso/server/grpc_service.go b/pkg/mcs/tso/server/grpc_service.go index d8c399b2960..dd0a96b1cba 100644 --- a/pkg/mcs/tso/server/grpc_service.go +++ b/pkg/mcs/tso/server/grpc_service.go @@ -18,32 +18,26 @@ import ( "context" "io" "net/http" + "strings" "time" - "github.com/pingcap/kvproto/pkg/pdpb" "github.com/pingcap/kvproto/pkg/tsopb" "github.com/pingcap/log" "github.com/pkg/errors" bs "github.com/tikv/pd/pkg/basicserver" - "github.com/tikv/pd/pkg/errs" "github.com/tikv/pd/pkg/mcs/registry" "github.com/tikv/pd/pkg/utils/apiutil" "github.com/tikv/pd/pkg/utils/grpcutil" - "go.uber.org/zap" + "github.com/tikv/pd/pkg/utils/tsoutil" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) -const ( - // tso - maxMergeTSORequests = 10000 - defaultTSOProxyTimeout = 3 * time.Second -) - // gRPC errors var ( - ErrNotStarted = status.Errorf(codes.Unavailable, "server not started") + ErrNotStarted = status.Errorf(codes.Unavailable, "server not started") + ErrClusterMismatched = status.Errorf(codes.Unavailable, "cluster mismatched") ) var _ tsopb.TSOServer = (*Service)(nil) @@ -115,16 +109,20 @@ func (s *Service) Tso(stream tsopb.TSO_TsoServer) error { streamCtx := stream.Context() forwardedHost := grpcutil.GetForwardedHost(streamCtx) if !s.IsLocalRequest(forwardedHost) { + clientConn, err := s.GetDelegateClient(s.ctx, forwardedHost) + if err != nil { + return errors.WithStack(err) + } + if errCh == nil { doneCh = make(chan struct{}) defer close(doneCh) errCh = make(chan error) } - s.dispatchTSORequest(ctx, &tsoRequest{ - forwardedHost, - request, - stream, - }, forwardedHost, doneCh, errCh) + + tsoProtoFactory := s.tsoProtoFactory + tsoRequest := tsoutil.NewTSOProtoRequest(forwardedHost, clientConn, request, stream) + s.tsoDispatcher.DispatchRequest(ctx, tsoRequest, tsoProtoFactory, doneCh, errCh) continue } @@ -134,16 +132,19 @@ func (s *Service) Tso(stream tsopb.TSO_TsoServer) error { return status.Errorf(codes.Unknown, "server not started") } if request.GetHeader().GetClusterId() != s.clusterID { - return status.Errorf(codes.FailedPrecondition, "mismatch cluster id, need %d but got %d", s.clusterID, request.GetHeader().GetClusterId()) + return status.Errorf( + codes.FailedPrecondition, "mismatch cluster id, need %d but got %d", + s.clusterID, request.GetHeader().GetClusterId()) } count := request.GetCount() - ts, err := s.tsoAllocatorManager.HandleTSORequest(request.GetDcLocation(), count) + ts, keyspaceGroupBelongTo, err := s.keyspaceGroupManager.HandleTSORequest( + request.Header.KeyspaceId, request.Header.KeyspaceGroupId, request.GetDcLocation(), count) if err != nil { return status.Errorf(codes.Unknown, err.Error()) } tsoHandleDuration.Observe(time.Since(start).Seconds()) response := &tsopb.TsoResponse{ - Header: s.header(), + Header: s.header(keyspaceGroupBelongTo), Timestamp: &ts, Count: count, } @@ -153,199 +154,118 @@ func (s *Service) Tso(stream tsopb.TSO_TsoServer) error { } } -func (s *Service) header() *tsopb.ResponseHeader { - if s.clusterID == 0 { - return s.wrapErrorToHeader(tsopb.ErrorType_NOT_BOOTSTRAPPED, "cluster id is not ready") +// FindGroupByKeyspaceID returns the keyspace group that the keyspace belongs to. +func (s *Service) FindGroupByKeyspaceID( + ctx context.Context, request *tsopb.FindGroupByKeyspaceIDRequest, +) (*tsopb.FindGroupByKeyspaceIDResponse, error) { + respKeyspaceGroup := request.GetHeader().GetKeyspaceGroupId() + if errorType, err := s.validRequest(request.GetHeader()); err != nil { + return &tsopb.FindGroupByKeyspaceIDResponse{ + Header: s.wrapErrorToHeader(errorType, err.Error(), respKeyspaceGroup), + }, nil } - return &tsopb.ResponseHeader{ClusterId: s.clusterID} -} - -func (s *Service) wrapErrorToHeader(errorType tsopb.ErrorType, message string) *tsopb.ResponseHeader { - return s.errorHeader(&tsopb.Error{ - Type: errorType, - Message: message, - }) -} -func (s *Service) errorHeader(err *tsopb.Error) *tsopb.ResponseHeader { - return &tsopb.ResponseHeader{ - ClusterId: s.clusterID, - Error: err, + keyspaceID := request.GetKeyspaceId() + am, keyspaceGroup, keyspaceGroupID, err := s.keyspaceGroupManager.FindGroupByKeyspaceID(keyspaceID) + if err != nil { + return &tsopb.FindGroupByKeyspaceIDResponse{ + Header: s.wrapErrorToHeader(tsopb.ErrorType_UNKNOWN, err.Error(), keyspaceGroupID), + }, nil } -} - -type tsoRequest struct { - forwardedHost string - request *tsopb.TsoRequest - stream tsopb.TSO_TsoServer -} - -func (s *Service) dispatchTSORequest(ctx context.Context, request *tsoRequest, forwardedHost string, doneCh <-chan struct{}, errCh chan<- error) { - tsoRequestChInterface, loaded := s.tsoDispatcher.LoadOrStore(forwardedHost, make(chan *tsoRequest, maxMergeTSORequests)) - if !loaded { - tsDeadlineCh := make(chan deadline, 1) - go s.handleDispatcher(ctx, forwardedHost, tsoRequestChInterface.(chan *tsoRequest), tsDeadlineCh, doneCh, errCh) - go watchTSDeadline(ctx, tsDeadlineCh) + if keyspaceGroup == nil { + return &tsopb.FindGroupByKeyspaceIDResponse{ + Header: s.wrapErrorToHeader( + tsopb.ErrorType_UNKNOWN, "keyspace group not found", keyspaceGroupID), + }, nil } - tsoRequestChInterface.(chan *tsoRequest) <- request -} - -func (s *Service) handleDispatcher(ctx context.Context, forwardedHost string, tsoRequestCh <-chan *tsoRequest, tsDeadlineCh chan<- deadline, doneCh <-chan struct{}, errCh chan<- error) { - dispatcherCtx, ctxCancel := context.WithCancel(ctx) - defer ctxCancel() - defer s.tsoDispatcher.Delete(forwardedHost) - var ( - forwardStream tsopb.TSO_TsoClient - cancel context.CancelFunc - ) - client, err := s.GetDelegateClient(ctx, forwardedHost) - if err != nil { - goto errHandling - } - log.Info("create tso forward stream", zap.String("forwarded-host", forwardedHost)) - forwardStream, cancel, err = s.CreateTsoForwardStream(client) -errHandling: - if err != nil || forwardStream == nil { - log.Error("create tso forwarding stream error", zap.String("forwarded-host", forwardedHost), errs.ZapError(errs.ErrGRPCCreateStream, err)) - select { - case <-dispatcherCtx.Done(): - return - case _, ok := <-doneCh: - if !ok { - return - } - case errCh <- err: - close(errCh) - return - } + members := make([]*tsopb.KeyspaceGroupMember, 0, len(keyspaceGroup.Members)) + for _, member := range keyspaceGroup.Members { + members = append(members, &tsopb.KeyspaceGroupMember{ + Address: member.Address, + // TODO: watch the keyspace groups' primary serving address changes + // to get the latest primary serving addresses of all keyspace groups. + IsPrimary: strings.EqualFold(member.Address, am.GetLeaderAddr()), + }) } - defer cancel() - requests := make([]*tsoRequest, maxMergeTSORequests+1) - for { - select { - case first := <-tsoRequestCh: - pendingTSOReqCount := len(tsoRequestCh) + 1 - requests[0] = first - for i := 1; i < pendingTSOReqCount; i++ { - requests[i] = <-tsoRequestCh - } - done := make(chan struct{}) - dl := deadline{ - timer: time.After(defaultTSOProxyTimeout), - done: done, - cancel: cancel, - } - select { - case tsDeadlineCh <- dl: - case <-dispatcherCtx.Done(): - return - } - err = s.processTSORequests(forwardStream, requests[:pendingTSOReqCount]) - close(done) - if err != nil { - log.Error("proxy forward tso error", zap.String("forwarded-host", forwardedHost), errs.ZapError(errs.ErrGRPCSend, err)) - select { - case <-dispatcherCtx.Done(): - return - case _, ok := <-doneCh: - if !ok { - return - } - case errCh <- err: - close(errCh) - return - } - } - case <-dispatcherCtx.Done(): - return + var splitState *tsopb.SplitState + if keyspaceGroup.SplitState != nil { + splitState = &tsopb.SplitState{ + SplitSource: keyspaceGroup.SplitState.SplitSource, } } + + return &tsopb.FindGroupByKeyspaceIDResponse{ + Header: s.header(keyspaceGroupID), + KeyspaceGroup: &tsopb.KeyspaceGroup{ + Id: keyspaceGroupID, + UserKind: keyspaceGroup.UserKind, + SplitState: splitState, + Members: members, + }, + }, nil } -func (s *Service) processTSORequests(forwardStream tsopb.TSO_TsoClient, requests []*tsoRequest) error { - start := time.Now() - // Merge the requests - count := uint32(0) - for _, request := range requests { - count += request.request.GetCount() - } - req := &tsopb.TsoRequest{ - Header: requests[0].request.GetHeader(), - Count: count, - // TODO: support Local TSO proxy forwarding. - DcLocation: requests[0].request.GetDcLocation(), +// GetMinTS gets the minimum timestamp across all keyspace groups served by the TSO server +// who receives and handles the request. +func (s *Service) GetMinTS( + ctx context.Context, request *tsopb.GetMinTSRequest, +) (*tsopb.GetMinTSResponse, error) { + respKeyspaceGroup := request.GetHeader().GetKeyspaceGroupId() + if errorType, err := s.validRequest(request.GetHeader()); err != nil { + return &tsopb.GetMinTSResponse{ + Header: s.wrapErrorToHeader(errorType, err.Error(), respKeyspaceGroup), + }, nil } - // Send to the leader stream. - if err := forwardStream.Send(req); err != nil { - return err - } - resp, err := forwardStream.Recv() + + minTS, kgAskedCount, kgTotalCount, err := s.keyspaceGroupManager.GetMinTS(request.GetDcLocation()) if err != nil { - return err + return &tsopb.GetMinTSResponse{ + Header: s.wrapErrorToHeader( + tsopb.ErrorType_UNKNOWN, err.Error(), respKeyspaceGroup), + Timestamp: &minTS, + KeyspaceGroupsServing: kgAskedCount, + KeyspaceGroupsTotal: kgTotalCount, + }, nil } - tsoProxyHandleDuration.Observe(time.Since(start).Seconds()) - tsoProxyBatchSize.Observe(float64(count)) - // Split the response - physical, logical, suffixBits := resp.GetTimestamp().GetPhysical(), resp.GetTimestamp().GetLogical(), resp.GetTimestamp().GetSuffixBits() - // `logical` is the largest ts's logical part here, we need to do the subtracting before we finish each TSO request. - // This is different from the logic of client batch, for example, if we have a largest ts whose logical part is 10, - // count is 5, then the splitting results should be 5 and 10. - firstLogical := addLogical(logical, -int64(count), suffixBits) - return s.finishTSORequest(requests, physical, firstLogical, suffixBits) + + return &tsopb.GetMinTSResponse{ + Header: s.header(respKeyspaceGroup), + Timestamp: &minTS, + KeyspaceGroupsServing: kgAskedCount, + KeyspaceGroupsTotal: kgTotalCount, + }, nil } -// Because of the suffix, we need to shift the count before we add it to the logical part. -func addLogical(logical, count int64, suffixBits uint32) int64 { - return logical + count</tso//primary" in which - // is 5 digits integer with leading zeros. For now we use 0 as the default cluster id. - tsoPrimaryPrefix = "/ms/0/tso" -) - var _ bs.Server = (*Server)(nil) -var _ tso.Member = (*member.Participant)(nil) +var _ tso.ElectionMember = (*member.Participant)(nil) // Server is the TSO server, and it implements bs.Server. type Server struct { diagnosticspb.DiagnosticsServer - // Server state. 0 is not serving, 1 is serving. - isServing int64 + // Server state. 0 is not running, 1 is running. + isRunning int64 // Server start timestamp startTimestamp int64 @@ -91,34 +79,39 @@ type Server struct { handler *Handler - cfg *Config - clusterID uint64 - defaultGroupRootPath string - defaultGroupStorage endpoint.TSOStorage - listenURL *url.URL - backendUrls []url.URL + cfg *Config + clusterID uint64 + listenURL *url.URL + backendUrls []url.URL - // for the primary election in the TSO cluster - participant *member.Participant // etcd client etcdClient *clientv3.Client // http client httpClient *http.Client - muxListener net.Listener - service *Service - tsoAllocatorManager *tso.AllocatorManager + secure bool + muxListener net.Listener + httpListener net.Listener + grpcServer *grpc.Server + httpServer *http.Server + service *Service + keyspaceGroupManager *tso.KeyspaceGroupManager // Store as map[string]*grpc.ClientConn clientConns sync.Map - // Store as map[string]chan *tsoRequest - tsoDispatcher sync.Map + // tsoDispatcher is used to dispatch the TSO requests to + // the corresponding forwarding TSO channels. + tsoDispatcher *tsoutil.TSODispatcher + // tsoProtoFactory is the abstract factory for creating tso + // related data structures defined in the tso grpc protocol + tsoProtoFactory *tsoutil.TSOProtoFactory // Callback functions for different stages // startCallbacks will be called after the server is started. startCallbacks []func() - // primaryCallbacks will be called after the server becomes the primary. - primaryCallbacks []func(context.Context) - serviceRegister *discovery.ServiceRegister + + // for service registry + serviceID *discovery.ServiceRegistryEntry + serviceRegister *discovery.ServiceRegister } // Implement the following methods defined in bs.Server @@ -143,153 +136,56 @@ func (s *Server) GetBasicServer() bs.Server { return s } -// Run runs the TSO server. -func (s *Server) Run() error { - go systimemon.StartMonitor(s.ctx, time.Now, func() { - log.Error("system time jumps backward", errs.ZapError(errs.ErrIncorrectSystemTime)) - timeJumpBackCounter.Inc() - }) - - if err := s.initClient(); err != nil { - return err - } - if err := s.startServer(); err != nil { - return err - } - - s.startServerLoop() - - return nil -} - -func (s *Server) startServerLoop() { - s.serverLoopCtx, s.serverLoopCancel = context.WithCancel(s.ctx) - s.serverLoopWg.Add(2) - go s.primaryElectionLoop() - go s.tsoAllocatorLoop() +// GetAddr returns the address of the server. +func (s *Server) GetAddr() string { + return s.cfg.ListenAddr } -// tsoAllocatorLoop is used to run the TSO Allocator updating daemon. -func (s *Server) tsoAllocatorLoop() { - defer logutil.LogPanic() - defer s.serverLoopWg.Done() - - ctx, cancel := context.WithCancel(s.serverLoopCtx) - defer cancel() - s.tsoAllocatorManager.AllocatorDaemon(ctx) - log.Info("tso server is closed, exit allocator loop") +// GetBackendEndpoints returns the backend endpoints. +func (s *Server) GetBackendEndpoints() string { + return s.cfg.BackendEndpoints } -func (s *Server) primaryElectionLoop() { - defer logutil.LogPanic() - defer s.serverLoopWg.Done() - - for { - if s.IsClosed() { - log.Info("server is closed, exit tso primary election loop") - return - } - - primary, rev, checkAgain := s.participant.CheckLeader() - if checkAgain { - continue - } - if primary != nil { - // TODO: if enable-local-tso is true, check the cluster dc-location after the primary/leader is elected - // go s.tsoAllocatorManager.ClusterDCLocationChecker() - - log.Info("start to watch the primary/leader", zap.Stringer("tso-primary", primary)) - // WatchLeader will keep looping and never return unless the primary/leader has changed. - s.participant.WatchLeader(s.serverLoopCtx, primary, rev) - log.Info("the tso primary/leader has changed, try to re-campaign a primary/leader") - } - - s.campaignLeader() - } +// GetClientConns returns the client connections. +func (s *Server) GetClientConns() *sync.Map { + return &s.clientConns } -func (s *Server) campaignLeader() { - log.Info("start to campaign the primary/leader", zap.String("campaign-tso-primary-name", s.participant.Member().Name)) - if err := s.participant.CampaignLeader(s.cfg.LeaderLease); err != nil { - if err.Error() == errs.ErrEtcdTxnConflict.Error() { - log.Info("campaign tso primary/leader meets error due to txn conflict, another tso server may campaign successfully", - zap.String("campaign-tso-primary-name", s.participant.Member().Name)) - } else { - log.Error("campaign tso primary/leader meets error due to etcd error", - zap.String("campaign-tso-primary-name", s.participant.Member().Name), - errs.ZapError(err)) +// Run runs the TSO server. +func (s *Server) Run() error { + skipWaitAPIServiceReady := false + failpoint.Inject("skipWaitAPIServiceReady", func() { + skipWaitAPIServiceReady = true + }) + if !skipWaitAPIServiceReady { + if err := utils.WaitAPIServiceReady(s); err != nil { + return err } - return } - - // Start keepalive the leadership and enable TSO service. - // TSO service is strictly enabled/disabled by the leader lease for 2 reasons: - // 1. lease based approach is not affected by thread pause, slow runtime schedule, etc. - // 2. load region could be slow. Based on lease we can recover TSO service faster. - ctx, cancel := context.WithCancel(s.serverLoopCtx) - var resetLeaderOnce sync.Once - defer resetLeaderOnce.Do(func() { - cancel() - s.participant.ResetLeader() + go systimemon.StartMonitor(s.ctx, time.Now, func() { + log.Error("system time jumps backward", errs.ZapError(errs.ErrIncorrectSystemTime)) + timeJumpBackCounter.Inc() }) - // maintain the the leadership, after this, TSO can be service. - s.participant.KeepLeader(ctx) - log.Info("campaign tso primary ok", zap.String("campaign-tso-primary-name", s.participant.Member().Name)) - - allocator, err := s.tsoAllocatorManager.GetAllocator(tso.GlobalDCLocation) - if err != nil { - log.Error("failed to get the global tso allocator", errs.ZapError(err)) - return - } - log.Info("initializing the global tso allocator") - if err := allocator.Initialize(0); err != nil { - log.Error("failed to initialize the global tso allocator", errs.ZapError(err)) - return - } - defer func() { - s.tsoAllocatorManager.ResetAllocatorGroup(tso.GlobalDCLocation) - }() - - log.Info("triggering the primary callback functions") - for _, cb := range s.primaryCallbacks { - cb(ctx) - } - - s.participant.EnableLeader() - // TODO: if enable-local-tso is true, check the cluster dc-location after the primary/leader is elected - // go s.tsoAllocatorManager.ClusterDCLocationChecker() - log.Info("tso primary is ready to serve", zap.String("tso-primary-name", s.participant.Member().Name)) - - leaderTicker := time.NewTicker(utils.LeaderTickInterval) - defer leaderTicker.Stop() - - for { - select { - case <-leaderTicker.C: - if !s.participant.IsLeader() { - log.Info("no longer a primary/leader because lease has expired, the tso primary/leader will step down") - return - } - case <-ctx.Done(): - // Server is closed and it should return nil. - log.Info("server is closed") - return - } + if err := s.initClient(); err != nil { + return err } + return s.startServer() } // Close closes the server. func (s *Server) Close() { - if !atomic.CompareAndSwapInt64(&s.isServing, 1, 0) { + if !atomic.CompareAndSwapInt64(&s.isRunning, 1, 0) { // server is already closed return } log.Info("closing tso server ...") + // close tso service loops in the keyspace group manager + s.keyspaceGroupManager.Close() s.serviceRegister.Deregister() - // TODO: double check when muxListener is closed, grpc.Server.serve() and http.Server.serve() - // will also close with error cmux.ErrListenerClosed. + s.stopHTTPServer() + s.stopGRPCServer() s.muxListener.Close() s.serverLoopCancel() s.serverLoopWg.Wait() @@ -324,17 +220,62 @@ func (s *Server) AddStartCallback(callbacks ...func()) { // IsServing implements basicserver. It returns whether the server is the leader // if there is embedded etcd, or the primary otherwise. func (s *Server) IsServing() bool { - return s.participant.IsLeader() && atomic.LoadInt64(&s.isServing) == 1 + return s.IsKeyspaceServing(utils.DefaultKeyspaceID, utils.DefaultKeyspaceGroupID) } -// GetPrimary returns the primary provider of this tso server. -func (s *Server) GetPrimary() bs.MemberProvider { - return s.participant.GetLeader() +// IsKeyspaceServing returns whether the server is the primary of the given keyspace. +// TODO: update basicserver interface to support keyspace. +func (s *Server) IsKeyspaceServing(keyspaceID, keyspaceGroupID uint32) bool { + if atomic.LoadInt64(&s.isRunning) == 0 { + return false + } + + member, err := s.keyspaceGroupManager.GetElectionMember( + keyspaceID, keyspaceGroupID) + if err != nil { + log.Error("failed to get election member", errs.ZapError(err)) + return false + } + return member.IsLeader() } -// AddServiceReadyCallback implements basicserver. It adds callbacks when the server becomes the primary. +// GetLeaderListenUrls gets service endpoints from the leader in election group. +// The entry at the index 0 is the primary's service endpoint. +func (s *Server) GetLeaderListenUrls() []string { + member, err := s.keyspaceGroupManager.GetElectionMember( + utils.DefaultKeyspaceID, utils.DefaultKeyspaceGroupID) + if err != nil { + log.Error("failed to get election member", errs.ZapError(err)) + return nil + } + + return member.GetLeaderListenUrls() +} + +// GetMember returns the election member of the given keyspace and keyspace group. +func (s *Server) GetMember(keyspaceID, keyspaceGroupID uint32) (tso.ElectionMember, error) { + member, err := s.keyspaceGroupManager.GetElectionMember(keyspaceID, keyspaceGroupID) + if err != nil { + return nil, err + } + return member, nil +} + +// ResignPrimary resigns the primary of the given keyspace. +func (s *Server) ResignPrimary(keyspaceID, keyspaceGroupID uint32) error { + member, err := s.keyspaceGroupManager.GetElectionMember(keyspaceID, keyspaceGroupID) + if err != nil { + return err + } + member.ResetLeader() + return nil +} + +// AddServiceReadyCallback implements basicserver. +// It adds callbacks when it's ready for providing tso service. func (s *Server) AddServiceReadyCallback(callbacks ...func(context.Context)) { - s.primaryCallbacks = append(s.primaryCallbacks, callbacks...) + // Do nothing here. The primary of each keyspace group assigned to this host + // will respond to the requests accordingly. } // Implement the other methods @@ -346,17 +287,17 @@ func (s *Server) ClusterID() uint64 { // IsClosed checks if the server loop is closed func (s *Server) IsClosed() bool { - return atomic.LoadInt64(&s.isServing) == 0 + return atomic.LoadInt64(&s.isRunning) == 0 } -// GetTSOAllocatorManager returns the manager of TSO Allocator. -func (s *Server) GetTSOAllocatorManager() *tso.AllocatorManager { - return s.tsoAllocatorManager +// GetKeyspaceGroupManager returns the manager of keyspace group. +func (s *Server) GetKeyspaceGroupManager() *tso.KeyspaceGroupManager { + return s.keyspaceGroupManager } -// GetTSODispatcher gets the TSO Dispatcher -func (s *Server) GetTSODispatcher() *sync.Map { - return &s.tsoDispatcher +// GetTSOAllocatorManager returns the manager of TSO Allocator. +func (s *Server) GetTSOAllocatorManager(keyspaceGroupID uint32) (*tso.AllocatorManager, error) { + return s.keyspaceGroupManager.GetAllocatorManager(keyspaceGroupID) } // IsLocalRequest checks if the forwarded host is the current host @@ -368,16 +309,6 @@ func (s *Server) IsLocalRequest(forwardedHost string) bool { return forwardedHost == "" } -// CreateTsoForwardStream creates the forward stream -func (s *Server) CreateTsoForwardStream(client *grpc.ClientConn) (tsopb.TSO_TsoClient, context.CancelFunc, error) { - done := make(chan struct{}) - ctx, cancel := context.WithCancel(s.ctx) - go checkStream(ctx, cancel, done) - forwardStream, err := tsopb.NewTSOClient(client).Tso(ctx) - done <- struct{}{} - return forwardStream, cancel, err -} - // GetDelegateClient returns grpc client connection talking to the forwarded host func (s *Server) GetDelegateClient(ctx context.Context, forwardedHost string) (*grpc.ClientConn, error) { client, ok := s.clientConns.Load(forwardedHost) @@ -418,15 +349,6 @@ func (s *Server) ValidateRequest(header *tsopb.RequestHeader) error { return nil } -// GetGlobalTS returns global tso. -func (s *Server) GetGlobalTS() (uint64, error) { - ts, err := s.tsoAllocatorManager.GetGlobalTSO() - if err != nil { - return 0, err - } - return tsoutil.GenerateTS(ts), nil -} - // GetExternalTS returns external timestamp from the cache or the persistent storage. // TODO: Implement GetExternalTS func (s *Server) GetExternalTS() uint64 { @@ -439,22 +361,6 @@ func (s *Server) SetExternalTS(externalTS uint64) error { return nil } -func checkStream(streamCtx context.Context, cancel context.CancelFunc, done chan struct{}) { - select { - case <-done: - return - case <-time.After(3 * time.Second): - cancel() - case <-streamCtx.Done(): - } - <-done -} - -// GetListenURL gets the listen URL. -func (s *Server) GetListenURL() *url.URL { - return s.listenURL -} - // GetConfig gets the config. func (s *Server) GetConfig() *Config { return s.cfg @@ -479,84 +385,133 @@ func (s *Server) initClient() error { } func (s *Server) startGRPCServer(l net.Listener) { + defer logutil.LogPanic() defer s.serverLoopWg.Done() - gs := grpc.NewServer() - s.service.RegisterGRPCService(gs) - diagnosticspb.RegisterDiagnosticsServer(gs, s) - serverr := gs.Serve(l) - log.Info("grpc server stopped serving") - - // Attempt graceful stop (waits for pending RPCs), but force a stop if - // it doesn't happen in a reasonable amount of time. - done := make(chan struct{}) - go func() { - log.Info("try to gracefully stop the server now") - gs.GracefulStop() - close(done) - }() - select { - case <-done: - case <-time.After(utils.DefaultGRPCGracefulStopTimeout): - log.Info("stopping grpc gracefully is taking longer than expected and force stopping now") - gs.Stop() - } - + log.Info("grpc server starts serving", zap.String("address", l.Addr().String())) + err := s.grpcServer.Serve(l) if s.IsClosed() { log.Info("grpc server stopped") } else { - log.Fatal("grpc server stopped unexpectedly", errs.ZapError(serverr)) + log.Fatal("grpc server stopped unexpectedly", errs.ZapError(err)) } } func (s *Server) startHTTPServer(l net.Listener) { + defer logutil.LogPanic() defer s.serverLoopWg.Done() - handler, _ := SetUpRestHandler(s.service) - hs := &http.Server{ - Handler: handler, - ReadTimeout: 5 * time.Minute, - ReadHeaderTimeout: 5 * time.Second, - } - serverr := hs.Serve(l) - log.Info("http server stopped serving") - - ctx, cancel := context.WithTimeout(context.Background(), utils.DefaultHTTPGracefulShutdownTimeout) - defer cancel() - if err := hs.Shutdown(ctx); err != nil { - log.Error("http server shutdown encountered problem", errs.ZapError(err)) - } else { - log.Info("all http(s) requests finished") - } + log.Info("http server starts serving", zap.String("address", l.Addr().String())) + err := s.httpServer.Serve(l) if s.IsClosed() { log.Info("http server stopped") } else { - log.Fatal("http server stopped unexpectedly", errs.ZapError(serverr)) + log.Fatal("http server stopped unexpectedly", errs.ZapError(err)) } } -func (s *Server) startGRPCAndHTTPServers(l net.Listener) { +func (s *Server) startGRPCAndHTTPServers(serverReadyChan chan<- struct{}, l net.Listener) { + defer logutil.LogPanic() defer s.serverLoopWg.Done() mux := cmux.New(l) + // Don't hang on matcher after closing listener + mux.SetReadTimeout(3 * time.Second) grpcL := mux.MatchWithWriters(cmux.HTTP2MatchHeaderFieldSendSettings("content-type", "application/grpc")) - httpL := mux.Match(cmux.Any()) + if s.secure { + s.httpListener = mux.Match(cmux.Any()) + } else { + s.httpListener = mux.Match(cmux.HTTP1()) + } - s.serverLoopWg.Add(2) + s.grpcServer = grpc.NewServer() + s.service.RegisterGRPCService(s.grpcServer) + diagnosticspb.RegisterDiagnosticsServer(s.grpcServer, s) + s.serverLoopWg.Add(1) go s.startGRPCServer(grpcL) - go s.startHTTPServer(httpL) + handler, _ := SetUpRestHandler(s.service) + s.httpServer = &http.Server{ + Handler: handler, + ReadTimeout: 3 * time.Second, + } + s.serverLoopWg.Add(1) + go s.startHTTPServer(s.httpListener) + + serverReadyChan <- struct{}{} if err := mux.Serve(); err != nil { if s.IsClosed() { - log.Info("mux stop serving", errs.ZapError(err)) + log.Info("mux stopped serving", errs.ZapError(err)) } else { - log.Panic("mux stop serving unexpectedly", errs.ZapError(err)) + log.Fatal("mux stopped serving unexpectedly", errs.ZapError(err)) } } } +func (s *Server) stopHTTPServer() { + log.Info("stopping http server") + defer log.Info("http server stopped") + + ctx, cancel := context.WithTimeout(context.Background(), utils.DefaultHTTPGracefulShutdownTimeout) + defer cancel() + + // First, try to gracefully shutdown the http server + ch := make(chan struct{}) + go func() { + defer close(ch) + s.httpServer.Shutdown(ctx) + }() + + select { + case <-ch: + case <-ctx.Done(): + // Took too long, manually close open transports + log.Warn("http server graceful shutdown timeout, forcing close") + s.httpServer.Close() + // concurrent Graceful Shutdown should be interrupted + <-ch + } +} + +func (s *Server) stopGRPCServer() { + log.Info("stopping grpc server") + defer log.Info("grpc server stopped") + + // Do not grpc.Server.GracefulStop with TLS enabled etcd server + // See https://github.com/grpc/grpc-go/issues/1384#issuecomment-317124531 + // and https://github.com/etcd-io/etcd/issues/8916 + if s.secure { + s.grpcServer.Stop() + return + } + + ctx, cancel := context.WithTimeout(context.Background(), utils.DefaultGRPCGracefulStopTimeout) + defer cancel() + + // First, try to gracefully shutdown the grpc server + ch := make(chan struct{}) + go func() { + defer close(ch) + // Close listeners to stop accepting new connections, + // will block on any existing transports + s.grpcServer.GracefulStop() + }() + + // Wait until all pending RPCs are finished + select { + case <-ch: + case <-ctx.Done(): + // Took too long, manually close open transports + // e.g. watch streams + log.Warn("grpc server graceful shutdown timeout, forcing close") + s.grpcServer.Stop() + // concurrent GracefulStop should be interrupted + <-ch + } +} + func (s *Server) startServer() (err error) { - if s.clusterID, err = etcdutil.GetClusterID(s.etcdClient, utils.ClusterIDPath); err != nil { + if s.clusterID, err = utils.InitClusterID(s.ctx, s.etcdClient); err != nil { return err } log.Info("init cluster id", zap.Uint64("cluster-id", s.clusterID)) @@ -566,38 +521,33 @@ func (s *Server) startServer() (err error) { // The independent TSO service still reuses PD version info since PD and TSO are just // different service modes provided by the same pd-server binary serverInfo.WithLabelValues(versioninfo.PDReleaseVersion, versioninfo.PDGitHash).Set(float64(time.Now().Unix())) - s.defaultGroupRootPath = path.Join(pdRootPath, strconv.FormatUint(s.clusterID, 10)) - - // TODO: Figure out how we should generated the unique id and name passed to Participant. - // For now, set the name to be listen address and generate the unique id from the name with sha256. - uniqueName := s.cfg.ListenAddr - uniqueID := memberutil.GenerateUniqueID(uniqueName) - log.Info("joining primary election", zap.String("participant-name", uniqueName), zap.Uint64("participant-id", uniqueID)) - - s.participant = member.NewParticipant(s.etcdClient, uniqueID) - s.participant.InitInfo(uniqueName, path.Join(tsoPrimaryPrefix, fmt.Sprintf("%05d", 0)), "primary", "keyspace group primary election", s.cfg.ListenAddr) - s.participant.SetMemberDeployPath(s.participant.ID()) - s.participant.SetMemberBinaryVersion(s.participant.ID(), versioninfo.PDReleaseVersion) - s.participant.SetMemberGitHash(s.participant.ID(), versioninfo.PDGitHash) - - s.defaultGroupStorage = endpoint.NewStorageEndpoint(kv.NewEtcdKVBase(s.GetClient(), s.defaultGroupRootPath), nil) - s.tsoAllocatorManager = tso.NewAllocatorManager( - s.participant, s.defaultGroupRootPath, s.defaultGroupStorage, s.cfg.IsLocalTSOEnabled(), s.cfg.GetTSOSaveInterval(), s.cfg.GetTSOUpdatePhysicalInterval(), - s.cfg.GetTLSConfig(), func() time.Duration { return s.cfg.MaxResetTSGap.Duration }) - // Set up the Global TSO Allocator here, it will be initialized once this TSO participant campaigns leader successfully. - s.tsoAllocatorManager.SetUpAllocator(s.ctx, tso.GlobalDCLocation, s.participant.GetLeadership()) - s.service = &Service{Server: s} - - tlsConfig, err := s.cfg.Security.ToTLSConfig() + s.listenURL, err = url.Parse(s.cfg.ListenAddr) if err != nil { return err } - s.listenURL, err = url.Parse(s.cfg.ListenAddr) + + // Initialize the TSO service. + s.serverLoopCtx, s.serverLoopCancel = context.WithCancel(s.ctx) + legacySvcRootPath := endpoint.LegacyRootPath(s.clusterID) + tsoSvcRootPath := endpoint.TSOSvcRootPath(s.clusterID) + s.serviceID = &discovery.ServiceRegistryEntry{ServiceAddr: s.cfg.AdvertiseListenAddr} + s.keyspaceGroupManager = tso.NewKeyspaceGroupManager( + s.serverLoopCtx, s.serviceID, s.etcdClient, s.httpClient, s.cfg.AdvertiseListenAddr, + discovery.TSOPath(s.clusterID), legacySvcRootPath, tsoSvcRootPath, s.cfg) + if err := s.keyspaceGroupManager.Initialize(); err != nil { + return err + } + + s.tsoProtoFactory = &tsoutil.TSOProtoFactory{} + s.service = &Service{Server: s} + + tlsConfig, err := s.cfg.Security.ToTLSConfig() if err != nil { return err } if tlsConfig != nil { + s.secure = true s.muxListener, err = tls.Listen(utils.TCPNetworkStr, s.listenURL.Host, tlsConfig) } else { s.muxListener, err = net.Listen(utils.TCPNetworkStr, s.listenURL.Host) @@ -606,8 +556,11 @@ func (s *Server) startServer() (err error) { return err } + serverReadyChan := make(chan struct{}) + defer close(serverReadyChan) s.serverLoopWg.Add(1) - go s.startGRPCAndHTTPServers(s.muxListener) + go s.startGRPCAndHTTPServers(serverReadyChan, s.muxListener) + <-serverReadyChan // Run callbacks log.Info("triggering the start callback functions") @@ -616,15 +569,23 @@ func (s *Server) startServer() (err error) { } // Server has started. - atomic.StoreInt64(&s.isServing, 1) - s.serviceRegister = discovery.NewServiceRegister(s.ctx, s.etcdClient, "tso", s.cfg.ListenAddr, s.cfg.ListenAddr, discovery.DefaultLeaseInSeconds) - s.serviceRegister.Register() + serializedEntry, err := s.serviceID.Serialize() + if err != nil { + return err + } + s.serviceRegister = discovery.NewServiceRegister(s.ctx, s.etcdClient, strconv.FormatUint(s.clusterID, 10), + utils.TSOServiceName, s.cfg.AdvertiseListenAddr, serializedEntry, discovery.DefaultLeaseInSeconds) + if err := s.serviceRegister.Register(); err != nil { + log.Error("failed to register the service", zap.String("service-name", utils.TSOServiceName), errs.ZapError(err)) + return err + } + + atomic.StoreInt64(&s.isRunning, 1) return nil } // CreateServer creates the Server func CreateServer(ctx context.Context, cfg *Config) *Server { - rand.New(rand.NewSource(time.Now().UnixNano())) svr := &Server{ DiagnosticsServer: sysutil.NewDiagnosticsServer(cfg.Log.File.Filename), startTimestamp: time.Now().Unix(), diff --git a/pkg/mcs/tso/server/testutil.go b/pkg/mcs/tso/server/testutil.go new file mode 100644 index 00000000000..626d1474673 --- /dev/null +++ b/pkg/mcs/tso/server/testutil.go @@ -0,0 +1,61 @@ +// Copyright 2023 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "strings" + + "github.com/pingcap/kvproto/pkg/tsopb" + "github.com/spf13/pflag" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" +) + +// MustNewGrpcClient must create a new TSO grpc client. +func MustNewGrpcClient(re *require.Assertions, addr string) (*grpc.ClientConn, tsopb.TSOClient) { + conn, err := grpc.Dial(strings.TrimPrefix(addr, "http://"), grpc.WithInsecure()) + re.NoError(err) + return conn, tsopb.NewTSOClient(conn) +} + +// GenerateConfig generates a new config with the given options. +func GenerateConfig(c *Config) (*Config, error) { + arguments := []string{ + "--listen-addr=" + c.ListenAddr, + "--advertise-listen-addr=" + c.AdvertiseListenAddr, + "--backend-endpoints=" + c.BackendEndpoints, + } + + flagSet := pflag.NewFlagSet("test", pflag.ContinueOnError) + flagSet.BoolP("version", "V", false, "print version information and exit") + flagSet.StringP("config", "", "", "config file") + flagSet.StringP("backend-endpoints", "", "", "url for etcd client") + flagSet.StringP("listen-addr", "", "", "listen address for tso service") + flagSet.StringP("advertise-listen-addr", "", "", "advertise urls for listen address (default '${listen-addr}')") + flagSet.StringP("cacert", "", "", "path of file that contains list of trusted TLS CAs") + flagSet.StringP("cert", "", "", "path of file that contains X509 certificate in PEM format") + flagSet.StringP("key", "", "", "path of file that contains X509 key in PEM format") + err := flagSet.Parse(arguments) + if err != nil { + return nil, err + } + cfg := NewConfig() + err = cfg.Parse(flagSet) + if err != nil { + return nil, err + } + + return cfg, nil +} diff --git a/pkg/mcs/utils/constant.go b/pkg/mcs/utils/constant.go index 628b4a05461..6174852d89f 100644 --- a/pkg/mcs/utils/constant.go +++ b/pkg/mcs/utils/constant.go @@ -17,12 +17,15 @@ package utils import "time" const ( + // MaxRetryTimesWaitAPIService is the max retry times for initializing the cluster ID. + MaxRetryTimesWaitAPIService = 360 + // RetryIntervalWaitAPIService is the interval to retry. + // Note: the interval must be less than the timeout of tidb and tikv, which is 2s by default in tikv. + RetryIntervalWaitAPIService = 500 * time.Millisecond + // TCPNetworkStr is the string of tcp network TCPNetworkStr = "tcp" - // ClusterIDPath is the path to store cluster id - ClusterIDPath = "/pd/cluster_id" - // DefaultEnableGRPCGateway is the default value of EnableGRPCGateway DefaultEnableGRPCGateway = true // DefaultGRPCGracefulStopTimeout is the default timeout to wait for grpc server to gracefully stop @@ -34,7 +37,57 @@ const ( // DefaultDisableErrorVerbose is the default value of DisableErrorVerbose DefaultDisableErrorVerbose = true // DefaultLeaderLease is the default value of LeaderLease - DefaultLeaderLease = 3 + DefaultLeaderLease = int64(3) // LeaderTickInterval is the interval to check leader LeaderTickInterval = 50 * time.Millisecond + + // DefaultKeyspaceName is the name reserved for default keyspace. + DefaultKeyspaceName = "DEFAULT" + + // DefaultKeyspaceID is the default key space id. + // Valid keyspace id range is [0, 0xFFFFFF](uint24max, or 16777215) + // ​0 is reserved for default keyspace with the name "DEFAULT", It's initialized when PD bootstrap + // and reserved for users who haven't been assigned keyspace. + DefaultKeyspaceID = uint32(0) + // NullKeyspaceID is used for api v1 or legacy path where is keyspace agnostic. + NullKeyspaceID = uint32(0xFFFFFFFF) + // DefaultKeyspaceGroupID is the default key space group id. + // We also reserved 0 for the keyspace group for the same purpose. + DefaultKeyspaceGroupID = uint32(0) + + // MicroserviceRootPath is the root path of microservice in etcd. + MicroserviceRootPath = "/ms" + // APIServiceName is the name of api server. + APIServiceName = "api" + // TSOServiceName is the name of tso server. + TSOServiceName = "tso" + // ResourceManagerServiceName is the name of resource manager server. + ResourceManagerServiceName = "resource_manager" + // SchedulingServiceName is the name of scheduling server. + SchedulingServiceName = "scheduling" + // KeyspaceGroupsKey is the path component of keyspace groups. + KeyspaceGroupsKey = "keyspace_groups" + // PrimaryKey is the path component of primary. + PrimaryKey = "primary" + + // MaxKeyspaceGroupCount is the max count of keyspace groups. keyspace group in tso + // is the sharding unit, i.e., by the definition here, the max count of the shards + // that we support is MaxKeyspaceGroupCount. The keyspace group id is in the range + // [0, 99999], which explains we use five-digits number (%05d) to render the keyspace + // group id in the storage endpoint path. + MaxKeyspaceGroupCount = uint32(100000) + // MaxKeyspaceGroupCountInUse is the max count of keyspace groups in use, which should + // never exceed MaxKeyspaceGroupCount defined above. Compared to MaxKeyspaceGroupCount, + // MaxKeyspaceGroupCountInUse is a much more reasonable value of the max count in the + // foreseen future, and the former is just for extensibility in theory. + MaxKeyspaceGroupCountInUse = uint32(4096) + + // DefaultKeyspaceGroupReplicaCount is the default replica count of keyspace group. + DefaultKeyspaceGroupReplicaCount = 2 + + // DefaultKeyspaceGroupReplicaPriority is the default priority of a keyspace group replica. + // It's used in keyspace group primary weighted-election to balance primaries' distribution. + // Among multiple replicas of a keyspace group, the higher the priority, the more likely + // the replica is to be elected as primary. + DefaultKeyspaceGroupReplicaPriority = 0 ) diff --git a/pkg/mcs/utils/util.go b/pkg/mcs/utils/util.go new file mode 100644 index 00000000000..cd85bb6ba60 --- /dev/null +++ b/pkg/mcs/utils/util.go @@ -0,0 +1,131 @@ +// Copyright 2023 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package utils + +import ( + "context" + "strings" + "sync" + "time" + + "github.com/gin-gonic/gin" + "github.com/pingcap/kvproto/pkg/pdpb" + "github.com/pingcap/log" + "github.com/pkg/errors" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" + "github.com/tikv/pd/pkg/errs" + "github.com/tikv/pd/pkg/utils/etcdutil" + "github.com/tikv/pd/pkg/utils/grpcutil" + "go.etcd.io/etcd/clientv3" + "go.uber.org/zap" + "google.golang.org/grpc" +) + +const ( + // maxRetryTimes is the max retry times for initializing the cluster ID. + maxRetryTimes = 5 + // clusterIDPath is the path to store cluster id + clusterIDPath = "/pd/cluster_id" + // retryInterval is the interval to retry. + retryInterval = time.Second +) + +// InitClusterID initializes the cluster ID. +func InitClusterID(ctx context.Context, client *clientv3.Client) (id uint64, err error) { + ticker := time.NewTicker(retryInterval) + defer ticker.Stop() + for i := 0; i < maxRetryTimes; i++ { + if clusterID, err := etcdutil.GetClusterID(client, clusterIDPath); err == nil && clusterID != 0 { + return clusterID, nil + } + select { + case <-ctx.Done(): + return 0, err + case <-ticker.C: + } + } + return 0, errors.Errorf("failed to init cluster ID after retrying %d times", maxRetryTimes) +} + +// PromHandler is a handler to get prometheus metrics. +func PromHandler() gin.HandlerFunc { + return func(c *gin.Context) { + // register promhttp.HandlerOpts DisableCompression + promhttp.InstrumentMetricHandler(prometheus.DefaultRegisterer, promhttp.HandlerFor(prometheus.DefaultGatherer, promhttp.HandlerOpts{ + DisableCompression: true, + })).ServeHTTP(c.Writer, c.Request) + } +} + +type server interface { + GetBackendEndpoints() string + Context() context.Context + GetTLSConfig() *grpcutil.TLSConfig + GetClientConns() *sync.Map + GetDelegateClient(ctx context.Context, forwardedHost string) (*grpc.ClientConn, error) +} + +// WaitAPIServiceReady waits for the api service ready. +func WaitAPIServiceReady(s server) error { + var ( + ready bool + err error + ) + ticker := time.NewTicker(RetryIntervalWaitAPIService) + defer ticker.Stop() + for i := 0; i < MaxRetryTimesWaitAPIService; i++ { + ready, err = isAPIServiceReady(s) + if err == nil && ready { + return nil + } + log.Debug("api server is not ready, retrying", errs.ZapError(err), zap.Bool("ready", ready)) + select { + case <-s.Context().Done(): + return errors.New("context canceled while waiting api server ready") + case <-ticker.C: + } + } + if err != nil { + log.Warn("failed to check api server ready", errs.ZapError(err)) + } + return errors.Errorf("failed to wait api server ready after retrying %d times", MaxRetryTimesWaitAPIService) +} + +func isAPIServiceReady(s server) (bool, error) { + urls := strings.Split(s.GetBackendEndpoints(), ",") + if len(urls) == 0 { + return false, errors.New("no backend endpoints") + } + cc, err := s.GetDelegateClient(s.Context(), urls[0]) + if err != nil { + return false, err + } + clusterInfo, err := pdpb.NewPDClient(cc).GetClusterInfo(s.Context(), &pdpb.GetClusterInfoRequest{}) + if err != nil { + return false, err + } + if clusterInfo.GetHeader().GetError() != nil { + return false, errors.Errorf(clusterInfo.GetHeader().GetError().String()) + } + modes := clusterInfo.ServiceModes + if len(modes) == 0 { + return false, errors.New("no service mode") + } + if modes[0] == pdpb.ServiceMode_API_SVC_MODE { + return true, nil + } + return false, nil +} diff --git a/pkg/member/election_leader.go b/pkg/member/election_leader.go new file mode 100644 index 00000000000..8c0496f670e --- /dev/null +++ b/pkg/member/election_leader.go @@ -0,0 +1,90 @@ +// Copyright 2023 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package member + +import ( + "context" + + "github.com/pingcap/kvproto/pkg/pdpb" + "github.com/pingcap/kvproto/pkg/tsopb" +) + +// ElectionLeader defines the common interface of the leader, which is the pdpb.Member +// for in PD/API service or the tsopb.Participant in the microserives. +type ElectionLeader interface { + // GetListenUrls returns the listen urls + GetListenUrls() []string + // GetRevision the revision of the leader in etcd + GetRevision() int64 + // String declares fmt.Stringer + String() string + // Watch on itself, the leader in the election group + Watch(context.Context) +} + +// EmbeddedEtcdLeader is the leader in the election group backed by the embedded etcd. +type EmbeddedEtcdLeader struct { + wrapper *EmbeddedEtcdMember + member *pdpb.Member + revision int64 +} + +// GetListenUrls returns current leader's client urls +func (l *EmbeddedEtcdLeader) GetListenUrls() []string { + return l.member.GetClientUrls() +} + +// GetRevision the revision of the leader in etcd +func (l *EmbeddedEtcdLeader) GetRevision() int64 { + return l.revision +} + +// String declares fmt.Stringer +func (l *EmbeddedEtcdLeader) String() string { + return l.member.String() +} + +// Watch on the leader +func (l *EmbeddedEtcdLeader) Watch(ctx context.Context) { + l.wrapper.WatchLeader(ctx, l.member, l.revision) +} + +// EtcdLeader is the leader in the election group backed by the etcd, but it's +// decoupled from the embedded etcd. +type EtcdLeader struct { + wrapper *Participant + pariticipant *tsopb.Participant + revision int64 +} + +// GetListenUrls returns current leader's client urls +func (l *EtcdLeader) GetListenUrls() []string { + return l.pariticipant.GetListenUrls() +} + +// GetRevision the revision of the leader in etcd +func (l *EtcdLeader) GetRevision() int64 { + return l.revision +} + +// String declares fmt.Stringer +func (l *EtcdLeader) String() string { + return l.pariticipant.String() +} + +// Watch on the leader +func (l *EtcdLeader) Watch(ctx context.Context) { + l.wrapper.WatchLeader(ctx, l.pariticipant, l.revision) +} diff --git a/pkg/member/member.go b/pkg/member/member.go index fb84e0716e9..80332a65f94 100644 --- a/pkg/member/member.go +++ b/pkg/member/member.go @@ -58,6 +58,8 @@ type EmbeddedEtcdMember struct { // etcd leader key when the PD node is successfully elected as the PD leader // of the cluster. Every write will use it to check PD leadership. memberValue string + // lastLeaderUpdatedTime is the last time when the leader is updated. + lastLeaderUpdatedTime atomic.Value } // NewMember create a new Member. @@ -74,6 +76,16 @@ func (m *EmbeddedEtcdMember) ID() uint64 { return m.id } +// Name returns the unique etcd Name for this server in etcd cluster. +func (m *EmbeddedEtcdMember) Name() string { + return m.member.Name +} + +// GetMember returns the member. +func (m *EmbeddedEtcdMember) GetMember() interface{} { + return m.member +} + // MemberValue returns the member value. func (m *EmbeddedEtcdMember) MemberValue() string { return m.memberValue @@ -99,6 +111,16 @@ func (m *EmbeddedEtcdMember) IsLeader() bool { return m.leadership.Check() && m.GetLeader().GetMemberId() == m.member.GetMemberId() } +// IsLeaderElected returns true if the leader exists; otherwise false +func (m *EmbeddedEtcdMember) IsLeaderElected() bool { + return m.GetLeader() != nil +} + +// GetLeaderListenUrls returns current leader's listen urls +func (m *EmbeddedEtcdMember) GetLeaderListenUrls() []string { + return m.GetLeader().GetClientUrls() +} + // GetLeaderID returns current PD leader's member ID. func (m *EmbeddedEtcdMember) GetLeaderID() uint64 { return m.GetLeader().GetMemberId() @@ -120,11 +142,13 @@ func (m *EmbeddedEtcdMember) GetLeader() *pdpb.Member { // setLeader sets the member's PD leader. func (m *EmbeddedEtcdMember) setLeader(member *pdpb.Member) { m.leader.Store(member) + m.lastLeaderUpdatedTime.Store(time.Now()) } // unsetLeader unsets the member's PD leader. func (m *EmbeddedEtcdMember) unsetLeader() { m.leader.Store(&pdpb.Member{}) + m.lastLeaderUpdatedTime.Store(time.Now()) } // EnableLeader sets the member itself to a PD leader. @@ -142,6 +166,15 @@ func (m *EmbeddedEtcdMember) GetLeadership() *election.Leadership { return m.leadership } +// GetLastLeaderUpdatedTime returns the last time when the leader is updated. +func (m *EmbeddedEtcdMember) GetLastLeaderUpdatedTime() time.Time { + lastLeaderUpdatedTime := m.lastLeaderUpdatedTime.Load() + if lastLeaderUpdatedTime == nil { + return time.Time{} + } + return lastLeaderUpdatedTime.(time.Time) +} + // CampaignLeader is used to campaign a PD member's leadership // and make it become a PD leader. func (m *EmbeddedEtcdMember) CampaignLeader(leaseTimeout int64) error { @@ -153,50 +186,73 @@ func (m *EmbeddedEtcdMember) KeepLeader(ctx context.Context) { m.leadership.Keep(ctx) } -// PrecheckLeader does some pre-check before checking whether or not it's the leader. -func (m *EmbeddedEtcdMember) PrecheckLeader() error { +// PreCheckLeader does some pre-check before checking whether or not it's the leader. +func (m *EmbeddedEtcdMember) PreCheckLeader() error { if m.GetEtcdLeader() == 0 { return errs.ErrEtcdLeaderNotFound } return nil } -// CheckLeader checks returns true if it is needed to check later. -func (m *EmbeddedEtcdMember) CheckLeader() (*pdpb.Member, int64, bool) { - if err := m.PrecheckLeader(); err != nil { +// getPersistentLeader gets the corresponding leader from etcd by given leaderPath (as the key). +func (m *EmbeddedEtcdMember) getPersistentLeader() (*pdpb.Member, int64, error) { + leader := &pdpb.Member{} + ok, rev, err := etcdutil.GetProtoMsgWithModRev(m.client, m.GetLeaderPath(), leader) + if err != nil { + return nil, 0, err + } + if !ok { + return nil, 0, nil + } + + return leader, rev, nil +} + +// CheckLeader checks if someone else is taking the leadership. If yes, returns the leader; +// otherwise returns a bool which indicates if it is needed to check later. +func (m *EmbeddedEtcdMember) CheckLeader() (ElectionLeader, bool) { + if err := m.PreCheckLeader(); err != nil { log.Error("failed to pass pre-check, check pd leader later", errs.ZapError(err)) time.Sleep(200 * time.Millisecond) - return nil, 0, true + return nil, true } - leader, rev, err := election.GetLeader(m.client, m.GetLeaderPath()) + leader, revision, err := m.getPersistentLeader() if err != nil { log.Error("getting pd leader meets error", errs.ZapError(err)) time.Sleep(200 * time.Millisecond) - return nil, 0, true - } - if leader != nil { - if m.IsSameLeader(leader) { - // oh, we are already a PD leader, which indicates we may meet something wrong - // in previous CampaignLeader. We should delete the leadership and campaign again. - log.Warn("the pd leader has not changed, delete and campaign again", zap.Stringer("old-pd-leader", leader)) - // Delete the leader itself and let others start a new election again. - if err = m.leadership.DeleteLeaderKey(); err != nil { - log.Error("deleting pd leader key meets error", errs.ZapError(err)) - time.Sleep(200 * time.Millisecond) - return nil, 0, true - } - // Return nil and false to make sure the campaign will start immediately. - return nil, 0, false + return nil, true + } + if leader == nil { + // no leader yet + return nil, false + } + + if m.IsSameLeader(leader) { + // oh, we are already a PD leader, which indicates we may meet something wrong + // in previous CampaignLeader. We should delete the leadership and campaign again. + log.Warn("the pd leader has not changed, delete and campaign again", zap.Stringer("old-pd-leader", leader)) + // Delete the leader itself and let others start a new election again. + if err = m.leadership.DeleteLeaderKey(); err != nil { + log.Error("deleting pd leader key meets error", errs.ZapError(err)) + time.Sleep(200 * time.Millisecond) + return nil, true } + // Return nil and false to make sure the campaign will start immediately. + return nil, false } - return leader, rev, false + + return &EmbeddedEtcdLeader{ + wrapper: m, + member: leader, + revision: revision, + }, false } // WatchLeader is used to watch the changes of the leader. -func (m *EmbeddedEtcdMember) WatchLeader(serverCtx context.Context, leader *pdpb.Member, revision int64) { +func (m *EmbeddedEtcdMember) WatchLeader(ctx context.Context, leader *pdpb.Member, revision int64) { m.setLeader(leader) - m.leadership.Watch(serverCtx, revision) + m.leadership.Watch(ctx, revision) m.unsetLeader() } @@ -274,6 +330,7 @@ func (m *EmbeddedEtcdMember) InitMemberInfo(advertiseClientUrls, advertisePeerUr m.memberValue = string(data) m.rootPath = rootPath m.leadership = election.NewLeadership(m.client, m.GetLeaderPath(), "leader election") + log.Info("member joining election", zap.Stringer("member-info", m.member), zap.String("root-path", m.rootPath)) } // ResignEtcdLeader resigns current PD's etcd leadership. If nextLeader is empty, all @@ -300,7 +357,8 @@ func (m *EmbeddedEtcdMember) ResignEtcdLeader(ctx context.Context, from string, if len(etcdLeaderIDs) == 0 { return errors.New("no valid pd to transfer etcd leader") } - nextEtcdLeaderID := etcdLeaderIDs[rand.Intn(len(etcdLeaderIDs))] + r := rand.New(rand.NewSource(time.Now().UnixNano())) + nextEtcdLeaderID := etcdLeaderIDs[r.Intn(len(etcdLeaderIDs))] return m.MoveEtcdLeader(ctx, m.ID(), nextEtcdLeaderID) } diff --git a/pkg/member/participant.go b/pkg/member/participant.go index 843a690ec88..102bfcbce5f 100644 --- a/pkg/member/participant.go +++ b/pkg/member/participant.go @@ -17,62 +17,57 @@ package member import ( "context" "fmt" - "os" "path" - "path/filepath" "strconv" "sync/atomic" "time" - "github.com/pingcap/errors" - "github.com/pingcap/kvproto/pkg/pdpb" + "github.com/pingcap/kvproto/pkg/tsopb" "github.com/pingcap/log" "github.com/tikv/pd/pkg/election" "github.com/tikv/pd/pkg/errs" - "github.com/tikv/pd/pkg/storage/kv" "github.com/tikv/pd/pkg/utils/etcdutil" "go.etcd.io/etcd/clientv3" "go.uber.org/zap" ) +type leadershipCheckFunc func(*election.Leadership) bool + // Participant is used for the election related logic. Compared to its counterpart // EmbeddedEtcdMember, Participant relies on etcd for election, but it's decoupled -// with the embedded etcd. It implements Member interface. +// from the embedded etcd. It implements Member interface. type Participant struct { leadership *election.Leadership // stored as member type - leader atomic.Value - client *clientv3.Client - // id is unique among all participants - id uint64 - // member is the current participant's info. - // TODO: use a more general instead of pdpb.Member. The latter is tightly - // coupled with the embedded etcd. For now, we can reuse it and ignore the - // fields which are related to embedded etcd, such as PeerUrls and ClientUrls. - member *pdpb.Member + leader atomic.Value + client *clientv3.Client rootPath string leaderPath string + member *tsopb.Participant // memberValue is the serialized string of `member`. It will be saved in the - // eader key when this participant is successfully elected as the leader of + // leader key when this participant is successfully elected as the leader of // the group. Every write will use it to check the leadership. memberValue string + // campaignChecker is used to check whether the additional constraints for a + // campaign are satisfied. If it returns false, the campaign will fail. + campaignChecker atomic.Value // Store as leadershipCheckFunc + // lastLeaderUpdatedTime is the last time when the leader is updated. + lastLeaderUpdatedTime atomic.Value } // NewParticipant create a new Participant. -func NewParticipant(client *clientv3.Client, id uint64) *Participant { +func NewParticipant(client *clientv3.Client) *Participant { return &Participant{ client: client, - id: id, } } // InitInfo initializes the member info. The leader key is path.Join(rootPath, leaderName) -func (m *Participant) InitInfo(name string, rootPath string, leaderName string, purpose string, listenURL string) { - leader := &pdpb.Member{ - Name: name, - MemberId: m.ID(), - // TODO: need refactor - ClientUrls: []string{listenURL}, +func (m *Participant) InitInfo(name string, id uint64, rootPath string, leaderName string, purpose string, advertiseListenAddr string) { + leader := &tsopb.Participant{ + Name: name, + Id: id, // id is unique among all participants + ListenUrls: []string{advertiseListenAddr}, } data, err := leader.Marshal() @@ -85,25 +80,30 @@ func (m *Participant) InitInfo(name string, rootPath string, leaderName string, m.rootPath = rootPath m.leaderPath = path.Join(rootPath, leaderName) m.leadership = election.NewLeadership(m.client, m.GetLeaderPath(), purpose) + m.lastLeaderUpdatedTime.Store(time.Now()) + log.Info("participant joining election", zap.Stringer("participant-info", m.member), zap.String("leader-path", m.leaderPath)) } -// ID returns the unique ID for this participant in the group. For example, it can be -// unique server id of a cluster or the unique keyspace group replica id of the election -// group comprised of the replicas of a keyspace group. +// ID returns the unique ID for this participant in the election group func (m *Participant) ID() uint64 { - return m.id + return m.member.Id } -// MemberValue returns the member value. -func (m *Participant) MemberValue() string { - return m.memberValue +// Name returns the unique name in the election group. +func (m *Participant) Name() string { + return m.member.Name } -// Member returns the member. -func (m *Participant) Member() *pdpb.Member { +// GetMember returns the member. +func (m *Participant) GetMember() interface{} { return m.member } +// MemberValue returns the member value. +func (m *Participant) MemberValue() string { + return m.memberValue +} + // Client returns the etcd client. func (m *Participant) Client() *clientv3.Client { return m.client @@ -112,35 +112,47 @@ func (m *Participant) Client() *clientv3.Client { // IsLeader returns whether the participant is the leader or not by checking its leadership's // lease and leader info. func (m *Participant) IsLeader() bool { - return m.leadership.Check() && m.GetLeader().GetMemberId() == m.member.GetMemberId() + return m.leadership.Check() && m.GetLeader().GetId() == m.member.GetId() && m.campaignCheck() +} + +// IsLeaderElected returns true if the leader exists; otherwise false +func (m *Participant) IsLeaderElected() bool { + return m.GetLeader() != nil +} + +// GetLeaderListenUrls returns current leader's listen urls +func (m *Participant) GetLeaderListenUrls() []string { + return m.GetLeader().GetListenUrls() } // GetLeaderID returns current leader's member ID. func (m *Participant) GetLeaderID() uint64 { - return m.GetLeader().GetMemberId() + return m.GetLeader().GetId() } // GetLeader returns current leader of the election group. -func (m *Participant) GetLeader() *pdpb.Member { +func (m *Participant) GetLeader() *tsopb.Participant { leader := m.leader.Load() if leader == nil { return nil } - member := leader.(*pdpb.Member) - if member.GetMemberId() == 0 { + member := leader.(*tsopb.Participant) + if member.GetId() == 0 { return nil } return member } // setLeader sets the member's leader. -func (m *Participant) setLeader(member *pdpb.Member) { +func (m *Participant) setLeader(member *tsopb.Participant) { m.leader.Store(member) + m.lastLeaderUpdatedTime.Store(time.Now()) } // unsetLeader unsets the member's leader. func (m *Participant) unsetLeader() { - m.leader.Store(&pdpb.Member{}) + m.leader.Store(&tsopb.Participant{}) + m.lastLeaderUpdatedTime.Store(time.Now()) } // EnableLeader declares the member itself to be the leader. @@ -153,13 +165,25 @@ func (m *Participant) GetLeaderPath() string { return m.leaderPath } +// GetLastLeaderUpdatedTime returns the last time when the leader is updated. +func (m *Participant) GetLastLeaderUpdatedTime() time.Time { + lastLeaderUpdatedTime := m.lastLeaderUpdatedTime.Load() + if lastLeaderUpdatedTime == nil { + return time.Time{} + } + return lastLeaderUpdatedTime.(time.Time) +} + // GetLeadership returns the leadership of the member. func (m *Participant) GetLeadership() *election.Leadership { return m.leadership } -// CampaignLeader is used to campaign a member's leadership and make it become a leader. +// CampaignLeader is used to campaign the leadership and make it become a leader. func (m *Participant) CampaignLeader(leaseTimeout int64) error { + if !m.campaignCheck() { + return errs.ErrCheckCampaign + } return m.leadership.Campaign(leaseTimeout, m.MemberValue()) } @@ -168,49 +192,72 @@ func (m *Participant) KeepLeader(ctx context.Context) { m.leadership.Keep(ctx) } -// PrecheckLeader does some pre-check before checking whether or not it's the leader. +// PreCheckLeader does some pre-check before checking whether or not it's the leader. // It returns true if it passes the pre-check, false otherwise. -func (m *Participant) PrecheckLeader() error { +func (m *Participant) PreCheckLeader() error { // No specific thing to check. Returns no error. return nil } -// CheckLeader checks returns true if it is needed to check later. -func (m *Participant) CheckLeader() (*pdpb.Member, int64, bool) { - if err := m.PrecheckLeader(); err != nil { +// getPersistentLeader gets the corresponding leader from etcd by given leaderPath (as the key). +func (m *Participant) getPersistentLeader() (*tsopb.Participant, int64, error) { + leader := &tsopb.Participant{} + ok, rev, err := etcdutil.GetProtoMsgWithModRev(m.client, m.GetLeaderPath(), leader) + if err != nil { + return nil, 0, err + } + if !ok { + return nil, 0, nil + } + + return leader, rev, nil +} + +// CheckLeader checks if someone else is taking the leadership. If yes, returns the leader; +// otherwise returns a bool which indicates if it is needed to check later. +func (m *Participant) CheckLeader() (ElectionLeader, bool) { + if err := m.PreCheckLeader(); err != nil { log.Error("failed to pass pre-check, check the leader later", errs.ZapError(errs.ErrEtcdLeaderNotFound)) time.Sleep(200 * time.Millisecond) - return nil, 0, true + return nil, true } - leader, rev, err := election.GetLeader(m.client, m.GetLeaderPath()) + leader, revision, err := m.getPersistentLeader() if err != nil { log.Error("getting the leader meets error", errs.ZapError(err)) time.Sleep(200 * time.Millisecond) - return nil, 0, true + return nil, true } - if leader != nil { - if m.IsSameLeader(leader) { - // oh, we are already the leader, which indicates we may meet something wrong - // in previous CampaignLeader. We should delete the leadership and campaign again. - log.Warn("the leader has not changed, delete and campaign again", zap.Stringer("old-leader", leader)) - // Delete the leader itself and let others start a new election again. - if err = m.leadership.DeleteLeaderKey(); err != nil { - log.Error("deleting the leader key meets error", errs.ZapError(err)) - time.Sleep(200 * time.Millisecond) - return nil, 0, true - } - // Return nil and false to make sure the campaign will start immediately. - return nil, 0, false + if leader == nil { + // no leader yet + return nil, false + } + + if m.IsSameLeader(leader) { + // oh, we are already the leader, which indicates we may meet something wrong + // in previous CampaignLeader. We should delete the leadership and campaign again. + log.Warn("the leader has not changed, delete and campaign again", zap.Stringer("old-leader", leader)) + // Delete the leader itself and let others start a new election again. + if err = m.leadership.DeleteLeaderKey(); err != nil { + log.Error("deleting the leader key meets error", errs.ZapError(err)) + time.Sleep(200 * time.Millisecond) + return nil, true } + // Return nil and false to make sure the campaign will start immediately. + return nil, false } - return leader, rev, false + + return &EtcdLeader{ + wrapper: m, + pariticipant: leader, + revision: revision, + }, false } // WatchLeader is used to watch the changes of the leader. -func (m *Participant) WatchLeader(serverCtx context.Context, leader *pdpb.Member, revision int64) { +func (m *Participant) WatchLeader(ctx context.Context, leader *tsopb.Participant, revision int64) { m.setLeader(leader) - m.leadership.Watch(serverCtx, revision) + m.leadership.Watch(ctx, revision) m.unsetLeader() } @@ -222,8 +269,8 @@ func (m *Participant) ResetLeader() { } // IsSameLeader checks whether a server is the leader itself. -func (m *Participant) IsSameLeader(leader *pdpb.Member) bool { - return leader.GetMemberId() == m.ID() +func (m *Participant) IsSameLeader(leader *tsopb.Participant) bool { + return leader.GetId() == m.ID() } // CheckPriority checks whether there is another participant has higher priority and resign it as the leader if so. @@ -231,8 +278,8 @@ func (m *Participant) CheckPriority(ctx context.Context) { // TODO: implement weighted-election when it's in need } -func (m *Participant) getMemberLeaderPriorityPath(id uint64) string { - return path.Join(m.rootPath, fmt.Sprintf("member/%d/leader_priority", id)) +func (m *Participant) getLeaderPriorityPath(id uint64) string { + return path.Join(m.rootPath, fmt.Sprintf("participant/%d/leader_priority", id)) } // GetDCLocationPathPrefix returns the dc-location path prefix of the cluster. @@ -245,9 +292,9 @@ func (m *Participant) GetDCLocationPath(id uint64) string { return path.Join(m.GetDCLocationPathPrefix(), fmt.Sprint(id)) } -// SetMemberLeaderPriority saves a member's priority to be elected as the etcd leader. -func (m *Participant) SetMemberLeaderPriority(id uint64, priority int) error { - key := m.getMemberLeaderPriorityPath(id) +// SetLeaderPriority saves the priority to be elected as the etcd leader. +func (m *Participant) SetLeaderPriority(id uint64, priority int) error { + key := m.getLeaderPriorityPath(id) res, err := m.leadership.LeaderTxn().Then(clientv3.OpPut(key, strconv.Itoa(priority))).Commit() if err != nil { return errs.ErrEtcdTxnInternal.Wrap(err).GenWithStackByCause() @@ -259,9 +306,9 @@ func (m *Participant) SetMemberLeaderPriority(id uint64, priority int) error { return nil } -// DeleteMemberLeaderPriority removes a member's etcd leader priority config. -func (m *Participant) DeleteMemberLeaderPriority(id uint64) error { - key := m.getMemberLeaderPriorityPath(id) +// DeleteLeaderPriority removes the etcd leader priority config. +func (m *Participant) DeleteLeaderPriority(id uint64) error { + key := m.getLeaderPriorityPath(id) res, err := m.leadership.LeaderTxn().Then(clientv3.OpDelete(key)).Commit() if err != nil { return errs.ErrEtcdTxnInternal.Wrap(err).GenWithStackByCause() @@ -273,8 +320,8 @@ func (m *Participant) DeleteMemberLeaderPriority(id uint64) error { return nil } -// DeleteMemberDCLocationInfo removes a member's dc-location info. -func (m *Participant) DeleteMemberDCLocationInfo(id uint64) error { +// DeleteDCLocationInfo removes the dc-location info. +func (m *Participant) DeleteDCLocationInfo(id uint64) error { key := m.GetDCLocationPath(id) res, err := m.leadership.LeaderTxn().Then(clientv3.OpDelete(key)).Commit() if err != nil { @@ -287,9 +334,9 @@ func (m *Participant) DeleteMemberDCLocationInfo(id uint64) error { return nil } -// GetMemberLeaderPriority loads a member's priority to be elected as the etcd leader. -func (m *Participant) GetMemberLeaderPriority(id uint64) (int, error) { - key := m.getMemberLeaderPriorityPath(id) +// GetLeaderPriority loads the priority to be elected as the etcd leader. +func (m *Participant) GetLeaderPriority(id uint64) (int, error) { + key := m.getLeaderPriorityPath(id) res, err := etcdutil.EtcdKVGet(m.client, key) if err != nil { return 0, err @@ -304,100 +351,19 @@ func (m *Participant) GetMemberLeaderPriority(id uint64) (int, error) { return int(priority), nil } -func (m *Participant) getMemberBinaryDeployPath(id uint64) string { - return path.Join(m.rootPath, fmt.Sprintf("member/%d/deploy_path", id)) -} - -// GetMemberDeployPath loads a member's binary deploy path. -func (m *Participant) GetMemberDeployPath(id uint64) (string, error) { - key := m.getMemberBinaryDeployPath(id) - res, err := etcdutil.EtcdKVGet(m.client, key) - if err != nil { - return "", err +func (m *Participant) campaignCheck() bool { + checker := m.campaignChecker.Load() + if checker == nil { + return true } - if len(res.Kvs) == 0 { - return "", errs.ErrEtcdKVGetResponse.FastGenByArgs("no value") + checkerFunc, ok := checker.(leadershipCheckFunc) + if !ok || checkerFunc == nil { + return true } - return string(res.Kvs[0].Value), nil + return checkerFunc(m.leadership) } -// SetMemberDeployPath saves a member's binary deploy path. -func (m *Participant) SetMemberDeployPath(id uint64) error { - key := m.getMemberBinaryDeployPath(id) - txn := kv.NewSlowLogTxn(m.client) - execPath, err := os.Executable() - deployPath := filepath.Dir(execPath) - if err != nil { - return errors.WithStack(err) - } - res, err := txn.Then(clientv3.OpPut(key, deployPath)).Commit() - if err != nil { - return errors.WithStack(err) - } - if !res.Succeeded { - return errors.New("failed to save deploy path") - } - return nil -} - -func (m *Participant) getMemberGitHashPath(id uint64) string { - return path.Join(m.rootPath, fmt.Sprintf("member/%d/git_hash", id)) -} - -func (m *Participant) getMemberBinaryVersionPath(id uint64) string { - return path.Join(m.rootPath, fmt.Sprintf("member/%d/binary_version", id)) -} - -// GetMemberBinaryVersion loads a member's binary version. -func (m *Participant) GetMemberBinaryVersion(id uint64) (string, error) { - key := m.getMemberBinaryVersionPath(id) - res, err := etcdutil.EtcdKVGet(m.client, key) - if err != nil { - return "", err - } - if len(res.Kvs) == 0 { - return "", errs.ErrEtcdKVGetResponse.FastGenByArgs("no value") - } - return string(res.Kvs[0].Value), nil -} - -// GetMemberGitHash loads a member's git hash. -func (m *Participant) GetMemberGitHash(id uint64) (string, error) { - key := m.getMemberGitHashPath(id) - res, err := etcdutil.EtcdKVGet(m.client, key) - if err != nil { - return "", err - } - if len(res.Kvs) == 0 { - return "", errs.ErrEtcdKVGetResponse.FastGenByArgs("no value") - } - return string(res.Kvs[0].Value), nil -} - -// SetMemberBinaryVersion saves a member's binary version. -func (m *Participant) SetMemberBinaryVersion(id uint64, releaseVersion string) error { - key := m.getMemberBinaryVersionPath(id) - txn := kv.NewSlowLogTxn(m.client) - res, err := txn.Then(clientv3.OpPut(key, releaseVersion)).Commit() - if err != nil { - return errors.WithStack(err) - } - if !res.Succeeded { - return errors.New("failed to save binary version") - } - return nil -} - -// SetMemberGitHash saves a member's git hash. -func (m *Participant) SetMemberGitHash(id uint64, gitHash string) error { - key := m.getMemberGitHashPath(id) - txn := kv.NewSlowLogTxn(m.client) - res, err := txn.Then(clientv3.OpPut(key, gitHash)).Commit() - if err != nil { - return errors.WithStack(err) - } - if !res.Succeeded { - return errors.New("failed to save git hash") - } - return nil +// SetCampaignChecker sets the pre-campaign checker. +func (m *Participant) SetCampaignChecker(checker leadershipCheckFunc) { + m.campaignChecker.Store(checker) } diff --git a/pkg/memory/meminfo.go b/pkg/memory/meminfo.go index ee8bdd134af..0981ddacdfb 100644 --- a/pkg/memory/meminfo.go +++ b/pkg/memory/meminfo.go @@ -20,6 +20,7 @@ import ( "github.com/pingcap/failpoint" "github.com/pingcap/log" + "github.com/pingcap/sysutil" "github.com/shirou/gopsutil/v3/mem" "github.com/tikv/pd/pkg/cgroup" "go.uber.org/zap" @@ -161,6 +162,7 @@ func init() { if cgroup.InContainer() { MemTotal = MemTotalCGroup MemUsed = MemUsedCGroup + sysutil.RegisterGetMemoryCapacity(MemTotalCGroup) } else { MemTotal = MemTotalNormal MemUsed = MemUsedNormal diff --git a/pkg/mock/mockcluster/config.go b/pkg/mock/mockcluster/config.go index dc3565dc180..b8c35cb7046 100644 --- a/pkg/mock/mockcluster/config.go +++ b/pkg/mock/mockcluster/config.go @@ -17,6 +17,7 @@ package mockcluster import ( "time" + sc "github.com/tikv/pd/pkg/schedule/config" "github.com/tikv/pd/pkg/schedule/placement" "github.com/tikv/pd/pkg/utils/typeutil" "github.com/tikv/pd/server/config" @@ -24,97 +25,97 @@ import ( // SetMaxMergeRegionSize updates the MaxMergeRegionSize configuration. func (mc *Cluster) SetMaxMergeRegionSize(v int) { - mc.updateScheduleConfig(func(s *config.ScheduleConfig) { s.MaxMergeRegionSize = uint64(v) }) + mc.updateScheduleConfig(func(s *sc.ScheduleConfig) { s.MaxMergeRegionSize = uint64(v) }) } // SetMaxMergeRegionKeys updates the MaxMergeRegionKeys configuration. func (mc *Cluster) SetMaxMergeRegionKeys(v int) { - mc.updateScheduleConfig(func(s *config.ScheduleConfig) { s.MaxMergeRegionKeys = uint64(v) }) + mc.updateScheduleConfig(func(s *sc.ScheduleConfig) { s.MaxMergeRegionKeys = uint64(v) }) } // SetSplitMergeInterval updates the SplitMergeInterval configuration. func (mc *Cluster) SetSplitMergeInterval(v time.Duration) { - mc.updateScheduleConfig(func(s *config.ScheduleConfig) { s.SplitMergeInterval = typeutil.NewDuration(v) }) + mc.updateScheduleConfig(func(s *sc.ScheduleConfig) { s.SplitMergeInterval = typeutil.NewDuration(v) }) } // SetEnableOneWayMerge updates the EnableOneWayMerge configuration. func (mc *Cluster) SetEnableOneWayMerge(v bool) { - mc.updateScheduleConfig(func(s *config.ScheduleConfig) { s.EnableOneWayMerge = v }) + mc.updateScheduleConfig(func(s *sc.ScheduleConfig) { s.EnableOneWayMerge = v }) } // SetMaxSnapshotCount updates the MaxSnapshotCount configuration. func (mc *Cluster) SetMaxSnapshotCount(v int) { - mc.updateScheduleConfig(func(s *config.ScheduleConfig) { s.MaxSnapshotCount = uint64(v) }) + mc.updateScheduleConfig(func(s *sc.ScheduleConfig) { s.MaxSnapshotCount = uint64(v) }) } // SetEnableMakeUpReplica updates the EnableMakeUpReplica configuration. func (mc *Cluster) SetEnableMakeUpReplica(v bool) { - mc.updateScheduleConfig(func(s *config.ScheduleConfig) { s.EnableMakeUpReplica = v }) + mc.updateScheduleConfig(func(s *sc.ScheduleConfig) { s.EnableMakeUpReplica = v }) } // SetEnableRemoveExtraReplica updates the EnableRemoveExtraReplica configuration. func (mc *Cluster) SetEnableRemoveExtraReplica(v bool) { - mc.updateScheduleConfig(func(s *config.ScheduleConfig) { s.EnableRemoveExtraReplica = v }) + mc.updateScheduleConfig(func(s *sc.ScheduleConfig) { s.EnableRemoveExtraReplica = v }) } // SetEnableLocationReplacement updates the EnableLocationReplacement configuration. func (mc *Cluster) SetEnableLocationReplacement(v bool) { - mc.updateScheduleConfig(func(s *config.ScheduleConfig) { s.EnableLocationReplacement = v }) + mc.updateScheduleConfig(func(s *sc.ScheduleConfig) { s.EnableLocationReplacement = v }) } // SetEnableRemoveDownReplica updates the EnableRemoveDownReplica configuration. func (mc *Cluster) SetEnableRemoveDownReplica(v bool) { - mc.updateScheduleConfig(func(s *config.ScheduleConfig) { s.EnableRemoveDownReplica = v }) + mc.updateScheduleConfig(func(s *sc.ScheduleConfig) { s.EnableRemoveDownReplica = v }) } // SetEnableReplaceOfflineReplica updates the EnableReplaceOfflineReplica configuration. func (mc *Cluster) SetEnableReplaceOfflineReplica(v bool) { - mc.updateScheduleConfig(func(s *config.ScheduleConfig) { s.EnableReplaceOfflineReplica = v }) + mc.updateScheduleConfig(func(s *sc.ScheduleConfig) { s.EnableReplaceOfflineReplica = v }) } // SetLeaderSchedulePolicy updates the LeaderSchedulePolicy configuration. func (mc *Cluster) SetLeaderSchedulePolicy(v string) { - mc.updateScheduleConfig(func(s *config.ScheduleConfig) { s.LeaderSchedulePolicy = v }) + mc.updateScheduleConfig(func(s *sc.ScheduleConfig) { s.LeaderSchedulePolicy = v }) } // SetTolerantSizeRatio updates the TolerantSizeRatio configuration. func (mc *Cluster) SetTolerantSizeRatio(v float64) { - mc.updateScheduleConfig(func(s *config.ScheduleConfig) { s.TolerantSizeRatio = v }) + mc.updateScheduleConfig(func(s *sc.ScheduleConfig) { s.TolerantSizeRatio = v }) } // SetRegionScoreFormulaVersion updates the RegionScoreFormulaVersion configuration. func (mc *Cluster) SetRegionScoreFormulaVersion(v string) { - mc.updateScheduleConfig(func(s *config.ScheduleConfig) { s.RegionScoreFormulaVersion = v }) + mc.updateScheduleConfig(func(s *sc.ScheduleConfig) { s.RegionScoreFormulaVersion = v }) } // SetLeaderScheduleLimit updates the LeaderScheduleLimit configuration. func (mc *Cluster) SetLeaderScheduleLimit(v int) { - mc.updateScheduleConfig(func(s *config.ScheduleConfig) { s.LeaderScheduleLimit = uint64(v) }) + mc.updateScheduleConfig(func(s *sc.ScheduleConfig) { s.LeaderScheduleLimit = uint64(v) }) } // SetRegionScheduleLimit updates the RegionScheduleLimit configuration. func (mc *Cluster) SetRegionScheduleLimit(v int) { - mc.updateScheduleConfig(func(s *config.ScheduleConfig) { s.RegionScheduleLimit = uint64(v) }) + mc.updateScheduleConfig(func(s *sc.ScheduleConfig) { s.RegionScheduleLimit = uint64(v) }) } // SetMergeScheduleLimit updates the MergeScheduleLimit configuration. func (mc *Cluster) SetMergeScheduleLimit(v int) { - mc.updateScheduleConfig(func(s *config.ScheduleConfig) { s.MergeScheduleLimit = uint64(v) }) + mc.updateScheduleConfig(func(s *sc.ScheduleConfig) { s.MergeScheduleLimit = uint64(v) }) } // SetHotRegionScheduleLimit updates the HotRegionScheduleLimit configuration. func (mc *Cluster) SetHotRegionScheduleLimit(v int) { - mc.updateScheduleConfig(func(s *config.ScheduleConfig) { s.HotRegionScheduleLimit = uint64(v) }) + mc.updateScheduleConfig(func(s *sc.ScheduleConfig) { s.HotRegionScheduleLimit = uint64(v) }) } // SetHotRegionCacheHitsThreshold updates the HotRegionCacheHitsThreshold configuration. func (mc *Cluster) SetHotRegionCacheHitsThreshold(v int) { - mc.updateScheduleConfig(func(s *config.ScheduleConfig) { s.HotRegionCacheHitsThreshold = uint64(v) }) + mc.updateScheduleConfig(func(s *sc.ScheduleConfig) { s.HotRegionCacheHitsThreshold = uint64(v) }) } // SetEnablePlacementRules updates the EnablePlacementRules configuration. func (mc *Cluster) SetEnablePlacementRules(v bool) { - mc.updateReplicationConfig(func(r *config.ReplicationConfig) { r.EnablePlacementRules = v }) + mc.updateReplicationConfig(func(r *sc.ReplicationConfig) { r.EnablePlacementRules = v }) if v { mc.initRuleManager() } @@ -122,26 +123,26 @@ func (mc *Cluster) SetEnablePlacementRules(v bool) { // SetMaxReplicas updates the maxReplicas configuration. func (mc *Cluster) SetMaxReplicas(v int) { - mc.updateReplicationConfig(func(r *config.ReplicationConfig) { r.MaxReplicas = uint64(v) }) + mc.updateReplicationConfig(func(r *sc.ReplicationConfig) { r.MaxReplicas = uint64(v) }) } // SetLocationLabels updates the LocationLabels configuration. func (mc *Cluster) SetLocationLabels(v []string) { - mc.updateReplicationConfig(func(r *config.ReplicationConfig) { r.LocationLabels = v }) + mc.updateReplicationConfig(func(r *sc.ReplicationConfig) { r.LocationLabels = v }) } // SetIsolationLevel updates the IsolationLevel configuration. func (mc *Cluster) SetIsolationLevel(v string) { - mc.updateReplicationConfig(func(r *config.ReplicationConfig) { r.IsolationLevel = v }) + mc.updateReplicationConfig(func(r *sc.ReplicationConfig) { r.IsolationLevel = v }) } -func (mc *Cluster) updateScheduleConfig(f func(*config.ScheduleConfig)) { +func (mc *Cluster) updateScheduleConfig(f func(*sc.ScheduleConfig)) { s := mc.GetScheduleConfig().Clone() f(s) mc.SetScheduleConfig(s) } -func (mc *Cluster) updateReplicationConfig(f func(*config.ReplicationConfig)) { +func (mc *Cluster) updateReplicationConfig(f func(*sc.ReplicationConfig)) { r := mc.GetReplicationConfig().Clone() f(r) mc.SetReplicationConfig(r) diff --git a/pkg/mock/mockcluster/mockcluster.go b/pkg/mock/mockcluster/mockcluster.go index 6ff059d5fae..988a7788d06 100644 --- a/pkg/mock/mockcluster/mockcluster.go +++ b/pkg/mock/mockcluster/mockcluster.go @@ -24,10 +24,10 @@ import ( "github.com/pingcap/kvproto/pkg/metapb" "github.com/pingcap/kvproto/pkg/pdpb" "github.com/pingcap/log" + "github.com/pkg/errors" "github.com/tikv/pd/pkg/core" "github.com/tikv/pd/pkg/core/storelimit" "github.com/tikv/pd/pkg/errs" - "github.com/tikv/pd/pkg/id" "github.com/tikv/pd/pkg/mock/mockid" sc "github.com/tikv/pd/pkg/schedule/config" "github.com/tikv/pd/pkg/schedule/labeler" @@ -47,6 +47,7 @@ const ( // Cluster is used to mock a cluster for test purpose. type Cluster struct { + ctx context.Context *core.BasicCluster *mockid.IDAllocator *placement.RuleManager @@ -57,12 +58,13 @@ type Cluster struct { suspectRegions map[uint64]struct{} *config.StoreConfigManager *buckets.HotBucketCache - ctx context.Context + storage.Storage } // NewCluster creates a new Cluster func NewCluster(ctx context.Context, opts *config.PersistOptions) *Cluster { - clus := &Cluster{ + c := &Cluster{ + ctx: ctx, BasicCluster: core.NewBasicCluster(), IDAllocator: mockid.NewIDAllocator(), HotStat: statistics.NewHotStat(ctx), @@ -70,37 +72,55 @@ func NewCluster(ctx context.Context, opts *config.PersistOptions) *Cluster { PersistOptions: opts, suspectRegions: map[uint64]struct{}{}, StoreConfigManager: config.NewTestStoreConfigManager(nil), - ctx: ctx, + Storage: storage.NewStorageWithMemoryBackend(), } - if clus.PersistOptions.GetReplicationConfig().EnablePlacementRules { - clus.initRuleManager() + if c.PersistOptions.GetReplicationConfig().EnablePlacementRules { + c.initRuleManager() } // It should be updated to the latest feature version. - clus.PersistOptions.SetClusterVersion(versioninfo.MinSupportedVersion(versioninfo.HotScheduleWithQuery)) - clus.RegionLabeler, _ = labeler.NewRegionLabeler(ctx, storage.NewStorageWithMemoryBackend(), time.Second*5) - return clus + c.PersistOptions.SetClusterVersion(versioninfo.MinSupportedVersion(versioninfo.HotScheduleWithQuery)) + c.RegionLabeler, _ = labeler.NewRegionLabeler(ctx, c.Storage, time.Second*5) + return c } // GetStoreConfig returns the store config. -func (mc *Cluster) GetStoreConfig() sc.StoreConfig { +func (mc *Cluster) GetStoreConfig() sc.StoreConfigProvider { return mc.StoreConfigManager.GetStoreConfig() } -// GetOpts returns the cluster configuration. -func (mc *Cluster) GetOpts() sc.Config { - return mc.PersistOptions +// GetCheckerConfig returns the checker config. +func (mc *Cluster) GetCheckerConfig() sc.CheckerConfigProvider { + return mc +} + +// GetSchedulerConfig returns the scheduler config. +func (mc *Cluster) GetSchedulerConfig() sc.SchedulerConfigProvider { + return mc +} + +// GetSharedConfig returns the shared config. +func (mc *Cluster) GetSharedConfig() sc.SharedConfigProvider { + return mc } -// GetAllocator returns the ID allocator. -func (mc *Cluster) GetAllocator() id.Allocator { - return mc.IDAllocator +// GetStorage returns the storage. +func (mc *Cluster) GetStorage() storage.Storage { + return mc.Storage } -// ScanRegions scans region with start key, until number greater than limit. -func (mc *Cluster) ScanRegions(startKey, endKey []byte, limit int) []*core.RegionInfo { - return mc.ScanRange(startKey, endKey, limit) +// AllocID returns a new unique ID. +func (mc *Cluster) AllocID() (uint64, error) { + return mc.IDAllocator.Alloc() } +// GetPersistOptions returns the persist options. +func (mc *Cluster) GetPersistOptions() *config.PersistOptions { + return mc.PersistOptions +} + +// UpdateRegionsLabelLevelStats updates the label level stats for the regions. +func (mc *Cluster) UpdateRegionsLabelLevelStats(regions []*core.RegionInfo) {} + // LoadRegion puts region info without leader func (mc *Cluster) LoadRegion(regionID uint64, peerStoreIDs ...uint64) { // regions load from etcd will have no leader @@ -137,8 +157,8 @@ func (mc *Cluster) RegionReadStats() map[uint64][]*statistics.HotPeerStat { } // BucketsStats returns hot region's buckets stats. -func (mc *Cluster) BucketsStats(degree int) map[uint64][]*buckets.BucketStat { - task := buckets.NewCollectBucketStatsTask(degree) +func (mc *Cluster) BucketsStats(degree int, regions ...uint64) map[uint64][]*buckets.BucketStat { + task := buckets.NewCollectBucketStatsTask(degree, regions...) if !mc.HotBucketCache.CheckAsync(task) { return nil } @@ -174,7 +194,7 @@ func hotRegionsFromStore(w *statistics.HotCache, storeID uint64, kind statistics // AllocPeer allocs a new peer on a store. func (mc *Cluster) AllocPeer(storeID uint64) (*metapb.Peer, error) { - peerID, err := mc.GetAllocator().Alloc() + peerID, err := mc.AllocID() if err != nil { log.Error("failed to alloc peer", errs.ZapError(err)) return nil, err @@ -188,7 +208,7 @@ func (mc *Cluster) AllocPeer(storeID uint64) (*metapb.Peer, error) { func (mc *Cluster) initRuleManager() { if mc.RuleManager == nil { - mc.RuleManager = placement.NewRuleManager(storage.NewStorageWithMemoryBackend(), mc, mc.GetOpts()) + mc.RuleManager = placement.NewRuleManager(mc.GetStorage(), mc, mc.GetSharedConfig()) mc.RuleManager.Initialize(int(mc.GetReplicationConfig().MaxReplicas), mc.GetReplicationConfig().LocationLabels) } } @@ -252,6 +272,22 @@ func (mc *Cluster) SetStoreBusy(storeID uint64, busy bool) { mc.PutStore(newStore) } +// BuryStore marks a store as tombstone in cluster. +func (mc *Cluster) BuryStore(storeID uint64, forceBury bool) error { + store := mc.GetStore(storeID) + if store.IsUp() { + if !forceBury { + return errs.ErrStoreIsUp.FastGenByArgs() + } else if !store.IsDisconnected() { + return errors.Errorf("The store %v is not offline nor disconnected", storeID) + } + } + + newStore := store.Clone(core.TombstoneStore()) + mc.PutStore(newStore) + return nil +} + // AddLeaderStore adds store with specified count of leader. func (mc *Cluster) AddLeaderStore(storeID uint64, leaderCount int, leaderSizes ...int64) { stats := &pdpb.StoreStats{} @@ -278,7 +314,13 @@ func (mc *Cluster) AddLeaderStore(storeID uint64, leaderCount int, leaderSizes . } // AddRegionStore adds store with specified count of region. -func (mc *Cluster) AddRegionStore(storeID uint64, regionCount int) { +func (mc *Cluster) AddRegionStore(storeID uint64, regionCount int, regionSizes ...uint64) { + var regionSize uint64 + if len(regionSizes) == 0 { + regionSize = uint64(int64(regionCount) * defaultRegionSize / units.MiB) + } else { + regionSize = regionSizes[0] + } stats := &pdpb.StoreStats{} stats.Capacity = defaultStoreCapacity stats.UsedSize = uint64(regionCount) * defaultRegionSize @@ -292,8 +334,8 @@ func (mc *Cluster) AddRegionStore(storeID uint64, regionCount int) { }}, core.SetStoreStats(stats), core.SetRegionCount(regionCount), - core.SetRegionSize(int64(regionCount)*defaultRegionSize/units.MiB), core.SetLastHeartbeatTS(time.Now()), + core.SetRegionSize(int64(regionSize)), ) mc.SetStoreLimit(storeID, storelimit.AddPeer, 60) mc.SetStoreLimit(storeID, storelimit.RemovePeer, 60) @@ -325,7 +367,7 @@ func (mc *Cluster) AddRegionStoreWithLeader(storeID uint64, regionCount int, lea } mc.AddRegionStore(storeID, regionCount) for i := 0; i < leaderCount; i++ { - id, _ := mc.GetAllocator().Alloc() + id, _ := mc.AllocID() mc.AddLeaderRegion(id, storeID) } } @@ -355,6 +397,13 @@ func (mc *Cluster) AddLabelsStore(storeID uint64, regionCount int, labels map[st mc.PutStore(store) } +// AddLabersStoreWithLearnerCount adds store with specified count of region, learner and labels. +func (mc *Cluster) AddLabersStoreWithLearnerCount(storeID uint64, regionCount int, learnerCount int, labels map[string]string) { + mc.AddLabelsStore(storeID, regionCount, labels) + store := mc.GetStore(storeID).Clone(core.SetLearnerCount(learnerCount)) + mc.PutStore(store) +} + // AddLeaderRegion adds region with specified leader and followers. func (mc *Cluster) AddLeaderRegion(regionID uint64, leaderStoreID uint64, otherPeerStoreIDs ...uint64) *core.RegionInfo { origin := mc.newMockRegionInfo(regionID, leaderStoreID, otherPeerStoreIDs...) @@ -508,6 +557,11 @@ func (mc *Cluster) AddLeaderRegionWithWriteInfo( return items } +// DropCacheAllRegion removes all regions from the cache. +func (mc *Cluster) DropCacheAllRegion() { + mc.ResetRegionCache() +} + // UpdateStoreLeaderWeight updates store leader weight. func (mc *Cluster) UpdateStoreLeaderWeight(storeID uint64, weight float64) { store := mc.GetStore(storeID) @@ -758,19 +812,14 @@ func (mc *Cluster) PutStoreWithLabels(id uint64, labelPairs ...string) { mc.AddLabelsStore(id, 0, labels) } -// RemoveScheduler mocks method. -func (mc *Cluster) RemoveScheduler(name string) error { - return nil -} - // MockRegionInfo returns a mock region // If leaderStoreID is zero, the regions would have no leader func (mc *Cluster) MockRegionInfo(regionID uint64, leaderStoreID uint64, followerStoreIDs, learnerStoreIDs []uint64, epoch *metapb.RegionEpoch) *core.RegionInfo { region := &metapb.Region{ Id: regionID, - StartKey: []byte(fmt.Sprintf("%20d", regionID)), - EndKey: []byte(fmt.Sprintf("%20d", regionID+1)), + StartKey: []byte(fmt.Sprintf("%20d0", regionID)), + EndKey: []byte(fmt.Sprintf("%20d0", regionID+1)), RegionEpoch: epoch, } var leader *metapb.Peer @@ -808,10 +857,6 @@ func (mc *Cluster) AddSuspectRegions(ids ...uint64) { } } -// SetHotPendingInfluenceMetrics mock method -func (mc *Cluster) SetHotPendingInfluenceMetrics(storeLabel, rwTy, dim string, load float64) { -} - // GetBasicCluster mock method func (mc *Cluster) GetBasicCluster() *core.BasicCluster { return mc.BasicCluster @@ -895,6 +940,3 @@ func (mc *Cluster) ObserveRegionsStats() { storeIDs, writeBytesRates, writeKeysRates := mc.BasicCluster.GetStoresWriteRate() mc.HotStat.ObserveRegionsStats(storeIDs, writeBytesRates, writeKeysRates) } - -// RecordOpStepWithTTL records OpStep with TTL -func (mc *Cluster) RecordOpStepWithTTL(regionID uint64) {} diff --git a/pkg/mock/mockconfig/mockconfig.go b/pkg/mock/mockconfig/mockconfig.go index c412e97ab80..0516f9cd467 100644 --- a/pkg/mock/mockconfig/mockconfig.go +++ b/pkg/mock/mockconfig/mockconfig.go @@ -22,7 +22,7 @@ import ( // NewTestOptions creates default options for testing. func NewTestOptions() *config.PersistOptions { // register default schedulers in case config check fail. - for _, d := range config.DefaultSchedulers { + for _, d := range sc.DefaultSchedulers { sc.RegisterScheduler(d.Type) } c := config.NewConfig() diff --git a/pkg/movingaverage/avg_over_time.go b/pkg/movingaverage/avg_over_time.go index 227ee32e8c7..664466fe98c 100644 --- a/pkg/movingaverage/avg_over_time.go +++ b/pkg/movingaverage/avg_over_time.go @@ -32,7 +32,7 @@ type deltaWithInterval struct { // then calculates the change rate by (sum of changes) / (sum of intervals). type AvgOverTime struct { que *queue.Queue // The element is `deltaWithInterval`, sum of all elements' interval is less than `avgInterval` - margin deltaWithInterval // The last element from `PopFront` in `que` + margin deltaWithInterval // The last element from `PopFront` in `que`, it will be changed if the sum exclude it is greater than `avgInterval` deltaSum float64 // Including `margin` and all elements in `que` intervalSum time.Duration // Including `margin` and all elements in `que` avgInterval time.Duration @@ -57,6 +57,7 @@ func (aot *AvgOverTime) Get() float64 { if aot.intervalSum < aot.avgInterval { return 0 } + marginDelta := aot.margin.delta * (aot.intervalSum.Seconds() - aot.avgInterval.Seconds()) / aot.margin.interval.Seconds() return (aot.deltaSum - marginDelta) / aot.avgInterval.Seconds() } @@ -75,6 +76,11 @@ func (aot *AvgOverTime) Clear() { } // Add adds recent change to AvgOverTime. +// It will pop item until the retain item's sum is greater than avgInterval. +// such as: +// que [1,1,1,6], avgInterval is 5. +// It will pop 6 if adding 2, the retaining item's sum is 5(2,1,1,1) >= avgInterval. +// It can't pop 6 if adding 1, the retaining item's sum is 4(1,1,1,1) < avgInterval. func (aot *AvgOverTime) Add(delta float64, interval time.Duration) { if interval == 0 { return @@ -135,7 +141,11 @@ func (aot *AvgOverTime) GetIntervalSum() time.Duration { // GetInstantaneous returns the value just added. func (aot *AvgOverTime) GetInstantaneous() float64 { if aot.que.Len() == 0 || aot.que.Back() == nil { - return 0 + if aot.margin.interval != 0 { + return aot.margin.delta / aot.margin.interval.Seconds() + } + return aot.margin.delta } - return aot.que.Back().(deltaWithInterval).delta + data := aot.que.Back().(deltaWithInterval) + return data.delta / data.interval.Seconds() } diff --git a/pkg/movingaverage/avg_over_time_test.go b/pkg/movingaverage/avg_over_time_test.go index a0787f5af81..43553d9d608 100644 --- a/pkg/movingaverage/avg_over_time_test.go +++ b/pkg/movingaverage/avg_over_time_test.go @@ -42,6 +42,20 @@ func TestPulse(t *testing.T) { } } +func TestPulse2(t *testing.T) { + t.Parallel() + re := require.New(t) + dur := 5 * time.Second + aot := NewAvgOverTime(dur) + re.Equal(float64(0), aot.GetInstantaneous()) + aot.Add(1000, dur) + re.Equal(float64(1000)/dur.Seconds(), aot.GetInstantaneous()) + re.True(aot.IsFull()) + aot.Clear() + aot.Add(1000, dur) + re.Equal(float64(1000)/dur.Seconds(), aot.GetInstantaneous()) +} + func TestChange(t *testing.T) { t.Parallel() re := require.New(t) diff --git a/pkg/movingaverage/moving_average_test.go b/pkg/movingaverage/moving_average_test.go index f258b6d1b58..77782869ce6 100644 --- a/pkg/movingaverage/moving_average_test.go +++ b/pkg/movingaverage/moving_average_test.go @@ -25,9 +25,9 @@ import ( ) func addRandData(ma MovingAvg, n int, mx float64) { - rand.New(rand.NewSource(time.Now().UnixNano())) + r := rand.New(rand.NewSource(time.Now().UnixNano())) for i := 0; i < n; i++ { - ma.Add(rand.Float64() * mx) + ma.Add(r.Float64() * mx) } } diff --git a/server/replication/metrics.go b/pkg/replication/metrics.go similarity index 100% rename from server/replication/metrics.go rename to pkg/replication/metrics.go diff --git a/server/replication/replication_mode.go b/pkg/replication/replication_mode.go similarity index 96% rename from server/replication/replication_mode.go rename to pkg/replication/replication_mode.go index 52978c78fa9..703dad03d26 100644 --- a/server/replication/replication_mode.go +++ b/pkg/replication/replication_mode.go @@ -29,7 +29,7 @@ import ( "github.com/pingcap/log" "github.com/tikv/pd/pkg/core" "github.com/tikv/pd/pkg/errs" - "github.com/tikv/pd/pkg/schedule" + sche "github.com/tikv/pd/pkg/schedule/core" "github.com/tikv/pd/pkg/slice" "github.com/tikv/pd/pkg/storage/endpoint" "github.com/tikv/pd/pkg/utils/logutil" @@ -62,7 +62,7 @@ type FileReplicater interface { } const drStatusFile = "DR_STATE" -const persistFileTimeout = time.Second * 10 +const persistFileTimeout = time.Second * 3 // ModeManager is used to control how raft logs are synchronized between // different tikv nodes. @@ -72,7 +72,7 @@ type ModeManager struct { syncutil.RWMutex config config.ReplicationModeConfig storage endpoint.ReplicationStatusStorage - cluster schedule.Cluster + cluster sche.ClusterInformer fileReplicater FileReplicater replicatedMembers []uint64 @@ -91,7 +91,7 @@ type ModeManager struct { } // NewReplicationModeManager creates the replicate mode manager. -func NewReplicationModeManager(config config.ReplicationModeConfig, storage endpoint.ReplicationStatusStorage, cluster schedule.Cluster, fileReplicater FileReplicater) (*ModeManager, error) { +func NewReplicationModeManager(config config.ReplicationModeConfig, storage endpoint.ReplicationStatusStorage, cluster sche.ClusterInformer, fileReplicater FileReplicater) (*ModeManager, error) { m := &ModeManager{ initTime: time.Now(), config: config, @@ -234,7 +234,7 @@ func (m *ModeManager) drSwitchToAsyncWait(availableStores []uint64) error { m.Lock() defer m.Unlock() - id, err := m.cluster.GetAllocator().Alloc() + id, err := m.cluster.AllocID() if err != nil { log.Warn("failed to switch to async wait state", zap.String("replicate-mode", modeDRAutoSync), errs.ZapError(err)) return err @@ -257,7 +257,7 @@ func (m *ModeManager) drSwitchToAsync(availableStores []uint64) error { } func (m *ModeManager) drSwitchToAsyncWithLock(availableStores []uint64) error { - id, err := m.cluster.GetAllocator().Alloc() + id, err := m.cluster.AllocID() if err != nil { log.Warn("failed to switch to async state", zap.String("replicate-mode", modeDRAutoSync), errs.ZapError(err)) return err @@ -280,7 +280,7 @@ func (m *ModeManager) drSwitchToSyncRecover() error { } func (m *ModeManager) drSwitchToSyncRecoverWithLock() error { - id, err := m.cluster.GetAllocator().Alloc() + id, err := m.cluster.AllocID() if err != nil { log.Warn("failed to switch to sync_recover state", zap.String("replicate-mode", modeDRAutoSync), errs.ZapError(err)) return err @@ -301,7 +301,7 @@ func (m *ModeManager) drSwitchToSyncRecoverWithLock() error { func (m *ModeManager) drSwitchToSync() error { m.Lock() defer m.Unlock() - id, err := m.cluster.GetAllocator().Alloc() + id, err := m.cluster.AllocID() if err != nil { log.Warn("failed to switch to sync state", zap.String("replicate-mode", modeDRAutoSync), errs.ZapError(err)) return err @@ -375,14 +375,18 @@ const ( // Run starts the background job. func (m *ModeManager) Run(ctx context.Context) { // Wait for a while when just start, in case tikv do not connect in time. + timer := time.NewTimer(idleTimeout) + defer timer.Stop() select { - case <-time.After(idleTimeout): + case <-timer.C: case <-ctx.Done(): return } + ticker := time.NewTicker(tickInterval) + defer ticker.Stop() for { select { - case <-time.After(tickInterval): + case <-ticker.C: case <-ctx.Done(): return } @@ -503,6 +507,10 @@ func (m *ModeManager) checkStoreStatus() [][]uint64 { if s.IsRemoved() { continue } + // learner peers do not participate in major commit or vote, so it should not count in primary/dr as a normal store. + if s.GetRegionCount() == s.GetLearnerCount() { + continue + } down := s.DownTime() >= m.config.DRAutoSync.WaitStoreTimeout.Duration labelValue := s.GetLabelValue(m.config.DRAutoSync.LabelKey) if labelValue == m.config.DRAutoSync.Primary { @@ -601,7 +609,7 @@ func (m *ModeManager) updateProgress() { key = r.GetEndKey() } m.drSampleTotalRegion = len(sampleRegions) - m.drTotalRegion = m.cluster.GetRegionCount() + m.drTotalRegion = m.cluster.GetTotalRegionCount() return } } diff --git a/server/replication/replication_mode_test.go b/pkg/replication/replication_mode_test.go similarity index 98% rename from server/replication/replication_mode_test.go rename to pkg/replication/replication_mode_test.go index 499c40b0412..09456893eb0 100644 --- a/server/replication/replication_mode_test.go +++ b/pkg/replication/replication_mode_test.go @@ -168,7 +168,7 @@ func TestStateSwitch(t *testing.T) { Primary: "zone1", DR: "zone2", PrimaryReplicas: 4, - DRReplicas: 1, + DRReplicas: 2, WaitStoreTimeout: typeutil.Duration{Duration: time.Minute}, }} cluster := mockcluster.NewCluster(ctx, mockconfig.NewTestOptions()) @@ -215,7 +215,7 @@ func TestStateSwitch(t *testing.T) { // add new store in dr zone. cluster.AddLabelsStore(5, 1, map[string]string{"zone": "zone2"}) - cluster.AddLabelsStore(6, 1, map[string]string{"zone": "zone2"}) + cluster.AddLabersStoreWithLearnerCount(6, 1, 1, map[string]string{"zone": "zone2"}) // async -> sync rep.tickDR() re.Equal(drStateSyncRecover, rep.drGetState()) @@ -234,10 +234,14 @@ func TestStateSwitch(t *testing.T) { rep.tickDR() re.Equal(drStateSync, rep.drGetState()) // cannot guarantee majority, keep sync. + setStoreState(cluster, "up", "up", "up", "up", "up", "down") + rep.tickDR() + re.Equal(drStateSync, rep.drGetState()) + + // once the voter node down, even learner node up, swith to async state. setStoreState(cluster, "up", "up", "up", "up", "down", "up") rep.tickDR() re.Equal(drStateAsyncWait, rep.drGetState()) - assertStateIDUpdate() rep.drSwitchToSync() replicator.errors[2] = errors.New("fail to replicate") diff --git a/pkg/schedule/checker/checker_controller.go b/pkg/schedule/checker/checker_controller.go index 7d2fda4a894..68b794f417a 100644 --- a/pkg/schedule/checker/checker_controller.go +++ b/pkg/schedule/checker/checker_controller.go @@ -22,8 +22,8 @@ import ( "github.com/tikv/pd/pkg/cache" "github.com/tikv/pd/pkg/core" "github.com/tikv/pd/pkg/errs" - "github.com/tikv/pd/pkg/schedule" "github.com/tikv/pd/pkg/schedule/config" + sche "github.com/tikv/pd/pkg/schedule/core" "github.com/tikv/pd/pkg/schedule/labeler" "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/placement" @@ -33,11 +33,13 @@ import ( // DefaultCacheSize is the default length of waiting list. const DefaultCacheSize = 1000 +var denyCheckersByLabelerCounter = labeler.LabelerEventCounter.WithLabelValues("checkers", "deny") + // Controller is used to manage all checkers. type Controller struct { - cluster schedule.Cluster - conf config.Config - opController *schedule.OperatorController + cluster sche.CheckerCluster + conf config.CheckerConfigProvider + opController *operator.Controller learnerChecker *LearnerChecker replicaChecker *ReplicaChecker ruleChecker *RuleChecker @@ -51,7 +53,7 @@ type Controller struct { } // NewController create a new Controller. -func NewController(ctx context.Context, cluster schedule.Cluster, conf config.Config, ruleManager *placement.RuleManager, labeler *labeler.RegionLabeler, opController *schedule.OperatorController) *Controller { +func NewController(ctx context.Context, cluster sche.CheckerCluster, conf config.CheckerConfigProvider, ruleManager *placement.RuleManager, labeler *labeler.RegionLabeler, opController *operator.Controller) *Controller { regionWaitingList := cache.NewDefaultCache(DefaultCacheSize) return &Controller{ cluster: cluster, @@ -72,7 +74,7 @@ func NewController(ctx context.Context, cluster schedule.Cluster, conf config.Co // CheckRegion will check the region and add a new operator if needed. func (c *Controller) CheckRegion(region *core.RegionInfo) []*operator.Operator { - // If PD has restarted, it need to check learners added before and promote them. + // If PD has restarted, it needs to check learners added before and promote them. // Don't check isRaftLearnerEnabled cause it maybe disable learner feature but there are still some learners to promote. opController := c.opController @@ -80,19 +82,12 @@ func (c *Controller) CheckRegion(region *core.RegionInfo) []*operator.Operator { return []*operator.Operator{op} } - if cl, ok := c.cluster.(interface{ GetRegionLabeler() *labeler.RegionLabeler }); ok { - l := cl.GetRegionLabeler() - if l.ScheduleDisabled(region) { - return nil - } - } - if op := c.splitChecker.Check(region); op != nil { return []*operator.Operator{op} } if c.conf.IsPlacementRulesEnabled() { - skipRuleCheck := c.cluster.GetOpts().IsPlacementRulesCacheEnabled() && + skipRuleCheck := c.cluster.GetCheckerConfig().IsPlacementRulesCacheEnabled() && c.cluster.GetRuleManager().IsRegionFitCached(c.cluster, region) if skipRuleCheck { // If the fit is fetched from cache, it seems that the region doesn't need check @@ -125,6 +120,15 @@ func (c *Controller) CheckRegion(region *core.RegionInfo) []*operator.Operator { c.regionWaitingList.Put(region.GetID(), nil) } } + // skip the joint checker, split checker and rule checker when region label is set to "schedule=deny". + // those checkers are help to make region health, it's necessary to skip them when region is set to deny. + if cl, ok := c.cluster.(interface{ GetRegionLabeler() *labeler.RegionLabeler }); ok { + l := cl.GetRegionLabeler() + if l.ScheduleDisabled(region) { + denyCheckersByLabelerCounter.Inc() + return nil + } + } if c.mergeChecker != nil { allowed := opController.OperatorCount(operator.OpMerge) < c.conf.GetMergeScheduleLimit() diff --git a/pkg/schedule/checker/joint_state_checker.go b/pkg/schedule/checker/joint_state_checker.go index 83a463b24fa..872c957f0a7 100644 --- a/pkg/schedule/checker/joint_state_checker.go +++ b/pkg/schedule/checker/joint_state_checker.go @@ -19,14 +19,14 @@ import ( "github.com/tikv/pd/pkg/core" "github.com/tikv/pd/pkg/core/constant" "github.com/tikv/pd/pkg/errs" - "github.com/tikv/pd/pkg/schedule" + sche "github.com/tikv/pd/pkg/schedule/core" "github.com/tikv/pd/pkg/schedule/operator" ) // JointStateChecker ensures region is in joint state will leave. type JointStateChecker struct { PauseController - cluster schedule.Cluster + cluster sche.CheckerCluster } const jointStateCheckerName = "joint_state_checker" @@ -41,7 +41,7 @@ var ( ) // NewJointStateChecker creates a joint state checker. -func NewJointStateChecker(cluster schedule.Cluster) *JointStateChecker { +func NewJointStateChecker(cluster sche.CheckerCluster) *JointStateChecker { return &JointStateChecker{ cluster: cluster, } diff --git a/pkg/schedule/checker/learner_checker.go b/pkg/schedule/checker/learner_checker.go index 04d2545c52b..c6f7d671ac3 100644 --- a/pkg/schedule/checker/learner_checker.go +++ b/pkg/schedule/checker/learner_checker.go @@ -18,14 +18,14 @@ import ( "github.com/pingcap/log" "github.com/tikv/pd/pkg/core" "github.com/tikv/pd/pkg/errs" - "github.com/tikv/pd/pkg/schedule" + sche "github.com/tikv/pd/pkg/schedule/core" "github.com/tikv/pd/pkg/schedule/operator" ) // LearnerChecker ensures region has a learner will be promoted. type LearnerChecker struct { PauseController - cluster schedule.Cluster + cluster sche.CheckerCluster } var ( @@ -34,7 +34,7 @@ var ( ) // NewLearnerChecker creates a learner checker. -func NewLearnerChecker(cluster schedule.Cluster) *LearnerChecker { +func NewLearnerChecker(cluster sche.CheckerCluster) *LearnerChecker { return &LearnerChecker{ cluster: cluster, } diff --git a/pkg/schedule/checker/merge_checker.go b/pkg/schedule/checker/merge_checker.go index 2ebc8537b4a..0243bfbe165 100644 --- a/pkg/schedule/checker/merge_checker.go +++ b/pkg/schedule/checker/merge_checker.go @@ -25,8 +25,8 @@ import ( "github.com/tikv/pd/pkg/core" "github.com/tikv/pd/pkg/core/constant" "github.com/tikv/pd/pkg/errs" - "github.com/tikv/pd/pkg/schedule" "github.com/tikv/pd/pkg/schedule/config" + sche "github.com/tikv/pd/pkg/schedule/core" "github.com/tikv/pd/pkg/schedule/filter" "github.com/tikv/pd/pkg/schedule/labeler" "github.com/tikv/pd/pkg/schedule/operator" @@ -76,14 +76,14 @@ var ( // MergeChecker ensures region to merge with adjacent region when size is small type MergeChecker struct { PauseController - cluster schedule.Cluster - conf config.Config + cluster sche.CheckerCluster + conf config.CheckerConfigProvider splitCache *cache.TTLUint64 startTime time.Time // it's used to judge whether server recently start. } // NewMergeChecker creates a merge checker. -func NewMergeChecker(ctx context.Context, cluster schedule.Cluster, conf config.Config) *MergeChecker { +func NewMergeChecker(ctx context.Context, cluster sche.CheckerCluster, conf config.CheckerConfigProvider) *MergeChecker { splitCache := cache.NewIDTTL(ctx, time.Minute, conf.GetSplitMergeInterval()) return &MergeChecker{ cluster: cluster, @@ -250,7 +250,7 @@ func (m *MergeChecker) checkTarget(region, adjacent *core.RegionInfo) bool { } // AllowMerge returns true if two regions can be merged according to the key type. -func AllowMerge(cluster schedule.Cluster, region, adjacent *core.RegionInfo) bool { +func AllowMerge(cluster sche.SharedCluster, region, adjacent *core.RegionInfo) bool { var start, end []byte if bytes.Equal(region.GetEndKey(), adjacent.GetStartKey()) && len(region.GetEndKey()) != 0 { start, end = region.GetStartKey(), adjacent.GetEndKey() @@ -266,7 +266,7 @@ func AllowMerge(cluster schedule.Cluster, region, adjacent *core.RegionInfo) boo // We can consider using dependency injection techniques to optimize in // the future. - if cluster.GetOpts().IsPlacementRulesEnabled() { + if cluster.GetSharedConfig().IsPlacementRulesEnabled() { cl, ok := cluster.(interface{ GetRuleManager() *placement.RuleManager }) if !ok || len(cl.GetRuleManager().GetSplitKeys(start, end)) > 0 { return false @@ -283,10 +283,10 @@ func AllowMerge(cluster schedule.Cluster, region, adjacent *core.RegionInfo) boo } } - policy := cluster.GetOpts().GetKeyType() + policy := cluster.GetSharedConfig().GetKeyType() switch policy { case constant.Table: - if cluster.GetOpts().IsCrossTableMergeEnabled() { + if cluster.GetSharedConfig().IsCrossTableMergeEnabled() { return true } return isTableIDSame(region, adjacent) @@ -306,7 +306,7 @@ func isTableIDSame(region, adjacent *core.RegionInfo) bool { // Check whether there is a peer of the adjacent region on an offline store, // while the source region has no peer on it. This is to prevent from bringing // any other peer into an offline store to slow down the offline process. -func checkPeerStore(cluster schedule.Cluster, region, adjacent *core.RegionInfo) bool { +func checkPeerStore(cluster sche.SharedCluster, region, adjacent *core.RegionInfo) bool { regionStoreIDs := region.GetStoreIDs() for _, peer := range adjacent.GetPeers() { storeID := peer.GetStoreId() diff --git a/pkg/schedule/checker/merge_checker_test.go b/pkg/schedule/checker/merge_checker_test.go index 2ce871f65de..6478eb0b2c4 100644 --- a/pkg/schedule/checker/merge_checker_test.go +++ b/pkg/schedule/checker/merge_checker_test.go @@ -26,12 +26,12 @@ import ( "github.com/tikv/pd/pkg/core/storelimit" "github.com/tikv/pd/pkg/mock/mockcluster" "github.com/tikv/pd/pkg/mock/mockconfig" - "github.com/tikv/pd/pkg/schedule" "github.com/tikv/pd/pkg/schedule/config" "github.com/tikv/pd/pkg/schedule/hbstream" "github.com/tikv/pd/pkg/schedule/labeler" "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/placement" + "github.com/tikv/pd/pkg/utils/operatorutil" "github.com/tikv/pd/pkg/utils/testutil" "github.com/tikv/pd/pkg/versioninfo" "go.uber.org/goleak" @@ -80,7 +80,7 @@ func (suite *mergeCheckerTestSuite) SetupTest() { for _, region := range suite.regions { suite.cluster.PutRegion(region) } - suite.mc = NewMergeChecker(suite.ctx, suite.cluster, suite.cluster.GetOpts()) + suite.mc = NewMergeChecker(suite.ctx, suite.cluster, suite.cluster.GetCheckerConfig()) } func (suite *mergeCheckerTestSuite) TearDownTest() { @@ -251,7 +251,7 @@ func (suite *mergeCheckerTestSuite) TestMatchPeers() { // partial store overlap not including leader ops := suite.mc.Check(suite.regions[2]) suite.NotNil(ops) - testutil.CheckSteps(suite.Require(), ops[0], []operator.OpStep{ + operatorutil.CheckSteps(suite.Require(), ops[0], []operator.OpStep{ operator.AddLearner{ToStore: 1}, operator.PromoteLearner{ToStore: 1}, operator.RemovePeer{FromStore: 2}, @@ -265,7 +265,7 @@ func (suite *mergeCheckerTestSuite) TestMatchPeers() { IsPassive: false, }, }) - testutil.CheckSteps(suite.Require(), ops[1], []operator.OpStep{ + operatorutil.CheckSteps(suite.Require(), ops[1], []operator.OpStep{ operator.MergeRegion{ FromRegion: suite.regions[2].GetMeta(), ToRegion: suite.regions[1].GetMeta(), @@ -285,7 +285,7 @@ func (suite *mergeCheckerTestSuite) TestMatchPeers() { suite.regions[2] = newRegion suite.cluster.PutRegion(suite.regions[2]) ops = suite.mc.Check(suite.regions[2]) - testutil.CheckSteps(suite.Require(), ops[0], []operator.OpStep{ + operatorutil.CheckSteps(suite.Require(), ops[0], []operator.OpStep{ operator.AddLearner{ToStore: 4}, operator.PromoteLearner{ToStore: 4}, operator.RemovePeer{FromStore: 6}, @@ -295,7 +295,7 @@ func (suite *mergeCheckerTestSuite) TestMatchPeers() { IsPassive: false, }, }) - testutil.CheckSteps(suite.Require(), ops[1], []operator.OpStep{ + operatorutil.CheckSteps(suite.Require(), ops[1], []operator.OpStep{ operator.MergeRegion{ FromRegion: suite.regions[2].GetMeta(), ToRegion: suite.regions[1].GetMeta(), @@ -311,14 +311,14 @@ func (suite *mergeCheckerTestSuite) TestMatchPeers() { })) suite.cluster.PutRegion(suite.regions[2]) ops = suite.mc.Check(suite.regions[2]) - testutil.CheckSteps(suite.Require(), ops[0], []operator.OpStep{ + operatorutil.CheckSteps(suite.Require(), ops[0], []operator.OpStep{ operator.MergeRegion{ FromRegion: suite.regions[2].GetMeta(), ToRegion: suite.regions[1].GetMeta(), IsPassive: false, }, }) - testutil.CheckSteps(suite.Require(), ops[1], []operator.OpStep{ + operatorutil.CheckSteps(suite.Require(), ops[1], []operator.OpStep{ operator.MergeRegion{ FromRegion: suite.regions[2].GetMeta(), ToRegion: suite.regions[1].GetMeta(), @@ -334,7 +334,7 @@ func (suite *mergeCheckerTestSuite) TestMatchPeers() { }), core.WithLeader(&metapb.Peer{Id: 109, StoreId: 2})) suite.cluster.PutRegion(suite.regions[2]) ops = suite.mc.Check(suite.regions[2]) - testutil.CheckSteps(suite.Require(), ops[0], []operator.OpStep{ + operatorutil.CheckSteps(suite.Require(), ops[0], []operator.OpStep{ operator.AddLearner{ToStore: 1}, operator.PromoteLearner{ToStore: 1}, operator.RemovePeer{FromStore: 3}, @@ -351,7 +351,7 @@ func (suite *mergeCheckerTestSuite) TestMatchPeers() { IsPassive: false, }, }) - testutil.CheckSteps(suite.Require(), ops[1], []operator.OpStep{ + operatorutil.CheckSteps(suite.Require(), ops[1], []operator.OpStep{ operator.MergeRegion{ FromRegion: suite.regions[2].GetMeta(), ToRegion: suite.regions[1].GetMeta(), @@ -370,7 +370,7 @@ func (suite *mergeCheckerTestSuite) TestMatchPeers() { ) suite.cluster.PutRegion(suite.regions[1]) ops = suite.mc.Check(suite.regions[2]) - testutil.CheckSteps(suite.Require(), ops[0], []operator.OpStep{ + operatorutil.CheckSteps(suite.Require(), ops[0], []operator.OpStep{ operator.AddLearner{ToStore: 1}, operator.PromoteLearner{ToStore: 1}, operator.RemovePeer{FromStore: 3}, @@ -390,7 +390,7 @@ func (suite *mergeCheckerTestSuite) TestMatchPeers() { IsPassive: false, }, }) - testutil.CheckSteps(suite.Require(), ops[1], []operator.OpStep{ + operatorutil.CheckSteps(suite.Require(), ops[1], []operator.OpStep{ operator.MergeRegion{ FromRegion: suite.regions[2].GetMeta(), ToRegion: suite.regions[1].GetMeta(), @@ -417,7 +417,7 @@ func (suite *mergeCheckerTestSuite) TestMatchPeers() { ) suite.cluster.PutRegion(suite.regions[1]) ops = suite.mc.Check(suite.regions[2]) - testutil.CheckSteps(suite.Require(), ops[0], []operator.OpStep{ + operatorutil.CheckSteps(suite.Require(), ops[0], []operator.OpStep{ operator.AddLearner{ToStore: 1}, operator.PromoteLearner{ToStore: 1}, operator.RemovePeer{FromStore: 3}, @@ -431,7 +431,7 @@ func (suite *mergeCheckerTestSuite) TestMatchPeers() { IsPassive: false, }, }) - testutil.CheckSteps(suite.Require(), ops[1], []operator.OpStep{ + operatorutil.CheckSteps(suite.Require(), ops[1], []operator.OpStep{ operator.MergeRegion{ FromRegion: suite.regions[2].GetMeta(), ToRegion: suite.regions[1].GetMeta(), @@ -461,9 +461,9 @@ func (suite *mergeCheckerTestSuite) TestStoreLimitWithMerge() { tc.PutRegion(region) } - mc := NewMergeChecker(suite.ctx, tc, tc.GetOpts()) + mc := NewMergeChecker(suite.ctx, tc, tc.GetCheckerConfig()) stream := hbstream.NewTestHeartbeatStreams(suite.ctx, tc.ID, tc, false /* no need to run */) - oc := schedule.NewOperatorController(suite.ctx, tc, stream) + oc := operator.NewController(suite.ctx, tc.GetBasicCluster(), tc.GetSharedConfig(), stream) regions[2] = regions[2].Clone( core.SetPeers([]*metapb.Peer{ @@ -484,7 +484,7 @@ func (suite *mergeCheckerTestSuite) TestStoreLimitWithMerge() { suite.NotNil(ops) suite.True(oc.AddOperator(ops...)) for _, op := range ops { - oc.RemoveOperator(op) + oc.RemoveOperator(op, operator.ExceedStoreLimit) } } regions[2] = regions[2].Clone( @@ -498,7 +498,7 @@ func (suite *mergeCheckerTestSuite) TestStoreLimitWithMerge() { suite.NotNil(ops) suite.True(oc.AddOperator(ops...)) for _, op := range ops { - oc.RemoveOperator(op) + oc.RemoveOperator(op, operator.ExceedStoreLimit) } } { @@ -530,7 +530,7 @@ func (suite *mergeCheckerTestSuite) TestCache() { suite.cluster.PutRegion(region) } - suite.mc = NewMergeChecker(suite.ctx, suite.cluster, suite.cluster.GetOpts()) + suite.mc = NewMergeChecker(suite.ctx, suite.cluster, suite.cluster.GetCheckerConfig()) ops := suite.mc.Check(suite.regions[1]) suite.Nil(ops) diff --git a/pkg/schedule/checker/priority_inspector.go b/pkg/schedule/checker/priority_inspector.go index 98e8f4e36ad..0bcbcc810d0 100644 --- a/pkg/schedule/checker/priority_inspector.go +++ b/pkg/schedule/checker/priority_inspector.go @@ -19,8 +19,8 @@ import ( "github.com/tikv/pd/pkg/cache" "github.com/tikv/pd/pkg/core" - "github.com/tikv/pd/pkg/schedule" "github.com/tikv/pd/pkg/schedule/config" + sche "github.com/tikv/pd/pkg/schedule/core" "github.com/tikv/pd/pkg/schedule/placement" ) @@ -29,13 +29,13 @@ const defaultPriorityQueueSize = 1280 // PriorityInspector ensures high priority region should run first type PriorityInspector struct { - cluster schedule.Cluster - conf config.Config + cluster sche.CheckerCluster + conf config.CheckerConfigProvider queue *cache.PriorityQueue } // NewPriorityInspector creates a priority inspector. -func NewPriorityInspector(cluster schedule.Cluster, conf config.Config) *PriorityInspector { +func NewPriorityInspector(cluster sche.CheckerCluster, conf config.CheckerConfigProvider) *PriorityInspector { return &PriorityInspector{ cluster: cluster, conf: conf, @@ -104,9 +104,11 @@ func (p *PriorityInspector) addOrRemoveRegion(priority int, regionID uint64) { e := entry.Value.(*RegionPriorityEntry) e.Attempt++ e.Last = time.Now() + p.queue.Put(priority, e) + } else { + entry := NewRegionEntry(regionID) + p.queue.Put(priority, entry) } - entry := NewRegionEntry(regionID) - p.queue.Put(priority, entry) } else { p.queue.Remove(regionID) } diff --git a/pkg/schedule/checker/priority_inspector_test.go b/pkg/schedule/checker/priority_inspector_test.go index fc88b3e428f..5aef4c01158 100644 --- a/pkg/schedule/checker/priority_inspector_test.go +++ b/pkg/schedule/checker/priority_inspector_test.go @@ -37,7 +37,7 @@ func TestCheckPriorityRegions(t *testing.T) { tc.AddLeaderRegion(2, 2, 3) tc.AddLeaderRegion(3, 2) - pc := NewPriorityInspector(tc, tc.GetOpts()) + pc := NewPriorityInspector(tc, tc.GetCheckerConfig()) checkPriorityRegionTest(re, pc, tc) opt.SetPlacementRuleEnabled(true) re.True(opt.IsPlacementRulesEnabled()) @@ -47,7 +47,7 @@ func TestCheckPriorityRegions(t *testing.T) { func checkPriorityRegionTest(re *require.Assertions, pc *PriorityInspector, tc *mockcluster.Cluster) { // case1: inspect region 1, it doesn't lack replica region := tc.GetRegion(1) - opt := tc.GetOpts() + opt := tc.GetCheckerConfig() pc.Inspect(region) re.Equal(0, pc.queue.Len()) @@ -75,6 +75,13 @@ func checkPriorityRegionTest(re *require.Assertions, pc *PriorityInspector, tc * pc.Inspect(region) re.Equal(1, pc.queue.Len()) + // case5: inspect region 3 again + region = tc.GetRegion(3) + pc.Inspect(region) + time.Sleep(opt.GetPatrolRegionInterval() * 10) + ids = pc.GetPriorityRegions() + re.Empty(ids) + // recover tc.AddLeaderRegion(2, 2, 3) pc.RemovePriorityRegion(uint64(3)) diff --git a/pkg/schedule/checker/replica_checker.go b/pkg/schedule/checker/replica_checker.go index 1149d8ccd5e..3e23f3bdcac 100644 --- a/pkg/schedule/checker/replica_checker.go +++ b/pkg/schedule/checker/replica_checker.go @@ -23,8 +23,8 @@ import ( "github.com/tikv/pd/pkg/core" "github.com/tikv/pd/pkg/core/constant" "github.com/tikv/pd/pkg/errs" - "github.com/tikv/pd/pkg/schedule" "github.com/tikv/pd/pkg/schedule/config" + sche "github.com/tikv/pd/pkg/schedule/core" "github.com/tikv/pd/pkg/schedule/operator" "go.uber.org/zap" ) @@ -61,13 +61,13 @@ var ( // Location management, mainly used for cross data center deployment. type ReplicaChecker struct { PauseController - cluster schedule.Cluster - conf config.Config + cluster sche.CheckerCluster + conf config.CheckerConfigProvider regionWaitingList cache.Cache } // NewReplicaChecker creates a replica checker. -func NewReplicaChecker(cluster schedule.Cluster, conf config.Config, regionWaitingList cache.Cache) *ReplicaChecker { +func NewReplicaChecker(cluster sche.CheckerCluster, conf config.CheckerConfigProvider, regionWaitingList cache.Cache) *ReplicaChecker { return &ReplicaChecker{ cluster: cluster, conf: conf, diff --git a/pkg/schedule/checker/replica_checker_test.go b/pkg/schedule/checker/replica_checker_test.go index 454937bd3dd..a326d39d451 100644 --- a/pkg/schedule/checker/replica_checker_test.go +++ b/pkg/schedule/checker/replica_checker_test.go @@ -29,7 +29,7 @@ import ( "github.com/tikv/pd/pkg/mock/mockconfig" "github.com/tikv/pd/pkg/schedule/config" "github.com/tikv/pd/pkg/schedule/operator" - "github.com/tikv/pd/pkg/utils/testutil" + "github.com/tikv/pd/pkg/utils/operatorutil" "github.com/tikv/pd/pkg/versioninfo" ) @@ -50,7 +50,7 @@ func (suite *replicaCheckerTestSuite) SetupTest() { suite.ctx, suite.cancel = context.WithCancel(context.Background()) suite.cluster = mockcluster.NewCluster(suite.ctx, cfg) suite.cluster.SetClusterVersion(versioninfo.MinSupportedVersion(versioninfo.Version4_0)) - suite.rc = NewReplicaChecker(suite.cluster, suite.cluster.GetOpts(), cache.NewDefaultCache(10)) + suite.rc = NewReplicaChecker(suite.cluster, suite.cluster.GetCheckerConfig(), cache.NewDefaultCache(10)) stats := &pdpb.StoreStats{ Capacity: 100, Available: 100, @@ -162,7 +162,7 @@ func (suite *replicaCheckerTestSuite) TestDownPeer() { suite.NotNil(op) suite.Equal("remove-extra-down-replica", op.Desc()) - // down a peer,the number of peers(except learner) is not enough. + // down a peer, the number of peers(except learner) is not enough. op = suite.downPeerAndCheck(metapb.PeerRole_Learner) suite.NotNil(op) suite.Equal("replace-down-replica", op.Desc()) @@ -207,7 +207,7 @@ func (suite *replicaCheckerTestSuite) TestBasic() { tc := mockcluster.NewCluster(suite.ctx, opt) tc.SetMaxSnapshotCount(2) tc.SetClusterVersion(versioninfo.MinSupportedVersion(versioninfo.Version4_0)) - rc := NewReplicaChecker(tc, tc.GetOpts(), cache.NewDefaultCache(10)) + rc := NewReplicaChecker(tc, tc.GetCheckerConfig(), cache.NewDefaultCache(10)) // Add stores 1,2,3,4. tc.AddRegionStore(1, 4) @@ -219,7 +219,7 @@ func (suite *replicaCheckerTestSuite) TestBasic() { // Region has 2 peers, we need to add a new peer. region := tc.GetRegion(1) - testutil.CheckAddPeer(suite.Require(), rc.Check(region), operator.OpReplica, 4) + operatorutil.CheckAddPeer(suite.Require(), rc.Check(region), operator.OpReplica, 4) // Disable make up replica feature. tc.SetEnableMakeUpReplica(false) @@ -229,17 +229,17 @@ func (suite *replicaCheckerTestSuite) TestBasic() { // Test healthFilter. // If store 4 is down, we add to store 3. tc.SetStoreDown(4) - testutil.CheckAddPeer(suite.Require(), rc.Check(region), operator.OpReplica, 3) + operatorutil.CheckAddPeer(suite.Require(), rc.Check(region), operator.OpReplica, 3) tc.SetStoreUp(4) - testutil.CheckAddPeer(suite.Require(), rc.Check(region), operator.OpReplica, 4) + operatorutil.CheckAddPeer(suite.Require(), rc.Check(region), operator.OpReplica, 4) // Test snapshotCountFilter. // If snapshotCount > MaxSnapshotCount, we add to store 3. tc.UpdateSnapshotCount(4, 3) - testutil.CheckAddPeer(suite.Require(), rc.Check(region), operator.OpReplica, 3) + operatorutil.CheckAddPeer(suite.Require(), rc.Check(region), operator.OpReplica, 3) // If snapshotCount < MaxSnapshotCount, we can add peer again. tc.UpdateSnapshotCount(4, 1) - testutil.CheckAddPeer(suite.Require(), rc.Check(region), operator.OpReplica, 4) + operatorutil.CheckAddPeer(suite.Require(), rc.Check(region), operator.OpReplica, 4) // Add peer in store 4, and we have enough replicas. peer4, _ := tc.AllocPeer(4) @@ -249,7 +249,7 @@ func (suite *replicaCheckerTestSuite) TestBasic() { // Add peer in store 3, and we have redundant replicas. peer3, _ := tc.AllocPeer(3) region = region.Clone(core.WithAddPeer(peer3)) - testutil.CheckRemovePeer(suite.Require(), rc.Check(region), 1) + operatorutil.CheckRemovePeer(suite.Require(), rc.Check(region), 1) // Disable remove extra replica feature. tc.SetEnableRemoveExtraReplica(false) @@ -266,13 +266,13 @@ func (suite *replicaCheckerTestSuite) TestBasic() { } region = region.Clone(core.WithDownPeers(append(region.GetDownPeers(), downPeer))) - testutil.CheckTransferPeer(suite.Require(), rc.Check(region), operator.OpReplica, 2, 1) + operatorutil.CheckTransferPeer(suite.Require(), rc.Check(region), operator.OpReplica, 2, 1) region = region.Clone(core.WithDownPeers(nil)) suite.Nil(rc.Check(region)) // Peer in store 3 is offline, transfer peer to store 1. tc.SetStoreOffline(3) - testutil.CheckTransferPeer(suite.Require(), rc.Check(region), operator.OpReplica, 3, 1) + operatorutil.CheckTransferPeer(suite.Require(), rc.Check(region), operator.OpReplica, 3, 1) } func (suite *replicaCheckerTestSuite) TestLostStore() { @@ -283,7 +283,7 @@ func (suite *replicaCheckerTestSuite) TestLostStore() { tc.AddRegionStore(1, 1) tc.AddRegionStore(2, 1) - rc := NewReplicaChecker(tc, tc.GetOpts(), cache.NewDefaultCache(10)) + rc := NewReplicaChecker(tc, tc.GetCheckerConfig(), cache.NewDefaultCache(10)) // now region peer in store 1,2,3.but we just have store 1,2 // This happens only in recovering the PD tc @@ -301,7 +301,7 @@ func (suite *replicaCheckerTestSuite) TestOffline() { tc.SetMaxReplicas(3) tc.SetLocationLabels([]string{"zone", "rack", "host"}) - rc := NewReplicaChecker(tc, tc.GetOpts(), cache.NewDefaultCache(10)) + rc := NewReplicaChecker(tc, tc.GetCheckerConfig(), cache.NewDefaultCache(10)) tc.AddLabelsStore(1, 1, map[string]string{"zone": "z1", "rack": "r1", "host": "h1"}) tc.AddLabelsStore(2, 2, map[string]string{"zone": "z2", "rack": "r1", "host": "h1"}) tc.AddLabelsStore(3, 3, map[string]string{"zone": "z3", "rack": "r1", "host": "h1"}) @@ -311,36 +311,36 @@ func (suite *replicaCheckerTestSuite) TestOffline() { region := tc.GetRegion(1) // Store 2 has different zone and smallest region score. - testutil.CheckAddPeer(suite.Require(), rc.Check(region), operator.OpReplica, 2) + operatorutil.CheckAddPeer(suite.Require(), rc.Check(region), operator.OpReplica, 2) peer2, _ := tc.AllocPeer(2) region = region.Clone(core.WithAddPeer(peer2)) // Store 3 has different zone and smallest region score. - testutil.CheckAddPeer(suite.Require(), rc.Check(region), operator.OpReplica, 3) + operatorutil.CheckAddPeer(suite.Require(), rc.Check(region), operator.OpReplica, 3) peer3, _ := tc.AllocPeer(3) region = region.Clone(core.WithAddPeer(peer3)) // Store 4 has the same zone with store 3 and larger region score. peer4, _ := tc.AllocPeer(4) region = region.Clone(core.WithAddPeer(peer4)) - testutil.CheckRemovePeer(suite.Require(), rc.Check(region), 4) + operatorutil.CheckRemovePeer(suite.Require(), rc.Check(region), 4) // Test offline // the number of region peers more than the maxReplicas // remove the peer tc.SetStoreOffline(3) - testutil.CheckRemovePeer(suite.Require(), rc.Check(region), 3) + operatorutil.CheckRemovePeer(suite.Require(), rc.Check(region), 3) region = region.Clone(core.WithRemoveStorePeer(4)) // the number of region peers equals the maxReplicas // Transfer peer to store 4. - testutil.CheckTransferPeer(suite.Require(), rc.Check(region), operator.OpReplica, 3, 4) + operatorutil.CheckTransferPeer(suite.Require(), rc.Check(region), operator.OpReplica, 3, 4) // Store 5 has a same label score with store 4, but the region score smaller than store 4, we will choose store 5. tc.AddLabelsStore(5, 3, map[string]string{"zone": "z4", "rack": "r1", "host": "h1"}) - testutil.CheckTransferPeer(suite.Require(), rc.Check(region), operator.OpReplica, 3, 5) + operatorutil.CheckTransferPeer(suite.Require(), rc.Check(region), operator.OpReplica, 3, 5) // Store 5 has too many snapshots, choose store 4 tc.UpdateSnapshotCount(5, 100) - testutil.CheckTransferPeer(suite.Require(), rc.Check(region), operator.OpReplica, 3, 4) + operatorutil.CheckTransferPeer(suite.Require(), rc.Check(region), operator.OpReplica, 3, 4) tc.UpdatePendingPeerCount(4, 100) suite.Nil(rc.Check(region)) } @@ -352,7 +352,7 @@ func (suite *replicaCheckerTestSuite) TestDistinctScore() { tc.SetMaxReplicas(3) tc.SetLocationLabels([]string{"zone", "rack", "host"}) - rc := NewReplicaChecker(tc, tc.GetOpts(), cache.NewDefaultCache(10)) + rc := NewReplicaChecker(tc, tc.GetCheckerConfig(), cache.NewDefaultCache(10)) tc.AddLabelsStore(1, 9, map[string]string{"zone": "z1", "rack": "r1", "host": "h1"}) tc.AddLabelsStore(2, 8, map[string]string{"zone": "z1", "rack": "r1", "host": "h1"}) @@ -360,49 +360,49 @@ func (suite *replicaCheckerTestSuite) TestDistinctScore() { // We need 3 replicas. tc.AddLeaderRegion(1, 1) region := tc.GetRegion(1) - testutil.CheckAddPeer(suite.Require(), rc.Check(region), operator.OpReplica, 2) + operatorutil.CheckAddPeer(suite.Require(), rc.Check(region), operator.OpReplica, 2) peer2, _ := tc.AllocPeer(2) region = region.Clone(core.WithAddPeer(peer2)) // Store 1,2,3 have the same zone, rack, and host. tc.AddLabelsStore(3, 5, map[string]string{"zone": "z1", "rack": "r1", "host": "h1"}) - testutil.CheckAddPeer(suite.Require(), rc.Check(region), operator.OpReplica, 3) + operatorutil.CheckAddPeer(suite.Require(), rc.Check(region), operator.OpReplica, 3) // Store 4 has smaller region score. tc.AddLabelsStore(4, 4, map[string]string{"zone": "z1", "rack": "r1", "host": "h1"}) - testutil.CheckAddPeer(suite.Require(), rc.Check(region), operator.OpReplica, 4) + operatorutil.CheckAddPeer(suite.Require(), rc.Check(region), operator.OpReplica, 4) // Store 5 has a different host. tc.AddLabelsStore(5, 5, map[string]string{"zone": "z1", "rack": "r1", "host": "h2"}) - testutil.CheckAddPeer(suite.Require(), rc.Check(region), operator.OpReplica, 5) + operatorutil.CheckAddPeer(suite.Require(), rc.Check(region), operator.OpReplica, 5) // Store 6 has a different rack. tc.AddLabelsStore(6, 6, map[string]string{"zone": "z1", "rack": "r2", "host": "h1"}) - testutil.CheckAddPeer(suite.Require(), rc.Check(region), operator.OpReplica, 6) + operatorutil.CheckAddPeer(suite.Require(), rc.Check(region), operator.OpReplica, 6) // Store 7 has a different zone. tc.AddLabelsStore(7, 7, map[string]string{"zone": "z2", "rack": "r1", "host": "h1"}) - testutil.CheckAddPeer(suite.Require(), rc.Check(region), operator.OpReplica, 7) + operatorutil.CheckAddPeer(suite.Require(), rc.Check(region), operator.OpReplica, 7) // Test stateFilter. tc.SetStoreOffline(7) - testutil.CheckAddPeer(suite.Require(), rc.Check(region), operator.OpReplica, 6) + operatorutil.CheckAddPeer(suite.Require(), rc.Check(region), operator.OpReplica, 6) tc.SetStoreUp(7) - testutil.CheckAddPeer(suite.Require(), rc.Check(region), operator.OpReplica, 7) + operatorutil.CheckAddPeer(suite.Require(), rc.Check(region), operator.OpReplica, 7) // Add peer to store 7. peer7, _ := tc.AllocPeer(7) region = region.Clone(core.WithAddPeer(peer7)) // Replace peer in store 1 with store 6 because it has a different rack. - testutil.CheckTransferPeer(suite.Require(), rc.Check(region), operator.OpReplica, 1, 6) + operatorutil.CheckTransferPeer(suite.Require(), rc.Check(region), operator.OpReplica, 1, 6) // Disable locationReplacement feature. tc.SetEnableLocationReplacement(false) suite.Nil(rc.Check(region)) tc.SetEnableLocationReplacement(true) peer6, _ := tc.AllocPeer(6) region = region.Clone(core.WithAddPeer(peer6)) - testutil.CheckRemovePeer(suite.Require(), rc.Check(region), 1) + operatorutil.CheckRemovePeer(suite.Require(), rc.Check(region), 1) region = region.Clone(core.WithRemoveStorePeer(1), core.WithLeader(region.GetStorePeer(2))) suite.Nil(rc.Check(region)) @@ -416,10 +416,10 @@ func (suite *replicaCheckerTestSuite) TestDistinctScore() { // Store 2 and 6 have the same distinct score, but store 2 has larger region score. // So replace peer in store 2 with store 10. tc.AddLabelsStore(10, 1, map[string]string{"zone": "z3", "rack": "r1", "host": "h1"}) - testutil.CheckTransferPeer(suite.Require(), rc.Check(region), operator.OpReplica, 2, 10) + operatorutil.CheckTransferPeer(suite.Require(), rc.Check(region), operator.OpReplica, 2, 10) peer10, _ := tc.AllocPeer(10) region = region.Clone(core.WithAddPeer(peer10)) - testutil.CheckRemovePeer(suite.Require(), rc.Check(region), 2) + operatorutil.CheckRemovePeer(suite.Require(), rc.Check(region), 2) region = region.Clone(core.WithRemoveStorePeer(2)) suite.Nil(rc.Check(region)) } @@ -431,7 +431,7 @@ func (suite *replicaCheckerTestSuite) TestDistinctScore2() { tc.SetMaxReplicas(5) tc.SetLocationLabels([]string{"zone", "host"}) - rc := NewReplicaChecker(tc, tc.GetOpts(), cache.NewDefaultCache(10)) + rc := NewReplicaChecker(tc, tc.GetCheckerConfig(), cache.NewDefaultCache(10)) tc.AddLabelsStore(1, 1, map[string]string{"zone": "z1", "host": "h1"}) tc.AddLabelsStore(2, 1, map[string]string{"zone": "z1", "host": "h2"}) @@ -443,11 +443,11 @@ func (suite *replicaCheckerTestSuite) TestDistinctScore2() { tc.AddLeaderRegion(1, 1, 2, 4) region := tc.GetRegion(1) - testutil.CheckAddPeer(suite.Require(), rc.Check(region), operator.OpReplica, 6) + operatorutil.CheckAddPeer(suite.Require(), rc.Check(region), operator.OpReplica, 6) peer6, _ := tc.AllocPeer(6) region = region.Clone(core.WithAddPeer(peer6)) - testutil.CheckAddPeer(suite.Require(), rc.Check(region), operator.OpReplica, 5) + operatorutil.CheckAddPeer(suite.Require(), rc.Check(region), operator.OpReplica, 5) peer5, _ := tc.AllocPeer(5) region = region.Clone(core.WithAddPeer(peer5)) @@ -459,7 +459,7 @@ func (suite *replicaCheckerTestSuite) TestStorageThreshold() { tc := mockcluster.NewCluster(suite.ctx, opt) tc.SetLocationLabels([]string{"zone"}) tc.SetClusterVersion(versioninfo.MinSupportedVersion(versioninfo.Version4_0)) - rc := NewReplicaChecker(tc, tc.GetOpts(), cache.NewDefaultCache(10)) + rc := NewReplicaChecker(tc, tc.GetCheckerConfig(), cache.NewDefaultCache(10)) tc.AddLabelsStore(1, 1, map[string]string{"zone": "z1"}) tc.UpdateStorageRatio(1, 0.5, 0.5) @@ -475,7 +475,7 @@ func (suite *replicaCheckerTestSuite) TestStorageThreshold() { // Move peer to better location. tc.UpdateStorageRatio(4, 0, 1) - testutil.CheckTransferPeer(suite.Require(), rc.Check(region), operator.OpReplica, 1, 4) + operatorutil.CheckTransferPeer(suite.Require(), rc.Check(region), operator.OpReplica, 1, 4) // If store4 is almost full, do not add peer on it. tc.UpdateStorageRatio(4, 0.9, 0.1) suite.Nil(rc.Check(region)) @@ -484,17 +484,17 @@ func (suite *replicaCheckerTestSuite) TestStorageThreshold() { region = tc.GetRegion(2) // Add peer on store4. tc.UpdateStorageRatio(4, 0, 1) - testutil.CheckAddPeer(suite.Require(), rc.Check(region), operator.OpReplica, 4) + operatorutil.CheckAddPeer(suite.Require(), rc.Check(region), operator.OpReplica, 4) // If store4 is almost full, do not add peer on it. tc.UpdateStorageRatio(4, 0.8, 0) - testutil.CheckAddPeer(suite.Require(), rc.Check(region), operator.OpReplica, 2) + operatorutil.CheckAddPeer(suite.Require(), rc.Check(region), operator.OpReplica, 2) } func (suite *replicaCheckerTestSuite) TestOpts() { opt := mockconfig.NewTestOptions() tc := mockcluster.NewCluster(suite.ctx, opt) tc.SetClusterVersion(versioninfo.MinSupportedVersion(versioninfo.Version4_0)) - rc := NewReplicaChecker(tc, tc.GetOpts(), cache.NewDefaultCache(10)) + rc := NewReplicaChecker(tc, tc.GetCheckerConfig(), cache.NewDefaultCache(10)) tc.AddRegionStore(1, 100) tc.AddRegionStore(2, 100) @@ -513,9 +513,9 @@ func (suite *replicaCheckerTestSuite) TestOpts() { })) tc.SetStoreOffline(2) // RemoveDownReplica has higher priority than replaceOfflineReplica. - testutil.CheckTransferPeer(suite.Require(), rc.Check(region), operator.OpReplica, 1, 4) + operatorutil.CheckTransferPeer(suite.Require(), rc.Check(region), operator.OpReplica, 1, 4) tc.SetEnableRemoveDownReplica(false) - testutil.CheckTransferPeer(suite.Require(), rc.Check(region), operator.OpReplica, 2, 4) + operatorutil.CheckTransferPeer(suite.Require(), rc.Check(region), operator.OpReplica, 2, 4) tc.SetEnableReplaceOfflineReplica(false) suite.Nil(rc.Check(region)) } @@ -526,7 +526,7 @@ func (suite *replicaCheckerTestSuite) TestFixDownPeer() { tc := mockcluster.NewCluster(suite.ctx, opt) tc.SetClusterVersion(versioninfo.MinSupportedVersion(versioninfo.Version4_0)) tc.SetLocationLabels([]string{"zone"}) - rc := NewReplicaChecker(tc, tc.GetOpts(), cache.NewDefaultCache(10)) + rc := NewReplicaChecker(tc, tc.GetCheckerConfig(), cache.NewDefaultCache(10)) tc.AddLabelsStore(1, 1, map[string]string{"zone": "z1"}) tc.AddLabelsStore(2, 1, map[string]string{"zone": "z1"}) @@ -542,10 +542,10 @@ func (suite *replicaCheckerTestSuite) TestFixDownPeer() { region = region.Clone(core.WithDownPeers([]*pdpb.PeerStats{ {Peer: region.GetStorePeer(4), DownSeconds: 6000}, })) - testutil.CheckTransferPeer(suite.Require(), rc.Check(region), operator.OpRegion, 4, 5) + operatorutil.CheckTransferPeer(suite.Require(), rc.Check(region), operator.OpRegion, 4, 5) tc.SetStoreDown(5) - testutil.CheckTransferPeer(suite.Require(), rc.Check(region), operator.OpRegion, 4, 2) + operatorutil.CheckTransferPeer(suite.Require(), rc.Check(region), operator.OpRegion, 4, 2) tc.SetIsolationLevel("zone") suite.Nil(rc.Check(region)) @@ -557,7 +557,7 @@ func (suite *replicaCheckerTestSuite) TestFixOfflinePeer() { tc := mockcluster.NewCluster(suite.ctx, opt) tc.SetClusterVersion(versioninfo.MinSupportedVersion(versioninfo.Version4_0)) tc.SetLocationLabels([]string{"zone"}) - rc := NewReplicaChecker(tc, tc.GetOpts(), cache.NewDefaultCache(10)) + rc := NewReplicaChecker(tc, tc.GetCheckerConfig(), cache.NewDefaultCache(10)) tc.AddLabelsStore(1, 1, map[string]string{"zone": "z1"}) tc.AddLabelsStore(2, 1, map[string]string{"zone": "z1"}) @@ -570,10 +570,10 @@ func (suite *replicaCheckerTestSuite) TestFixOfflinePeer() { suite.Nil(rc.Check(region)) tc.SetStoreOffline(4) - testutil.CheckTransferPeer(suite.Require(), rc.Check(region), operator.OpRegion, 4, 5) + operatorutil.CheckTransferPeer(suite.Require(), rc.Check(region), operator.OpRegion, 4, 5) tc.SetStoreOffline(5) - testutil.CheckTransferPeer(suite.Require(), rc.Check(region), operator.OpRegion, 4, 2) + operatorutil.CheckTransferPeer(suite.Require(), rc.Check(region), operator.OpRegion, 4, 2) tc.SetIsolationLevel("zone") suite.Nil(rc.Check(region)) diff --git a/pkg/schedule/checker/replica_strategy.go b/pkg/schedule/checker/replica_strategy.go index 8d3353dbe0c..fdf05a0c479 100644 --- a/pkg/schedule/checker/replica_strategy.go +++ b/pkg/schedule/checker/replica_strategy.go @@ -17,7 +17,8 @@ package checker import ( "github.com/pingcap/log" "github.com/tikv/pd/pkg/core" - "github.com/tikv/pd/pkg/schedule" + "github.com/tikv/pd/pkg/core/constant" + sche "github.com/tikv/pd/pkg/schedule/core" "github.com/tikv/pd/pkg/schedule/filter" "go.uber.org/zap" ) @@ -26,7 +27,7 @@ import ( // exists to allow replica_checker and rule_checker to reuse common logics. type ReplicaStrategy struct { checkerName string // replica-checker / rule-checker - cluster schedule.Cluster + cluster sche.CheckerCluster locationLabels []string isolationLevel string region *core.RegionInfo @@ -53,11 +54,15 @@ func (s *ReplicaStrategy) SelectStoreToAdd(coLocationStores []*core.StoreInfo, e // // The reason for it is to prevent the non-optimal replica placement due // to the short-term state, resulting in redundant scheduling. + level := constant.High + if s.fastFailover { + level = constant.Urgent + } filters := []filter.Filter{ filter.NewExcludedFilter(s.checkerName, nil, s.region.GetStoreIDs()), filter.NewStorageThresholdFilter(s.checkerName), filter.NewSpecialUseFilter(s.checkerName), - &filter.StoreStateFilter{ActionScope: s.checkerName, MoveRegion: true, AllowTemporaryStates: true}, + &filter.StoreStateFilter{ActionScope: s.checkerName, MoveRegion: true, AllowTemporaryStates: true, OperatorLevel: level}, } if len(s.locationLabels) > 0 && s.isolationLevel != "" { filters = append(filters, filter.NewIsolationFilter(s.checkerName, s.isolationLevel, s.locationLabels, coLocationStores)) @@ -70,15 +75,15 @@ func (s *ReplicaStrategy) SelectStoreToAdd(coLocationStores []*core.StoreInfo, e } isolationComparer := filter.IsolationComparer(s.locationLabels, coLocationStores) - strictStateFilter := &filter.StoreStateFilter{ActionScope: s.checkerName, MoveRegion: true, AllowFastFailover: s.fastFailover} + strictStateFilter := &filter.StoreStateFilter{ActionScope: s.checkerName, MoveRegion: true, AllowFastFailover: s.fastFailover, OperatorLevel: level} targetCandidate := filter.NewCandidates(s.cluster.GetStores()). - FilterTarget(s.cluster.GetOpts(), nil, nil, filters...). + FilterTarget(s.cluster.GetCheckerConfig(), nil, nil, filters...). KeepTheTopStores(isolationComparer, false) // greater isolation score is better if targetCandidate.Len() == 0 { return 0, false } - target := targetCandidate.FilterTarget(s.cluster.GetOpts(), nil, nil, strictStateFilter). - PickTheTopStore(filter.RegionScoreComparer(s.cluster.GetOpts()), true) // less region score is better + target := targetCandidate.FilterTarget(s.cluster.GetCheckerConfig(), nil, nil, strictStateFilter). + PickTheTopStore(filter.RegionScoreComparer(s.cluster.GetCheckerConfig()), true) // less region score is better if target == nil { return 0, true // filter by temporary states } @@ -88,6 +93,9 @@ func (s *ReplicaStrategy) SelectStoreToAdd(coLocationStores []*core.StoreInfo, e // SelectStoreToFix returns a store to replace down/offline old peer. The location // placement after scheduling is allowed to be worse than original. func (s *ReplicaStrategy) SelectStoreToFix(coLocationStores []*core.StoreInfo, old uint64) (uint64, bool) { + if len(coLocationStores) == 0 { + return 0, false + } // trick to avoid creating a slice with `old` removed. s.swapStoreToFirst(coLocationStores, old) return s.SelectStoreToAdd(coLocationStores[1:]) @@ -96,6 +104,9 @@ func (s *ReplicaStrategy) SelectStoreToFix(coLocationStores []*core.StoreInfo, o // SelectStoreToImprove returns a store to replace oldStore. The location // placement after scheduling should be better than original. func (s *ReplicaStrategy) SelectStoreToImprove(coLocationStores []*core.StoreInfo, old uint64) (uint64, bool) { + if len(coLocationStores) == 0 { + return 0, false + } // trick to avoid creating a slice with `old` removed. s.swapStoreToFirst(coLocationStores, old) oldStore := s.cluster.GetStore(old) @@ -123,10 +134,14 @@ func (s *ReplicaStrategy) swapStoreToFirst(stores []*core.StoreInfo, id uint64) // SelectStoreToRemove returns the best option to remove from the region. func (s *ReplicaStrategy) SelectStoreToRemove(coLocationStores []*core.StoreInfo) uint64 { isolationComparer := filter.IsolationComparer(s.locationLabels, coLocationStores) + level := constant.High + if s.fastFailover { + level = constant.Urgent + } source := filter.NewCandidates(coLocationStores). - FilterSource(s.cluster.GetOpts(), nil, nil, &filter.StoreStateFilter{ActionScope: replicaCheckerName, MoveRegion: true}). + FilterSource(s.cluster.GetCheckerConfig(), nil, nil, &filter.StoreStateFilter{ActionScope: s.checkerName, MoveRegion: true, OperatorLevel: level}). KeepTheTopStores(isolationComparer, true). - PickTheTopStore(filter.RegionScoreComparer(s.cluster.GetOpts()), false) + PickTheTopStore(filter.RegionScoreComparer(s.cluster.GetCheckerConfig()), false) if source == nil { log.Debug("no removable store", zap.Uint64("region-id", s.region.GetID())) return 0 diff --git a/pkg/schedule/checker/rule_checker.go b/pkg/schedule/checker/rule_checker.go index c4594091c2a..34492a99ec0 100644 --- a/pkg/schedule/checker/rule_checker.go +++ b/pkg/schedule/checker/rule_checker.go @@ -27,7 +27,7 @@ import ( "github.com/tikv/pd/pkg/core" "github.com/tikv/pd/pkg/core/constant" "github.com/tikv/pd/pkg/errs" - "github.com/tikv/pd/pkg/schedule" + sche "github.com/tikv/pd/pkg/schedule/core" "github.com/tikv/pd/pkg/schedule/filter" "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/placement" @@ -76,12 +76,13 @@ var ( ruleCheckerMoveToBetterLocationCounter = checkerCounter.WithLabelValues(ruleChecker, "move-to-better-location") ruleCheckerSkipRemoveOrphanPeerCounter = checkerCounter.WithLabelValues(ruleChecker, "skip-remove-orphan-peer") ruleCheckerRemoveOrphanPeerCounter = checkerCounter.WithLabelValues(ruleChecker, "remove-orphan-peer") + ruleCheckerReplaceOrphanPeerCounter = checkerCounter.WithLabelValues(ruleChecker, "replace-orphan-peer") ) // RuleChecker fix/improve region by placement rules. type RuleChecker struct { PauseController - cluster schedule.Cluster + cluster sche.CheckerCluster ruleManager *placement.RuleManager name string regionWaitingList cache.Cache @@ -91,14 +92,14 @@ type RuleChecker struct { } // NewRuleChecker creates a checker instance. -func NewRuleChecker(ctx context.Context, cluster schedule.Cluster, ruleManager *placement.RuleManager, regionWaitingList cache.Cache) *RuleChecker { +func NewRuleChecker(ctx context.Context, cluster sche.CheckerCluster, ruleManager *placement.RuleManager, regionWaitingList cache.Cache) *RuleChecker { return &RuleChecker{ cluster: cluster, ruleManager: ruleManager, name: ruleCheckerName, regionWaitingList: regionWaitingList, pendingList: cache.NewDefaultCache(maxPendingListLen), - switchWitnessCache: cache.NewIDTTL(ctx, time.Minute, cluster.GetOpts().GetSwitchWitnessInterval()), + switchWitnessCache: cache.NewIDTTL(ctx, time.Minute, cluster.GetCheckerConfig().GetSwitchWitnessInterval()), record: newRecord(), } } @@ -160,7 +161,7 @@ func (c *RuleChecker) CheckWithFit(region *core.RegionInfo, fit *placement.Regio return op } } - if c.cluster.GetOpts().IsPlacementRulesCacheEnabled() { + if c.cluster.GetCheckerConfig().IsPlacementRulesCacheEnabled() { if placement.ValidateFit(fit) && placement.ValidateRegion(region) && placement.ValidateStores(fit.GetRegionStores()) { // If there is no need to fix, we will cache the fit c.ruleManager.SetRegionFitCache(region, fit) @@ -173,12 +174,12 @@ func (c *RuleChecker) CheckWithFit(region *core.RegionInfo, fit *placement.Regio // RecordRegionPromoteToNonWitness put the recently switch non-witness region into cache. RuleChecker // will skip switch it back to witness for a while. func (c *RuleChecker) RecordRegionPromoteToNonWitness(regionID uint64) { - c.switchWitnessCache.PutWithTTL(regionID, nil, c.cluster.GetOpts().GetSwitchWitnessInterval()) + c.switchWitnessCache.PutWithTTL(regionID, nil, c.cluster.GetCheckerConfig().GetSwitchWitnessInterval()) } func (c *RuleChecker) isWitnessEnabled() bool { - return versioninfo.IsFeatureSupported(c.cluster.GetOpts().GetClusterVersion(), versioninfo.SwitchWitness) && - c.cluster.GetOpts().IsWitnessAllowed() + return versioninfo.IsFeatureSupported(c.cluster.GetCheckerConfig().GetClusterVersion(), versioninfo.SwitchWitness) && + c.cluster.GetCheckerConfig().IsWitnessAllowed() } func (c *RuleChecker) fixRulePeer(region *core.RegionInfo, fit *placement.RegionFit, rf *placement.RuleFit) (*operator.Operator, error) { @@ -205,13 +206,6 @@ func (c *RuleChecker) fixRulePeer(region *core.RegionInfo, fit *placement.Region ruleCheckerReplaceOfflineCounter.Inc() return c.replaceUnexpectRulePeer(region, rf, fit, peer, offlineStatus) } - - if c.isWitnessEnabled() && c.isPendingVoter(region, peer) { - if witness, ok := c.hasAvailableWitness(region, peer); ok { - ruleCheckerPromoteWitnessCounter.Inc() - return operator.CreateNonWitnessPeerOperator("promote-witness-for-pending", c.cluster, region, witness) - } - } } // fix loose matched peers. for _, peer := range rf.PeersWithDifferentRole { @@ -350,7 +344,7 @@ func (c *RuleChecker) fixLooseMatchPeer(region *core.RegionInfo, fit *placement. return nil, errPeerCannotBeWitness } if !core.IsWitness(peer) && rf.Rule.IsWitness && c.isWitnessEnabled() { - c.switchWitnessCache.UpdateTTL(c.cluster.GetOpts().GetSwitchWitnessInterval()) + c.switchWitnessCache.UpdateTTL(c.cluster.GetCheckerConfig().GetSwitchWitnessInterval()) if c.switchWitnessCache.Exists(region.GetID()) { ruleCheckerRecentlyPromoteToNonWitnessCounter.Inc() return nil, nil @@ -385,7 +379,7 @@ func (c *RuleChecker) allowLeader(fit *placement.RegionFit, peer *metapb.Peer) b return false } stateFilter := &filter.StoreStateFilter{ActionScope: "rule-checker", TransferLeader: true} - if !stateFilter.Target(c.cluster.GetOpts(), s).IsOK() { + if !stateFilter.Target(c.cluster.GetCheckerConfig(), s).IsOK() { return false } for _, rf := range fit.RuleFits { @@ -398,7 +392,7 @@ func (c *RuleChecker) allowLeader(fit *placement.RegionFit, peer *metapb.Peer) b } func (c *RuleChecker) fixBetterLocation(region *core.RegionInfo, rf *placement.RuleFit) (*operator.Operator, error) { - if len(rf.Rule.LocationLabels) == 0 || rf.Rule.Count <= 1 { + if len(rf.Rule.LocationLabels) == 0 { return nil, nil } @@ -410,7 +404,15 @@ func (c *RuleChecker) fixBetterLocation(region *core.RegionInfo, rf *placement.R if oldStore == 0 { return nil, nil } - newStore, filterByTempState := strategy.SelectStoreToImprove(ruleStores, oldStore) + var coLocationStores []*core.StoreInfo + regionStores := c.cluster.GetRegionStores(region) + for _, s := range regionStores { + if placement.MatchLabelConstraints(s, rf.Rule.LabelConstraints) { + coLocationStores = append(coLocationStores, s) + } + } + + newStore, filterByTempState := strategy.SelectStoreToImprove(coLocationStores, oldStore) if newStore == 0 { log.Debug("no replacement store", zap.Uint64("region-id", region.GetID())) c.handleFilterState(region, filterByTempState) @@ -425,14 +427,15 @@ func (c *RuleChecker) fixOrphanPeers(region *core.RegionInfo, fit *placement.Reg if len(fit.OrphanPeers) == 0 { return nil, nil } + var pinDownPeer *metapb.Peer isUnhealthyPeer := func(id uint64) bool { - for _, pendingPeer := range region.GetPendingPeers() { - if pendingPeer.GetId() == id { + for _, downPeer := range region.GetDownPeers() { + if downPeer.Peer.GetId() == id { return true } } - for _, downPeer := range region.GetDownPeers() { - if downPeer.Peer.GetId() == id { + for _, pendingPeer := range region.GetPendingPeers() { + if pendingPeer.GetId() == id { return true } } @@ -449,24 +452,71 @@ loopFits: } for _, p := range rf.Peers { if isUnhealthyPeer(p.GetId()) { + // make sure is down peer. + if region.GetDownPeer(p.GetId()) != nil { + pinDownPeer = p + } hasUnhealthyFit = true break loopFits } } } + // If hasUnhealthyFit is false, it is safe to delete the OrphanPeer. if !hasUnhealthyFit { ruleCheckerRemoveOrphanPeerCounter.Inc() return operator.CreateRemovePeerOperator("remove-orphan-peer", c.cluster, 0, region, fit.OrphanPeers[0].StoreId) } + + // try to use orphan peers to replace unhealthy down peers. + for _, orphanPeer := range fit.OrphanPeers { + if pinDownPeer != nil { + // make sure the orphan peer is healthy. + if isUnhealthyPeer(orphanPeer.GetId()) { + continue + } + // no consider witness in this path. + if pinDownPeer.GetIsWitness() || orphanPeer.GetIsWitness() { + continue + } + // down peer's store should be down. + if !c.isStoreDownTimeHitMaxDownTime(pinDownPeer.GetStoreId()) { + continue + } + // check if down peer can replace with orphan peer. + dstStore := c.cluster.GetStore(orphanPeer.GetStoreId()) + if fit.Replace(pinDownPeer.GetStoreId(), dstStore) { + destRole := pinDownPeer.GetRole() + orphanPeerRole := orphanPeer.GetRole() + ruleCheckerReplaceOrphanPeerCounter.Inc() + switch { + case orphanPeerRole == metapb.PeerRole_Learner && destRole == metapb.PeerRole_Voter: + return operator.CreatePromoteLearnerOperatorAndRemovePeer("replace-down-peer-with-orphan-peer", c.cluster, region, orphanPeer, pinDownPeer) + case orphanPeerRole == metapb.PeerRole_Voter && destRole == metapb.PeerRole_Learner: + return operator.CreateDemoteLearnerOperatorAndRemovePeer("replace-down-peer-with-orphan-peer", c.cluster, region, orphanPeer, pinDownPeer) + default: + // destRole should not same with orphanPeerRole. if role is same, it fit with orphanPeer should be better than now. + // destRole never be leader, so we not consider it. + } + } + } + } + // If hasUnhealthyFit is true, try to remove unhealthy orphan peers only if number of OrphanPeers is >= 2. // Ref https://github.com/tikv/pd/issues/4045 if len(fit.OrphanPeers) >= 2 { + hasHealthPeer := false for _, orphanPeer := range fit.OrphanPeers { if isUnhealthyPeer(orphanPeer.GetId()) { ruleCheckerRemoveOrphanPeerCounter.Inc() return operator.CreateRemovePeerOperator("remove-orphan-peer", c.cluster, 0, region, orphanPeer.StoreId) } + if hasHealthPeer { + // there already exists a healthy orphan peer, so we can remove other orphan Peers. + ruleCheckerRemoveOrphanPeerCounter.Inc() + return operator.CreateRemovePeerOperator("remove-orphan-peer", c.cluster, 0, region, orphanPeer.StoreId) + } + hasHealthPeer = true } } ruleCheckerSkipRemoveOrphanPeerCounter.Inc() @@ -490,7 +540,11 @@ func (c *RuleChecker) isDownPeer(region *core.RegionInfo, peer *metapb.Peer) boo func (c *RuleChecker) isStoreDownTimeHitMaxDownTime(storeID uint64) bool { store := c.cluster.GetStore(storeID) - return store.DownTime() >= c.cluster.GetOpts().GetMaxStoreDownTime() + if store == nil { + log.Warn("lost the store, maybe you are recovering the PD cluster", zap.Uint64("store-id", storeID)) + return false + } + return store.DownTime() >= c.cluster.GetCheckerConfig().GetMaxStoreDownTime() } func (c *RuleChecker) isOfflinePeer(peer *metapb.Peer) bool { @@ -502,10 +556,6 @@ func (c *RuleChecker) isOfflinePeer(peer *metapb.Peer) bool { return !store.IsPreparing() && !store.IsServing() } -func (c *RuleChecker) isPendingVoter(region *core.RegionInfo, peer *metapb.Peer) bool { - return region.GetPendingVoter(peer.Id) != nil -} - func (c *RuleChecker) hasAvailableWitness(region *core.RegionInfo, peer *metapb.Peer) (*metapb.Peer, bool) { witnesses := region.GetWitnesses() if len(witnesses) == 0 { @@ -583,7 +633,7 @@ func (o *recorder) incOfflineLeaderCount(storeID uint64) { // Offline is triggered manually and only appears when the node makes some adjustments. here is an operator timeout / 2. var offlineCounterTTL = 5 * time.Minute -func (o *recorder) refresh(cluster schedule.Cluster) { +func (o *recorder) refresh(cluster sche.CheckerCluster) { // re-count the offlineLeaderCounter if the store is already tombstone or store is gone. if len(o.offlineLeaderCounter) > 0 && time.Since(o.lastUpdateTime) > offlineCounterTTL { needClean := false diff --git a/pkg/schedule/checker/rule_checker_test.go b/pkg/schedule/checker/rule_checker_test.go index 6ed7e77811b..cbd7624f3b1 100644 --- a/pkg/schedule/checker/rule_checker_test.go +++ b/pkg/schedule/checker/rule_checker_test.go @@ -32,7 +32,7 @@ import ( "github.com/tikv/pd/pkg/mock/mockconfig" "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/placement" - "github.com/tikv/pd/pkg/utils/testutil" + "github.com/tikv/pd/pkg/utils/operatorutil" "github.com/tikv/pd/pkg/versioninfo" ) @@ -168,6 +168,39 @@ func (suite *ruleCheckerTestSuite) TestFixOrphanPeers() { suite.Equal(uint64(4), op.Step(0).(operator.RemovePeer).FromStore) } +func (suite *ruleCheckerTestSuite) TestFixToManyOrphanPeers() { + suite.cluster.AddLeaderStore(1, 1) + suite.cluster.AddLeaderStore(2, 1) + suite.cluster.AddLeaderStore(3, 1) + suite.cluster.AddLeaderStore(4, 1) + suite.cluster.AddLeaderStore(5, 1) + suite.cluster.AddLeaderStore(6, 1) + suite.cluster.AddRegionWithLearner(1, 1, []uint64{2, 3}, []uint64{4, 5, 6}) + // Case1: + // store 4, 5, 6 are orphan peers, and peer on store 3 is pending and down peer. + region := suite.cluster.GetRegion(1) + region = region.Clone( + core.WithDownPeers([]*pdpb.PeerStats{{Peer: region.GetStorePeer(3), DownSeconds: 60000}}), + core.WithPendingPeers([]*metapb.Peer{region.GetStorePeer(3)})) + suite.cluster.PutRegion(region) + op := suite.rc.Check(suite.cluster.GetRegion(1)) + suite.NotNil(op) + suite.Equal("remove-orphan-peer", op.Desc()) + suite.Equal(uint64(5), op.Step(0).(operator.RemovePeer).FromStore) + + // Case2: + // store 4, 5, 6 are orphan peers, and peer on store 3 is down peer. and peer on store 4, 5 are pending. + region = suite.cluster.GetRegion(1) + region = region.Clone( + core.WithDownPeers([]*pdpb.PeerStats{{Peer: region.GetStorePeer(3), DownSeconds: 60000}}), + core.WithPendingPeers([]*metapb.Peer{region.GetStorePeer(4), region.GetStorePeer(5)})) + suite.cluster.PutRegion(region) + op = suite.rc.Check(suite.cluster.GetRegion(1)) + suite.NotNil(op) + suite.Equal("remove-orphan-peer", op.Desc()) + suite.Equal(uint64(4), op.Step(0).(operator.RemovePeer).FromStore) +} + func (suite *ruleCheckerTestSuite) TestFixOrphanPeers2() { // check orphan peers can only be handled when all rules are satisfied. suite.cluster.AddLabelsStore(1, 1, map[string]string{"foo": "bar"}) @@ -312,7 +345,7 @@ func (suite *ruleCheckerTestSuite) TestFixRuleWitness() { suite.cluster.AddLabelsStore(1, 1, map[string]string{"A": "leader"}) suite.cluster.AddLabelsStore(2, 1, map[string]string{"B": "follower"}) suite.cluster.AddLabelsStore(3, 1, map[string]string{"C": "voter"}) - suite.cluster.AddLeaderRegion(1, 1, 2) + suite.cluster.AddLeaderRegion(1, 1) suite.ruleManager.SetRule(&placement.Rule{ GroupID: "pd", @@ -337,24 +370,25 @@ func (suite *ruleCheckerTestSuite) TestFixRuleWitness2() { suite.cluster.AddLabelsStore(1, 1, map[string]string{"A": "leader"}) suite.cluster.AddLabelsStore(2, 1, map[string]string{"B": "voter"}) suite.cluster.AddLabelsStore(3, 1, map[string]string{"C": "voter"}) - suite.cluster.AddLeaderRegion(1, 1, 2, 3) + suite.cluster.AddLabelsStore(4, 1, map[string]string{"D": "voter"}) + suite.cluster.AddLeaderRegion(1, 1, 2, 3, 4) suite.ruleManager.SetRule(&placement.Rule{ GroupID: "pd", ID: "r1", Index: 100, - Override: true, + Override: false, Role: placement.Voter, Count: 1, IsWitness: true, LabelConstraints: []placement.LabelConstraint{ - {Key: "C", Op: "in", Values: []string{"voter"}}, + {Key: "D", Op: "in", Values: []string{"voter"}}, }, }) op := suite.rc.Check(suite.cluster.GetRegion(1)) suite.NotNil(op) suite.Equal("fix-witness-peer", op.Desc()) - suite.Equal(uint64(3), op.Step(0).(operator.BecomeWitness).StoreID) + suite.Equal(uint64(4), op.Step(0).(operator.BecomeWitness).StoreID) } func (suite *ruleCheckerTestSuite) TestFixRuleWitness3() { @@ -366,7 +400,7 @@ func (suite *ruleCheckerTestSuite) TestFixRuleWitness3() { r := suite.cluster.GetRegion(1) // set peer3 to witness r = r.Clone(core.WithWitnesses([]*metapb.Peer{r.GetPeer(3)})) - + suite.cluster.PutRegion(r) op := suite.rc.Check(r) suite.NotNil(op) suite.Equal("fix-non-witness-peer", op.Desc()) @@ -651,6 +685,132 @@ func (suite *ruleCheckerTestSuite) TestPriorityFixOrphanPeer() { suite.Equal("remove-orphan-peer", op.Desc()) } +func (suite *ruleCheckerTestSuite) TestPriorityFitHealthWithDifferentRole1() { + suite.cluster.SetEnableUseJointConsensus(true) + suite.cluster.AddLabelsStore(1, 1, map[string]string{"host": "host1"}) + suite.cluster.AddLabelsStore(2, 1, map[string]string{"host": "host2"}) + suite.cluster.AddLabelsStore(3, 1, map[string]string{"host": "host3"}) + suite.cluster.AddLabelsStore(4, 1, map[string]string{"host": "host4"}) + suite.cluster.AddRegionWithLearner(1, 1, []uint64{2, 3}, []uint64{4}) + r1 := suite.cluster.GetRegion(1) + suite.cluster.GetStore(3).GetMeta().LastHeartbeat = time.Now().Add(-31 * time.Minute).UnixNano() + + // set peer3 to pending and down + r1 = r1.Clone(core.WithPendingPeers([]*metapb.Peer{r1.GetPeer(3)})) + r1 = r1.Clone(core.WithDownPeers([]*pdpb.PeerStats{ + { + Peer: r1.GetStorePeer(3), + DownSeconds: 30000, + }, + })) + suite.cluster.PutRegion(r1) + + op := suite.rc.Check(suite.cluster.GetRegion(1)) + suite.Equal(uint64(3), op.Step(0).(operator.ChangePeerV2Enter).DemoteVoters[0].ToStore) + suite.Equal(uint64(4), op.Step(0).(operator.ChangePeerV2Enter).PromoteLearners[0].ToStore) + suite.Equal(uint64(3), op.Step(1).(operator.ChangePeerV2Leave).DemoteVoters[0].ToStore) + suite.Equal(uint64(4), op.Step(1).(operator.ChangePeerV2Leave).PromoteLearners[0].ToStore) + suite.Equal("replace-down-peer-with-orphan-peer", op.Desc()) + + // set peer3 only pending + r1 = r1.Clone(core.WithDownPeers(nil)) + suite.cluster.PutRegion(r1) + op = suite.rc.Check(suite.cluster.GetRegion(1)) + suite.Nil(op) +} + +func (suite *ruleCheckerTestSuite) TestPriorityFitHealthWithDifferentRole2() { + suite.cluster.SetEnableUseJointConsensus(true) + suite.cluster.AddLabelsStore(1, 1, map[string]string{"host": "host1"}) + suite.cluster.AddLabelsStore(2, 1, map[string]string{"host": "host2"}) + suite.cluster.AddLabelsStore(3, 1, map[string]string{"host": "host3"}) + suite.cluster.AddLabelsStore(4, 1, map[string]string{"host": "host4"}) + suite.cluster.AddLabelsStore(5, 1, map[string]string{"host": "host5"}) + suite.cluster.AddLeaderRegion(1, 1, 2, 3, 4, 5) + r1 := suite.cluster.GetRegion(1) + + // set peer3 to pending and down, and peer 3 to learner, and store 3 is down + suite.cluster.GetStore(3).GetMeta().LastHeartbeat = time.Now().Add(-31 * time.Minute).UnixNano() + r1 = r1.Clone(core.WithLearners([]*metapb.Peer{r1.GetPeer(3)})) + r1 = r1.Clone( + core.WithPendingPeers([]*metapb.Peer{r1.GetPeer(3)}), + core.WithDownPeers([]*pdpb.PeerStats{ + { + Peer: r1.GetStorePeer(3), + DownSeconds: 30000, + }, + }), + ) + suite.cluster.PutRegion(r1) + + // default and test group => 3 voter + 1 learner + err := suite.ruleManager.SetRule(&placement.Rule{ + GroupID: "test", + ID: "10", + Role: placement.Learner, + Count: 1, + }) + suite.NoError(err) + + op := suite.rc.Check(suite.cluster.GetRegion(1)) + suite.Equal(uint64(5), op.Step(0).(operator.ChangePeerV2Enter).DemoteVoters[0].ToStore) + suite.Equal(uint64(3), op.Step(1).(operator.RemovePeer).FromStore) + suite.Equal("replace-down-peer-with-orphan-peer", op.Desc()) +} + +func (suite *ruleCheckerTestSuite) TestPriorityFitHealthPeersAndTiFlash() { + suite.cluster.SetEnableUseJointConsensus(true) + suite.cluster.AddLabelsStore(1, 1, map[string]string{"host": "host1"}) + suite.cluster.AddLabelsStore(2, 1, map[string]string{"host": "host2"}) + suite.cluster.AddLabelsStore(3, 1, map[string]string{"host": "host3"}) + suite.cluster.AddLabelsStore(4, 1, map[string]string{"host": "host4", "engine": "tiflash"}) + suite.cluster.AddRegionWithLearner(1, 1, []uint64{2, 3}, []uint64{4}) + rule := &placement.Rule{ + GroupID: "pd", + ID: "test", + Role: placement.Voter, + Count: 3, + } + rule2 := &placement.Rule{ + GroupID: "pd", + ID: "test2", + Role: placement.Learner, + Count: 1, + LabelConstraints: []placement.LabelConstraint{ + { + Key: "engine", + Op: placement.In, + Values: []string{"tiflash"}, + }, + }, + } + suite.ruleManager.SetRule(rule) + suite.ruleManager.SetRule(rule2) + suite.ruleManager.DeleteRule("pd", "default") + + r1 := suite.cluster.GetRegion(1) + // set peer3 to pending and down + r1 = r1.Clone(core.WithPendingPeers([]*metapb.Peer{r1.GetPeer(3)})) + r1 = r1.Clone(core.WithDownPeers([]*pdpb.PeerStats{ + { + Peer: r1.GetStorePeer(3), + DownSeconds: 30000, + }, + })) + suite.cluster.PutRegion(r1) + suite.cluster.GetStore(3).GetMeta().LastHeartbeat = time.Now().Add(-31 * time.Minute).UnixNano() + + op := suite.rc.Check(suite.cluster.GetRegion(1)) + // should not promote tiflash peer + suite.Nil(op) + + // scale a node, can replace the down peer + suite.cluster.AddLabelsStore(5, 1, map[string]string{"host": "host5"}) + op = suite.rc.Check(suite.cluster.GetRegion(1)) + suite.NotNil(op) + suite.Equal("fast-replace-rule-down-peer", op.Desc()) +} + func (suite *ruleCheckerTestSuite) TestIssue3293() { suite.cluster.AddLabelsStore(1, 1, map[string]string{"host": "host1"}) suite.cluster.AddLabelsStore(2, 1, map[string]string{"host": "host1"}) @@ -807,10 +967,10 @@ func (suite *ruleCheckerTestSuite) TestFixDownPeer() { region = region.Clone(core.WithDownPeers([]*pdpb.PeerStats{ {Peer: region.GetStorePeer(4), DownSeconds: 6000}, })) - testutil.CheckTransferPeer(suite.Require(), suite.rc.Check(region), operator.OpRegion, 4, 5) + operatorutil.CheckTransferPeer(suite.Require(), suite.rc.Check(region), operator.OpRegion, 4, 5) suite.cluster.SetStoreDown(5) - testutil.CheckTransferPeer(suite.Require(), suite.rc.Check(region), operator.OpRegion, 4, 2) + operatorutil.CheckTransferPeer(suite.Require(), suite.rc.Check(region), operator.OpRegion, 4, 2) rule.IsolationLevel = "zone" suite.ruleManager.SetRule(rule) @@ -1022,10 +1182,10 @@ func (suite *ruleCheckerTestSuite) TestFixOfflinePeer() { suite.Nil(suite.rc.Check(region)) suite.cluster.SetStoreOffline(4) - testutil.CheckTransferPeer(suite.Require(), suite.rc.Check(region), operator.OpRegion, 4, 5) + operatorutil.CheckTransferPeer(suite.Require(), suite.rc.Check(region), operator.OpRegion, 4, 5) suite.cluster.SetStoreOffline(5) - testutil.CheckTransferPeer(suite.Require(), suite.rc.Check(region), operator.OpRegion, 4, 2) + operatorutil.CheckTransferPeer(suite.Require(), suite.rc.Check(region), operator.OpRegion, 4, 2) rule.IsolationLevel = "zone" suite.ruleManager.SetRule(rule) @@ -1063,75 +1223,6 @@ func (suite *ruleCheckerTestSuite) TestFixOfflinePeerWithAvaliableWitness() { suite.Equal("replace-rule-offline-peer", op.Desc()) } -func (suite *ruleCheckerTestSuite) TestFixPendingVoterWithAvailableWitness() { - suite.cluster.AddLabelsStore(1, 1, map[string]string{"zone": "z1"}) - suite.cluster.AddLabelsStore(2, 1, map[string]string{"zone": "z2"}) - suite.cluster.AddLabelsStore(3, 1, map[string]string{"zone": "z3"}) - suite.cluster.AddLeaderRegion(1, 1, 2, 3) - - r := suite.cluster.GetRegion(1) - // set peer2 as pending voter - r = r.Clone(core.WithPendingPeers([]*metapb.Peer{r.GetPeer(2)})) - // set peer3 to witness - r = r.Clone(core.WithWitnesses([]*metapb.Peer{r.GetPeer(3)})) - - suite.ruleManager.SetRule(&placement.Rule{ - GroupID: "pd", - ID: "default", - Role: placement.Voter, - Count: 2, - }) - suite.ruleManager.SetRule(&placement.Rule{ - GroupID: "pd", - ID: "r1", - Role: placement.Voter, - Count: 1, - IsWitness: true, - }) - - op := suite.rc.Check(r) - - suite.NotNil(op) - suite.Equal("promote-witness-for-pending", op.Desc()) - suite.Equal(uint64(3), op.Step(0).(operator.RemovePeer).FromStore) - suite.Equal(uint64(3), op.Step(1).(operator.AddLearner).ToStore) - suite.Equal(uint64(3), op.Step(2).(operator.BecomeNonWitness).StoreID) - suite.Equal(uint64(3), op.Step(3).(operator.PromoteLearner).ToStore) -} - -func (suite *ruleCheckerTestSuite) TestFixPendingVoterWithAvailableWitness2() { - suite.cluster.AddLabelsStore(1, 1, map[string]string{"zone": "z1"}) - suite.cluster.AddLabelsStore(2, 1, map[string]string{"zone": "z2"}) - suite.cluster.AddLabelsStore(3, 1, map[string]string{"zone": "z3"}) - suite.cluster.AddLeaderRegion(1, 1, 2, 3) - - r := suite.cluster.GetRegion(1) - // set peer2 as pending learner - peer := r.GetPeer(2) - peer.Role = metapb.PeerRole_Learner - r = r.Clone(core.WithPendingPeers([]*metapb.Peer{peer})) - // set peer3 to witness - r = r.Clone(core.WithWitnesses([]*metapb.Peer{r.GetPeer(3)})) - - suite.ruleManager.SetRule(&placement.Rule{ - GroupID: "pd", - ID: "default", - Role: placement.Voter, - Count: 2, - }) - suite.ruleManager.SetRule(&placement.Rule{ - GroupID: "pd", - ID: "r1", - Role: placement.Voter, - Count: 1, - IsWitness: true, - }) - - op := suite.rc.Check(r) - - suite.Nil(op) -} - func (suite *ruleCheckerTestSuite) TestRuleCache() { suite.cluster.PersistOptions.SetPlacementRulesCacheEnabled(true) suite.cluster.AddLabelsStore(1, 1, map[string]string{"zone": "z1"}) @@ -1366,3 +1457,94 @@ func (suite *ruleCheckerTestSuite) TestPendingList() { _, exist = suite.rc.pendingList.Get(1) suite.False(exist) } + +func (suite *ruleCheckerTestSuite) TestLocationLabels() { + suite.cluster.AddLabelsStore(1, 1, map[string]string{"zone": "z1", "rack": "r1", "host": "h1"}) + suite.cluster.AddLabelsStore(2, 1, map[string]string{"zone": "z1", "rack": "r1", "host": "h1"}) + suite.cluster.AddLabelsStore(3, 1, map[string]string{"zone": "z1", "rack": "r2", "host": "h1"}) + suite.cluster.AddLabelsStore(4, 1, map[string]string{"zone": "z1", "rack": "r2", "host": "h1"}) + suite.cluster.AddLabelsStore(5, 1, map[string]string{"zone": "z2", "rack": "r3", "host": "h2"}) + suite.cluster.AddLabelsStore(6, 1, map[string]string{"zone": "z2", "rack": "r3", "host": "h2"}) + suite.cluster.AddLeaderRegionWithRange(1, "", "", 1, 2, 5) + rule1 := &placement.Rule{ + GroupID: "pd", + ID: "test1", + Role: placement.Leader, + Count: 1, + LabelConstraints: []placement.LabelConstraint{ + { + Key: "zone", + Op: placement.In, + Values: []string{"z1"}, + }, + }, + LocationLabels: []string{"rack"}, + } + rule2 := &placement.Rule{ + GroupID: "pd", + ID: "test2", + Role: placement.Voter, + Count: 1, + LabelConstraints: []placement.LabelConstraint{ + { + Key: "zone", + Op: placement.In, + Values: []string{"z1"}, + }, + }, + LocationLabels: []string{"rack"}, + } + rule3 := &placement.Rule{ + GroupID: "pd", + ID: "test3", + Role: placement.Voter, + Count: 1, + LabelConstraints: []placement.LabelConstraint{ + { + Key: "zone", + Op: placement.In, + Values: []string{"z2"}, + }, + }, + LocationLabels: []string{"rack"}, + } + suite.ruleManager.SetRule(rule1) + suite.ruleManager.SetRule(rule2) + suite.ruleManager.SetRule(rule3) + suite.ruleManager.DeleteRule("pd", "default") + op := suite.rc.Check(suite.cluster.GetRegion(1)) + suite.NotNil(op) + suite.Equal("move-to-better-location", op.Desc()) +} + +func (suite *ruleCheckerTestSuite) TestTiFlashLocationLabels() { + suite.cluster.SetEnableUseJointConsensus(true) + suite.cluster.AddLabelsStore(1, 1, map[string]string{"zone": "z1", "rack": "r1", "host": "h1"}) + suite.cluster.AddLabelsStore(2, 1, map[string]string{"zone": "z1", "rack": "r1", "host": "h1"}) + suite.cluster.AddLabelsStore(3, 1, map[string]string{"zone": "z1", "rack": "r2", "host": "h1"}) + suite.cluster.AddLabelsStore(4, 1, map[string]string{"zone": "z1", "rack": "r2", "host": "h1"}) + suite.cluster.AddLabelsStore(5, 1, map[string]string{"zone": "z2", "rack": "r3", "host": "h2"}) + suite.cluster.AddLabelsStore(6, 1, map[string]string{"zone": "z2", "rack": "r3", "host": "h2"}) + suite.cluster.AddLabelsStore(7, 1, map[string]string{"engine": "tiflash"}) + suite.cluster.AddRegionWithLearner(1, 1, []uint64{3, 5}, []uint64{7}) + + rule1 := &placement.Rule{ + GroupID: "tiflash", + ID: "test1", + Role: placement.Learner, + Count: 1, + LabelConstraints: []placement.LabelConstraint{ + { + Key: "engine", + Op: placement.In, + Values: []string{"tiflash"}, + }, + }, + } + suite.ruleManager.SetRule(rule1) + rule := suite.ruleManager.GetRule("pd", "default") + rule.LocationLabels = []string{"zone", "rack", "host"} + suite.ruleManager.SetRule(rule) + op := suite.rc.Check(suite.cluster.GetRegion(1)) + suite.Nil(op) +} diff --git a/pkg/schedule/checker/split_checker.go b/pkg/schedule/checker/split_checker.go index f89604909ea..072bdcf7a2e 100644 --- a/pkg/schedule/checker/split_checker.go +++ b/pkg/schedule/checker/split_checker.go @@ -19,7 +19,7 @@ import ( "github.com/pingcap/log" "github.com/tikv/pd/pkg/core" "github.com/tikv/pd/pkg/errs" - "github.com/tikv/pd/pkg/schedule" + sche "github.com/tikv/pd/pkg/schedule/core" "github.com/tikv/pd/pkg/schedule/labeler" "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/placement" @@ -28,7 +28,7 @@ import ( // SplitChecker splits regions when the key range spans across rule/label boundary. type SplitChecker struct { PauseController - cluster schedule.Cluster + cluster sche.CheckerCluster ruleManager *placement.RuleManager labeler *labeler.RegionLabeler } @@ -42,7 +42,7 @@ var ( ) // NewSplitChecker creates a new SplitChecker. -func NewSplitChecker(cluster schedule.Cluster, ruleManager *placement.RuleManager, labeler *labeler.RegionLabeler) *SplitChecker { +func NewSplitChecker(cluster sche.CheckerCluster, ruleManager *placement.RuleManager, labeler *labeler.RegionLabeler) *SplitChecker { return &SplitChecker{ cluster: cluster, ruleManager: ruleManager, @@ -71,7 +71,7 @@ func (c *SplitChecker) Check(region *core.RegionInfo) *operator.Operator { desc := "labeler-split-region" keys := c.labeler.GetSplitKeys(start, end) - if len(keys) == 0 && c.cluster.GetOpts().IsPlacementRulesEnabled() { + if len(keys) == 0 && c.cluster.GetCheckerConfig().IsPlacementRulesEnabled() { desc = "rule-split-region" keys = c.ruleManager.GetSplitKeys(start, end) } diff --git a/pkg/schedule/cluster.go b/pkg/schedule/cluster.go deleted file mode 100644 index da4bf41ef24..00000000000 --- a/pkg/schedule/cluster.go +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright 2017 TiKV Project Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package schedule - -import ( - "github.com/tikv/pd/pkg/core" - "github.com/tikv/pd/pkg/schedule/operator" - "github.com/tikv/pd/pkg/statistics" - "github.com/tikv/pd/pkg/statistics/buckets" -) - -// Cluster provides an overview of a cluster's regions distribution. -type Cluster interface { - core.RegionSetInformer - core.StoreSetInformer - core.StoreSetController - - statistics.RegionStatInformer - statistics.StoreStatInformer - buckets.BucketStatInformer - - operator.ClusterInformer - - RemoveScheduler(name string) error - AddSuspectRegions(ids ...uint64) - SetHotPendingInfluenceMetrics(storeLabel, rwTy, dim string, load float64) - RecordOpStepWithTTL(regionID uint64) -} diff --git a/pkg/schedule/config/config.go b/pkg/schedule/config/config.go index 6b35337a1db..f87b702e0b2 100644 --- a/pkg/schedule/config/config.go +++ b/pkg/schedule/config/config.go @@ -1,99 +1,614 @@ +// Copyright 2023 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package config import ( - "sync" "time" - "github.com/coreos/go-semver/semver" "github.com/pingcap/kvproto/pkg/metapb" - "github.com/tikv/pd/pkg/core/constant" + "github.com/pkg/errors" "github.com/tikv/pd/pkg/core/storelimit" + "github.com/tikv/pd/pkg/utils/configutil" + "github.com/tikv/pd/pkg/utils/syncutil" + "github.com/tikv/pd/pkg/utils/typeutil" +) + +const ( + // DefaultMaxReplicas is the default number of replicas for each region. + DefaultMaxReplicas = 3 + defaultMaxSnapshotCount = 64 + defaultMaxPendingPeerCount = 64 + defaultMaxMergeRegionSize = 20 + defaultLeaderScheduleLimit = 4 + defaultRegionScheduleLimit = 2048 + defaultWitnessScheduleLimit = 4 + defaultReplicaScheduleLimit = 64 + defaultMergeScheduleLimit = 8 + defaultHotRegionScheduleLimit = 4 + defaultTolerantSizeRatio = 0 + defaultLowSpaceRatio = 0.8 + defaultHighSpaceRatio = 0.7 + // defaultHotRegionCacheHitsThreshold is the low hit number threshold of the + // hot region. + defaultHotRegionCacheHitsThreshold = 3 + defaultSchedulerMaxWaitingOperator = 5 + defaultHotRegionsReservedDays = 7 + // When a slow store affected more than 30% of total stores, it will trigger evicting. + defaultSlowStoreEvictingAffectedStoreRatioThreshold = 0.3 + defaultMaxMovableHotPeerSize = int64(512) + + defaultEnableJointConsensus = true + defaultEnableTiKVSplitRegion = true + defaultEnableCrossTableMerge = true + defaultEnableDiagnostic = true + defaultStrictlyMatchLabel = false + defaultEnablePlacementRules = true + defaultEnableWitness = false + defaultHaltScheduling = false + + defaultRegionScoreFormulaVersion = "v2" + defaultLeaderSchedulePolicy = "count" + defaultStoreLimitVersion = "v1" + // DefaultSplitMergeInterval is the default value of config split merge interval. + DefaultSplitMergeInterval = time.Hour + defaultSwitchWitnessInterval = time.Hour + defaultPatrolRegionInterval = 10 * time.Millisecond + defaultMaxStoreDownTime = 30 * time.Minute + defaultHotRegionsWriteInterval = 10 * time.Minute + // It means we skip the preparing stage after the 48 hours no matter if the store has finished preparing stage. + defaultMaxStorePreparingTime = 48 * time.Hour ) -// RejectLeader is the label property type that suggests a store should not -// have any region leaders. -const RejectLeader = "reject-leader" - -var schedulerMap sync.Map - -// RegisterScheduler registers the scheduler type. -func RegisterScheduler(typ string) { - schedulerMap.Store(typ, struct{}{}) -} - -// IsSchedulerRegistered checks if the named scheduler type is registered. -func IsSchedulerRegistered(name string) bool { - _, ok := schedulerMap.Load(name) - return ok -} - -// Config is the interface that wraps the Config related methods. -type Config interface { - GetReplicaScheduleLimit() uint64 - GetRegionScheduleLimit() uint64 - GetMergeScheduleLimit() uint64 - GetLeaderScheduleLimit() uint64 - GetHotRegionScheduleLimit() uint64 - GetWitnessScheduleLimit() uint64 - - GetHotRegionCacheHitsThreshold() int - GetMaxMovableHotPeerSize() int64 - IsTraceRegionFlow() bool - - GetSplitMergeInterval() time.Duration - GetMaxMergeRegionSize() uint64 - GetMaxMergeRegionKeys() uint64 - GetKeyType() constant.KeyType - IsOneWayMergeEnabled() bool - IsCrossTableMergeEnabled() bool - - IsPlacementRulesEnabled() bool - IsPlacementRulesCacheEnabled() bool - - GetMaxReplicas() int - GetPatrolRegionInterval() time.Duration - GetMaxStoreDownTime() time.Duration - GetLocationLabels() []string - GetIsolationLevel() string - IsReplaceOfflineReplicaEnabled() bool - IsMakeUpReplicaEnabled() bool - IsRemoveExtraReplicaEnabled() bool - IsLocationReplacementEnabled() bool - IsRemoveDownReplicaEnabled() bool - - GetSwitchWitnessInterval() time.Duration - IsWitnessAllowed() bool - - GetLowSpaceRatio() float64 - GetHighSpaceRatio() float64 - GetTolerantSizeRatio() float64 - GetLeaderSchedulePolicy() constant.SchedulePolicy - GetRegionScoreFormulaVersion() string - - GetMaxSnapshotCount() uint64 - GetMaxPendingPeerCount() uint64 - GetSchedulerMaxWaitingOperator() uint64 - GetStoreLimitByType(uint64, storelimit.Type) float64 - SetAllStoresLimit(storelimit.Type, float64) - GetSlowStoreEvictingAffectedStoreRatioThreshold() float64 - IsUseJointConsensus() bool - CheckLabelProperty(string, []*metapb.StoreLabel) bool - IsDebugMetricsEnabled() bool - GetClusterVersion() *semver.Version - // for test purpose - SetPlacementRuleEnabled(bool) - SetSplitMergeInterval(time.Duration) - SetMaxReplicas(int) - SetPlacementRulesCacheEnabled(bool) - SetWitnessEnabled(bool) -} - -// StoreConfig is the interface that wraps the StoreConfig related methods. -type StoreConfig interface { - GetRegionMaxSize() uint64 - CheckRegionSize(uint64, uint64) error - CheckRegionKeys(uint64, uint64) error - IsEnableRegionBucket() bool - // for test purpose - SetRegionBucketEnabled(bool) +var ( + defaultLocationLabels = []string{} + // DefaultStoreLimit is the default store limit of add peer and remove peer. + DefaultStoreLimit = StoreLimit{AddPeer: 15, RemovePeer: 15} + // DefaultTiFlashStoreLimit is the default TiFlash store limit of add peer and remove peer. + DefaultTiFlashStoreLimit = StoreLimit{AddPeer: 30, RemovePeer: 30} +) + +// StoreLimit is the default limit of adding peer and removing peer when putting stores. +type StoreLimit struct { + mu syncutil.RWMutex + // AddPeer is the default rate of adding peers for store limit (per minute). + AddPeer float64 + // RemovePeer is the default rate of removing peers for store limit (per minute). + RemovePeer float64 +} + +// SetDefaultStoreLimit sets the default store limit for a given type. +func (sl *StoreLimit) SetDefaultStoreLimit(typ storelimit.Type, ratePerMin float64) { + sl.mu.Lock() + defer sl.mu.Unlock() + switch typ { + case storelimit.AddPeer: + sl.AddPeer = ratePerMin + case storelimit.RemovePeer: + sl.RemovePeer = ratePerMin + } +} + +// GetDefaultStoreLimit gets the default store limit for a given type. +func (sl *StoreLimit) GetDefaultStoreLimit(typ storelimit.Type) float64 { + sl.mu.RLock() + defer sl.mu.RUnlock() + switch typ { + case storelimit.AddPeer: + return sl.AddPeer + case storelimit.RemovePeer: + return sl.RemovePeer + default: + panic("invalid type") + } +} + +func adjustSchedulers(v *SchedulerConfigs, defValue SchedulerConfigs) { + if len(*v) == 0 { + // Make a copy to avoid changing DefaultSchedulers unexpectedly. + // When reloading from storage, the config is passed to json.Unmarshal. + // Without clone, the DefaultSchedulers could be overwritten. + *v = append(defValue[:0:0], defValue...) + } +} + +// ScheduleConfig is the schedule configuration. +// NOTE: This type is exported by HTTP API. Please pay more attention when modifying it. +type ScheduleConfig struct { + // If the snapshot count of one store is greater than this value, + // it will never be used as a source or target store. + MaxSnapshotCount uint64 `toml:"max-snapshot-count" json:"max-snapshot-count"` + MaxPendingPeerCount uint64 `toml:"max-pending-peer-count" json:"max-pending-peer-count"` + // If both the size of region is smaller than MaxMergeRegionSize + // and the number of rows in region is smaller than MaxMergeRegionKeys, + // it will try to merge with adjacent regions. + MaxMergeRegionSize uint64 `toml:"max-merge-region-size" json:"max-merge-region-size"` + MaxMergeRegionKeys uint64 `toml:"max-merge-region-keys" json:"max-merge-region-keys"` + // SplitMergeInterval is the minimum interval time to permit merge after split. + SplitMergeInterval typeutil.Duration `toml:"split-merge-interval" json:"split-merge-interval"` + // SwitchWitnessInterval is the minimum interval that allows a peer to become a witness again after it is promoted to non-witness. + SwitchWitnessInterval typeutil.Duration `toml:"switch-witness-interval" json:"switch-witness-interval"` + // EnableOneWayMerge is the option to enable one way merge. This means a Region can only be merged into the next region of it. + EnableOneWayMerge bool `toml:"enable-one-way-merge" json:"enable-one-way-merge,string"` + // EnableCrossTableMerge is the option to enable cross table merge. This means two Regions can be merged with different table IDs. + // This option only works when key type is "table". + EnableCrossTableMerge bool `toml:"enable-cross-table-merge" json:"enable-cross-table-merge,string"` + // PatrolRegionInterval is the interval for scanning region during patrol. + PatrolRegionInterval typeutil.Duration `toml:"patrol-region-interval" json:"patrol-region-interval"` + // MaxStoreDownTime is the max duration after which + // a store will be considered to be down if it hasn't reported heartbeats. + MaxStoreDownTime typeutil.Duration `toml:"max-store-down-time" json:"max-store-down-time"` + // MaxStorePreparingTime is the max duration after which + // a store will be considered to be preparing. + MaxStorePreparingTime typeutil.Duration `toml:"max-store-preparing-time" json:"max-store-preparing-time"` + // LeaderScheduleLimit is the max coexist leader schedules. + LeaderScheduleLimit uint64 `toml:"leader-schedule-limit" json:"leader-schedule-limit"` + // LeaderSchedulePolicy is the option to balance leader, there are some policies supported: ["count", "size"], default: "count" + LeaderSchedulePolicy string `toml:"leader-schedule-policy" json:"leader-schedule-policy"` + // RegionScheduleLimit is the max coexist region schedules. + RegionScheduleLimit uint64 `toml:"region-schedule-limit" json:"region-schedule-limit"` + // WitnessScheduleLimit is the max coexist witness schedules. + WitnessScheduleLimit uint64 `toml:"witness-schedule-limit" json:"witness-schedule-limit"` + // ReplicaScheduleLimit is the max coexist replica schedules. + ReplicaScheduleLimit uint64 `toml:"replica-schedule-limit" json:"replica-schedule-limit"` + // MergeScheduleLimit is the max coexist merge schedules. + MergeScheduleLimit uint64 `toml:"merge-schedule-limit" json:"merge-schedule-limit"` + // HotRegionScheduleLimit is the max coexist hot region schedules. + HotRegionScheduleLimit uint64 `toml:"hot-region-schedule-limit" json:"hot-region-schedule-limit"` + // HotRegionCacheHitThreshold is the cache hits threshold of the hot region. + // If the number of times a region hits the hot cache is greater than this + // threshold, it is considered a hot region. + HotRegionCacheHitsThreshold uint64 `toml:"hot-region-cache-hits-threshold" json:"hot-region-cache-hits-threshold"` + // StoreBalanceRate is the maximum of balance rate for each store. + // WARN: StoreBalanceRate is deprecated. + StoreBalanceRate float64 `toml:"store-balance-rate" json:"store-balance-rate,omitempty"` + // StoreLimit is the limit of scheduling for stores. + StoreLimit map[uint64]StoreLimitConfig `toml:"store-limit" json:"store-limit"` + // TolerantSizeRatio is the ratio of buffer size for balance scheduler. + TolerantSizeRatio float64 `toml:"tolerant-size-ratio" json:"tolerant-size-ratio"` + // + // high space stage transition stage low space stage + // |--------------------|-----------------------------|-------------------------| + // ^ ^ ^ ^ + // 0 HighSpaceRatio * capacity LowSpaceRatio * capacity capacity + // + // LowSpaceRatio is the lowest usage ratio of store which regraded as low space. + // When in low space, store region score increases to very large and varies inversely with available size. + LowSpaceRatio float64 `toml:"low-space-ratio" json:"low-space-ratio"` + // HighSpaceRatio is the highest usage ratio of store which regraded as high space. + // High space means there is a lot of spare capacity, and store region score varies directly with used size. + HighSpaceRatio float64 `toml:"high-space-ratio" json:"high-space-ratio"` + // RegionScoreFormulaVersion is used to control the formula used to calculate region score. + RegionScoreFormulaVersion string `toml:"region-score-formula-version" json:"region-score-formula-version"` + // SchedulerMaxWaitingOperator is the max coexist operators for each scheduler. + SchedulerMaxWaitingOperator uint64 `toml:"scheduler-max-waiting-operator" json:"scheduler-max-waiting-operator"` + // WARN: DisableLearner is deprecated. + // DisableLearner is the option to disable using AddLearnerNode instead of AddNode. + DisableLearner bool `toml:"disable-raft-learner" json:"disable-raft-learner,string,omitempty"` + // DisableRemoveDownReplica is the option to prevent replica checker from + // removing down replicas. + // WARN: DisableRemoveDownReplica is deprecated. + DisableRemoveDownReplica bool `toml:"disable-remove-down-replica" json:"disable-remove-down-replica,string,omitempty"` + // DisableReplaceOfflineReplica is the option to prevent replica checker from + // replacing offline replicas. + // WARN: DisableReplaceOfflineReplica is deprecated. + DisableReplaceOfflineReplica bool `toml:"disable-replace-offline-replica" json:"disable-replace-offline-replica,string,omitempty"` + // DisableMakeUpReplica is the option to prevent replica checker from making up + // replicas when replica count is less than expected. + // WARN: DisableMakeUpReplica is deprecated. + DisableMakeUpReplica bool `toml:"disable-make-up-replica" json:"disable-make-up-replica,string,omitempty"` + // DisableRemoveExtraReplica is the option to prevent replica checker from + // removing extra replicas. + // WARN: DisableRemoveExtraReplica is deprecated. + DisableRemoveExtraReplica bool `toml:"disable-remove-extra-replica" json:"disable-remove-extra-replica,string,omitempty"` + // DisableLocationReplacement is the option to prevent replica checker from + // moving replica to a better location. + // WARN: DisableLocationReplacement is deprecated. + DisableLocationReplacement bool `toml:"disable-location-replacement" json:"disable-location-replacement,string,omitempty"` + + // EnableRemoveDownReplica is the option to enable replica checker to remove down replica. + EnableRemoveDownReplica bool `toml:"enable-remove-down-replica" json:"enable-remove-down-replica,string"` + // EnableReplaceOfflineReplica is the option to enable replica checker to replace offline replica. + EnableReplaceOfflineReplica bool `toml:"enable-replace-offline-replica" json:"enable-replace-offline-replica,string"` + // EnableMakeUpReplica is the option to enable replica checker to make up replica. + EnableMakeUpReplica bool `toml:"enable-make-up-replica" json:"enable-make-up-replica,string"` + // EnableRemoveExtraReplica is the option to enable replica checker to remove extra replica. + EnableRemoveExtraReplica bool `toml:"enable-remove-extra-replica" json:"enable-remove-extra-replica,string"` + // EnableLocationReplacement is the option to enable replica checker to move replica to a better location. + EnableLocationReplacement bool `toml:"enable-location-replacement" json:"enable-location-replacement,string"` + // EnableDebugMetrics is the option to enable debug metrics. + EnableDebugMetrics bool `toml:"enable-debug-metrics" json:"enable-debug-metrics,string"` + // EnableJointConsensus is the option to enable using joint consensus as an operator step. + EnableJointConsensus bool `toml:"enable-joint-consensus" json:"enable-joint-consensus,string"` + // EnableTiKVSplitRegion is the option to enable tikv split region. + // on ebs-based BR we need to disable it with TTL + EnableTiKVSplitRegion bool `toml:"enable-tikv-split-region" json:"enable-tikv-split-region,string"` + + // Schedulers support for loading customized schedulers + Schedulers SchedulerConfigs `toml:"schedulers" json:"schedulers-v2"` // json v2 is for the sake of compatible upgrade + + // Only used to display + SchedulersPayload map[string]interface{} `toml:"schedulers-payload" json:"schedulers-payload"` + + // Controls the time interval between write hot regions info into leveldb. + HotRegionsWriteInterval typeutil.Duration `toml:"hot-regions-write-interval" json:"hot-regions-write-interval"` + + // The day of hot regions data to be reserved. 0 means close. + HotRegionsReservedDays uint64 `toml:"hot-regions-reserved-days" json:"hot-regions-reserved-days"` + + // MaxMovableHotPeerSize is the threshold of region size for balance hot region and split bucket scheduler. + // Hot region must be split before moved if it's region size is greater than MaxMovableHotPeerSize. + MaxMovableHotPeerSize int64 `toml:"max-movable-hot-peer-size" json:"max-movable-hot-peer-size,omitempty"` + + // EnableDiagnostic is the option to enable using diagnostic + EnableDiagnostic bool `toml:"enable-diagnostic" json:"enable-diagnostic,string"` + + // EnableWitness is the option to enable using witness + EnableWitness bool `toml:"enable-witness" json:"enable-witness,string"` + + // SlowStoreEvictingAffectedStoreRatioThreshold is the affected ratio threshold when judging a store is slow + // A store's slowness must affect more than `store-count * SlowStoreEvictingAffectedStoreRatioThreshold` to trigger evicting. + SlowStoreEvictingAffectedStoreRatioThreshold float64 `toml:"slow-store-evicting-affected-store-ratio-threshold" json:"slow-store-evicting-affected-store-ratio-threshold,omitempty"` + + // StoreLimitVersion is the version of store limit. + // v1: which is based on the region count by rate limit. + // v2: which is based on region size by window size. + StoreLimitVersion string `toml:"store-limit-version" json:"store-limit-version,omitempty"` + + // HaltScheduling is the option to halt the scheduling. Once it's on, PD will halt the scheduling, + // and any other scheduling configs will be ignored. + HaltScheduling bool `toml:"halt-scheduling" json:"halt-scheduling,string,omitempty"` +} + +// Clone returns a cloned scheduling configuration. +func (c *ScheduleConfig) Clone() *ScheduleConfig { + schedulers := append(c.Schedulers[:0:0], c.Schedulers...) + var storeLimit map[uint64]StoreLimitConfig + if c.StoreLimit != nil { + storeLimit = make(map[uint64]StoreLimitConfig, len(c.StoreLimit)) + for k, v := range c.StoreLimit { + storeLimit[k] = v + } + } + cfg := *c + cfg.StoreLimit = storeLimit + cfg.Schedulers = schedulers + cfg.SchedulersPayload = nil + return &cfg +} + +// Adjust adjusts the config. +func (c *ScheduleConfig) Adjust(meta *configutil.ConfigMetaData, reloading bool) error { + if !meta.IsDefined("max-snapshot-count") { + configutil.AdjustUint64(&c.MaxSnapshotCount, defaultMaxSnapshotCount) + } + if !meta.IsDefined("max-pending-peer-count") { + configutil.AdjustUint64(&c.MaxPendingPeerCount, defaultMaxPendingPeerCount) + } + if !meta.IsDefined("max-merge-region-size") { + configutil.AdjustUint64(&c.MaxMergeRegionSize, defaultMaxMergeRegionSize) + } + configutil.AdjustDuration(&c.SplitMergeInterval, DefaultSplitMergeInterval) + configutil.AdjustDuration(&c.SwitchWitnessInterval, defaultSwitchWitnessInterval) + configutil.AdjustDuration(&c.PatrolRegionInterval, defaultPatrolRegionInterval) + configutil.AdjustDuration(&c.MaxStoreDownTime, defaultMaxStoreDownTime) + configutil.AdjustDuration(&c.HotRegionsWriteInterval, defaultHotRegionsWriteInterval) + configutil.AdjustDuration(&c.MaxStorePreparingTime, defaultMaxStorePreparingTime) + if !meta.IsDefined("leader-schedule-limit") { + configutil.AdjustUint64(&c.LeaderScheduleLimit, defaultLeaderScheduleLimit) + } + if !meta.IsDefined("region-schedule-limit") { + configutil.AdjustUint64(&c.RegionScheduleLimit, defaultRegionScheduleLimit) + } + if !meta.IsDefined("witness-schedule-limit") { + configutil.AdjustUint64(&c.WitnessScheduleLimit, defaultWitnessScheduleLimit) + } + if !meta.IsDefined("replica-schedule-limit") { + configutil.AdjustUint64(&c.ReplicaScheduleLimit, defaultReplicaScheduleLimit) + } + if !meta.IsDefined("merge-schedule-limit") { + configutil.AdjustUint64(&c.MergeScheduleLimit, defaultMergeScheduleLimit) + } + if !meta.IsDefined("hot-region-schedule-limit") { + configutil.AdjustUint64(&c.HotRegionScheduleLimit, defaultHotRegionScheduleLimit) + } + if !meta.IsDefined("hot-region-cache-hits-threshold") { + configutil.AdjustUint64(&c.HotRegionCacheHitsThreshold, defaultHotRegionCacheHitsThreshold) + } + if !meta.IsDefined("tolerant-size-ratio") { + configutil.AdjustFloat64(&c.TolerantSizeRatio, defaultTolerantSizeRatio) + } + if !meta.IsDefined("scheduler-max-waiting-operator") { + configutil.AdjustUint64(&c.SchedulerMaxWaitingOperator, defaultSchedulerMaxWaitingOperator) + } + if !meta.IsDefined("leader-schedule-policy") { + configutil.AdjustString(&c.LeaderSchedulePolicy, defaultLeaderSchedulePolicy) + } + if !meta.IsDefined("store-limit-version") { + configutil.AdjustString(&c.StoreLimitVersion, defaultStoreLimitVersion) + } + + if !meta.IsDefined("enable-joint-consensus") { + c.EnableJointConsensus = defaultEnableJointConsensus + } + if !meta.IsDefined("enable-tikv-split-region") { + c.EnableTiKVSplitRegion = defaultEnableTiKVSplitRegion + } + if !meta.IsDefined("enable-cross-table-merge") { + c.EnableCrossTableMerge = defaultEnableCrossTableMerge + } + configutil.AdjustFloat64(&c.LowSpaceRatio, defaultLowSpaceRatio) + configutil.AdjustFloat64(&c.HighSpaceRatio, defaultHighSpaceRatio) + if !meta.IsDefined("enable-diagnostic") { + c.EnableDiagnostic = defaultEnableDiagnostic + } + + if !meta.IsDefined("enable-witness") { + c.EnableWitness = defaultEnableWitness + } + + // new cluster:v2, old cluster:v1 + if !meta.IsDefined("region-score-formula-version") && !reloading { + configutil.AdjustString(&c.RegionScoreFormulaVersion, defaultRegionScoreFormulaVersion) + } + + if !meta.IsDefined("halt-scheduling") { + c.HaltScheduling = defaultHaltScheduling + } + + adjustSchedulers(&c.Schedulers, DefaultSchedulers) + + for k, b := range c.migrateConfigurationMap() { + v, err := c.parseDeprecatedFlag(meta, k, *b[0], *b[1]) + if err != nil { + return err + } + *b[0], *b[1] = false, v // reset old flag false to make it ignored when marshal to JSON + } + + if c.StoreBalanceRate != 0 { + DefaultStoreLimit = StoreLimit{AddPeer: c.StoreBalanceRate, RemovePeer: c.StoreBalanceRate} + c.StoreBalanceRate = 0 + } + + if c.StoreLimit == nil { + c.StoreLimit = make(map[uint64]StoreLimitConfig) + } + + if !meta.IsDefined("hot-regions-reserved-days") { + configutil.AdjustUint64(&c.HotRegionsReservedDays, defaultHotRegionsReservedDays) + } + + if !meta.IsDefined("max-movable-hot-peer-size") { + configutil.AdjustInt64(&c.MaxMovableHotPeerSize, defaultMaxMovableHotPeerSize) + } + + if !meta.IsDefined("slow-store-evicting-affected-store-ratio-threshold") { + configutil.AdjustFloat64(&c.SlowStoreEvictingAffectedStoreRatioThreshold, defaultSlowStoreEvictingAffectedStoreRatioThreshold) + } + return c.Validate() +} + +func (c *ScheduleConfig) migrateConfigurationMap() map[string][2]*bool { + return map[string][2]*bool{ + "remove-down-replica": {&c.DisableRemoveDownReplica, &c.EnableRemoveDownReplica}, + "replace-offline-replica": {&c.DisableReplaceOfflineReplica, &c.EnableReplaceOfflineReplica}, + "make-up-replica": {&c.DisableMakeUpReplica, &c.EnableMakeUpReplica}, + "remove-extra-replica": {&c.DisableRemoveExtraReplica, &c.EnableRemoveExtraReplica}, + "location-replacement": {&c.DisableLocationReplacement, &c.EnableLocationReplacement}, + } +} + +// GetMaxMergeRegionKeys returns the max merge keys. +// it should keep consistent with tikv: https://github.com/tikv/tikv/pull/12484 +func (c *ScheduleConfig) GetMaxMergeRegionKeys() uint64 { + if keys := c.MaxMergeRegionKeys; keys != 0 { + return keys + } + return c.MaxMergeRegionSize * 10000 +} + +func (c *ScheduleConfig) parseDeprecatedFlag(meta *configutil.ConfigMetaData, name string, old, new bool) (bool, error) { + oldName, newName := "disable-"+name, "enable-"+name + defineOld, defineNew := meta.IsDefined(oldName), meta.IsDefined(newName) + switch { + case defineNew && defineOld: + if new == old { + return false, errors.Errorf("config item %s and %s(deprecated) are conflict", newName, oldName) + } + return new, nil + case defineNew && !defineOld: + return new, nil + case !defineNew && defineOld: + return !old, nil // use !disable-* + case !defineNew && !defineOld: + return true, nil // use default value true + } + return false, nil // unreachable. +} + +// MigrateDeprecatedFlags updates new flags according to deprecated flags. +func (c *ScheduleConfig) MigrateDeprecatedFlags() { + c.DisableLearner = false + if c.StoreBalanceRate != 0 { + DefaultStoreLimit = StoreLimit{AddPeer: c.StoreBalanceRate, RemovePeer: c.StoreBalanceRate} + c.StoreBalanceRate = 0 + } + for _, b := range c.migrateConfigurationMap() { + // If old=false (previously disabled), set both old and new to false. + if *b[0] { + *b[0], *b[1] = false, false + } + } +} + +// Validate is used to validate if some scheduling configurations are right. +func (c *ScheduleConfig) Validate() error { + if c.TolerantSizeRatio < 0 { + return errors.New("tolerant-size-ratio should be non-negative") + } + if c.LowSpaceRatio < 0 || c.LowSpaceRatio > 1 { + return errors.New("low-space-ratio should between 0 and 1") + } + if c.HighSpaceRatio < 0 || c.HighSpaceRatio > 1 { + return errors.New("high-space-ratio should between 0 and 1") + } + if c.LowSpaceRatio <= c.HighSpaceRatio { + return errors.New("low-space-ratio should be larger than high-space-ratio") + } + if c.LeaderSchedulePolicy != "count" && c.LeaderSchedulePolicy != "size" { + return errors.Errorf("leader-schedule-policy %v is invalid", c.LeaderSchedulePolicy) + } + if c.SlowStoreEvictingAffectedStoreRatioThreshold == 0 { + return errors.Errorf("slow-store-evicting-affected-store-ratio-threshold is not set") + } + return nil +} + +// Deprecated is used to find if there is an option has been deprecated. +func (c *ScheduleConfig) Deprecated() error { + if c.DisableLearner { + return errors.New("disable-raft-learner has already been deprecated") + } + if c.DisableRemoveDownReplica { + return errors.New("disable-remove-down-replica has already been deprecated") + } + if c.DisableReplaceOfflineReplica { + return errors.New("disable-replace-offline-replica has already been deprecated") + } + if c.DisableMakeUpReplica { + return errors.New("disable-make-up-replica has already been deprecated") + } + if c.DisableRemoveExtraReplica { + return errors.New("disable-remove-extra-replica has already been deprecated") + } + if c.DisableLocationReplacement { + return errors.New("disable-location-replacement has already been deprecated") + } + if c.StoreBalanceRate != 0 { + return errors.New("store-balance-rate has already been deprecated") + } + return nil +} + +// StoreLimitConfig is a config about scheduling rate limit of different types for a store. +type StoreLimitConfig struct { + AddPeer float64 `toml:"add-peer" json:"add-peer"` + RemovePeer float64 `toml:"remove-peer" json:"remove-peer"` +} + +// SchedulerConfigs is a slice of customized scheduler configuration. +type SchedulerConfigs []SchedulerConfig + +// SchedulerConfig is customized scheduler configuration +type SchedulerConfig struct { + Type string `toml:"type" json:"type"` + Args []string `toml:"args" json:"args"` + Disable bool `toml:"disable" json:"disable"` + ArgsPayload string `toml:"args-payload" json:"args-payload"` +} + +// DefaultSchedulers are the schedulers be created by default. +// If these schedulers are not in the persistent configuration, they +// will be created automatically when reloading. +var DefaultSchedulers = SchedulerConfigs{ + {Type: "balance-region"}, + {Type: "balance-leader"}, + {Type: "balance-witness"}, + {Type: "hot-region"}, + {Type: "transfer-witness-leader"}, +} + +// IsDefaultScheduler checks whether the scheduler is enable by default. +func IsDefaultScheduler(typ string) bool { + for _, c := range DefaultSchedulers { + if typ == c.Type { + return true + } + } + return false +} + +// ReplicationConfig is the replication configuration. +// NOTE: This type is exported by HTTP API. Please pay more attention when modifying it. +type ReplicationConfig struct { + // MaxReplicas is the number of replicas for each region. + MaxReplicas uint64 `toml:"max-replicas" json:"max-replicas"` + + // The label keys specified the location of a store. + // The placement priorities is implied by the order of label keys. + // For example, ["zone", "rack"] means that we should place replicas to + // different zones first, then to different racks if we don't have enough zones. + LocationLabels typeutil.StringSlice `toml:"location-labels" json:"location-labels"` + // StrictlyMatchLabel strictly checks if the label of TiKV is matched with LocationLabels. + StrictlyMatchLabel bool `toml:"strictly-match-label" json:"strictly-match-label,string"` + + // When PlacementRules feature is enabled. MaxReplicas, LocationLabels and IsolationLabels are not used any more. + EnablePlacementRules bool `toml:"enable-placement-rules" json:"enable-placement-rules,string"` + + // EnablePlacementRuleCache controls whether use cache during rule checker + EnablePlacementRulesCache bool `toml:"enable-placement-rules-cache" json:"enable-placement-rules-cache,string"` + + // IsolationLevel is used to isolate replicas explicitly and forcibly if it's not empty. + // Its value must be empty or one of LocationLabels. + // Example: + // location-labels = ["zone", "rack", "host"] + // isolation-level = "zone" + // With configuration like above, PD ensure that all replicas be placed in different zones. + // Even if a zone is down, PD will not try to make up replicas in other zone + // because other zones already have replicas on it. + IsolationLevel string `toml:"isolation-level" json:"isolation-level"` +} + +// Clone makes a deep copy of the config. +func (c *ReplicationConfig) Clone() *ReplicationConfig { + locationLabels := append(c.LocationLabels[:0:0], c.LocationLabels...) + cfg := *c + cfg.LocationLabels = locationLabels + return &cfg +} + +// Validate is used to validate if some replication configurations are right. +func (c *ReplicationConfig) Validate() error { + foundIsolationLevel := false + for _, label := range c.LocationLabels { + err := ValidateLabels([]*metapb.StoreLabel{{Key: label}}) + if err != nil { + return err + } + // IsolationLevel should be empty or one of LocationLabels + if !foundIsolationLevel && label == c.IsolationLevel { + foundIsolationLevel = true + } + } + if c.IsolationLevel != "" && !foundIsolationLevel { + return errors.New("isolation-level must be one of location-labels or empty") + } + return nil +} + +// Adjust adjusts the config. +func (c *ReplicationConfig) Adjust(meta *configutil.ConfigMetaData) error { + configutil.AdjustUint64(&c.MaxReplicas, DefaultMaxReplicas) + if !meta.IsDefined("enable-placement-rules") { + c.EnablePlacementRules = defaultEnablePlacementRules + } + if !meta.IsDefined("strictly-match-label") { + c.StrictlyMatchLabel = defaultStrictlyMatchLabel + } + if !meta.IsDefined("location-labels") { + c.LocationLabels = defaultLocationLabels + } + return c.Validate() } diff --git a/pkg/schedule/config/config_provider.go b/pkg/schedule/config/config_provider.go new file mode 100644 index 00000000000..3724d62e776 --- /dev/null +++ b/pkg/schedule/config/config_provider.go @@ -0,0 +1,141 @@ +// Copyright 2023 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import ( + "sync" + "time" + + "github.com/coreos/go-semver/semver" + "github.com/pingcap/kvproto/pkg/metapb" + "github.com/tikv/pd/pkg/core/constant" + "github.com/tikv/pd/pkg/core/storelimit" + "github.com/tikv/pd/pkg/storage/endpoint" +) + +// RejectLeader is the label property type that suggests a store should not +// have any region leaders. +const RejectLeader = "reject-leader" + +var schedulerMap sync.Map + +// RegisterScheduler registers the scheduler type. +func RegisterScheduler(typ string) { + schedulerMap.Store(typ, struct{}{}) +} + +// IsSchedulerRegistered checks if the named scheduler type is registered. +func IsSchedulerRegistered(name string) bool { + _, ok := schedulerMap.Load(name) + return ok +} + +// SchedulerConfigProvider is the interface for scheduler configurations. +type SchedulerConfigProvider interface { + SharedConfigProvider + + IsSchedulingHalted() bool + + IsSchedulerDisabled(string) bool + AddSchedulerCfg(string, []string) + RemoveSchedulerCfg(string) + Persist(endpoint.ConfigStorage) error + + GetRegionScheduleLimit() uint64 + GetLeaderScheduleLimit() uint64 + GetHotRegionScheduleLimit() uint64 + GetWitnessScheduleLimit() uint64 + + GetHotRegionCacheHitsThreshold() int + GetMaxMovableHotPeerSize() int64 + IsTraceRegionFlow() bool + + GetTolerantSizeRatio() float64 + GetLeaderSchedulePolicy() constant.SchedulePolicy + + IsDebugMetricsEnabled() bool + IsDiagnosticAllowed() bool + GetSlowStoreEvictingAffectedStoreRatioThreshold() float64 +} + +// CheckerConfigProvider is the interface for checker configurations. +type CheckerConfigProvider interface { + SharedConfigProvider + + GetSwitchWitnessInterval() time.Duration + IsRemoveExtraReplicaEnabled() bool + IsRemoveDownReplicaEnabled() bool + IsReplaceOfflineReplicaEnabled() bool + IsMakeUpReplicaEnabled() bool + IsLocationReplacementEnabled() bool + GetIsolationLevel() string + GetSplitMergeInterval() time.Duration + GetPatrolRegionInterval() time.Duration + GetMaxMergeRegionSize() uint64 + GetMaxMergeRegionKeys() uint64 + GetReplicaScheduleLimit() uint64 +} + +// SharedConfigProvider is the interface for shared configurations. +type SharedConfigProvider interface { + GetMaxReplicas() int + IsPlacementRulesEnabled() bool + GetMaxSnapshotCount() uint64 + GetMaxPendingPeerCount() uint64 + GetLowSpaceRatio() float64 + GetHighSpaceRatio() float64 + GetMaxStoreDownTime() time.Duration + GetLocationLabels() []string + CheckLabelProperty(string, []*metapb.StoreLabel) bool + GetClusterVersion() *semver.Version + IsUseJointConsensus() bool + GetKeyType() constant.KeyType + IsCrossTableMergeEnabled() bool + IsOneWayMergeEnabled() bool + GetMergeScheduleLimit() uint64 + GetRegionScoreFormulaVersion() string + GetSchedulerMaxWaitingOperator() uint64 + GetStoreLimitByType(uint64, storelimit.Type) float64 + IsWitnessAllowed() bool + IsPlacementRulesCacheEnabled() bool + + // for test purpose + SetPlacementRulesCacheEnabled(bool) + SetEnableWitness(bool) +} + +// ConfProvider is the interface that wraps the ConfProvider related methods. +type ConfProvider interface { + SchedulerConfigProvider + CheckerConfigProvider + // for test purpose + SetPlacementRuleEnabled(bool) + SetSplitMergeInterval(time.Duration) + SetMaxReplicas(int) + SetAllStoresLimit(typ storelimit.Type, ratePerMin float64) + // only for store configuration + UseRaftV2() +} + +// StoreConfigProvider is the interface that wraps the StoreConfigProvider related methods. +type StoreConfigProvider interface { + GetRegionMaxSize() uint64 + CheckRegionSize(uint64, uint64) error + CheckRegionKeys(uint64, uint64) error + IsEnableRegionBucket() bool + IsRaftKV2() bool + // for test purpose + SetRegionBucketEnabled(bool) +} diff --git a/pkg/schedule/config/util.go b/pkg/schedule/config/util.go new file mode 100644 index 00000000000..172bd947bba --- /dev/null +++ b/pkg/schedule/config/util.go @@ -0,0 +1,57 @@ +// Copyright 2023 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import ( + "regexp" + + "github.com/pingcap/errors" + "github.com/pingcap/kvproto/pkg/metapb" +) + +const ( + // Label key consists of alphanumeric characters, '-', '_', '.' or '/', and must start and end with an + // alphanumeric character. If can also contain an extra '$' at the beginning. + keyFormat = "^[$]?[A-Za-z0-9]([-A-Za-z0-9_./]*[A-Za-z0-9])?$" + // Value key can be any combination of alphanumeric characters, '-', '_', '.' or '/'. It can also be empty to + // mark the label as deleted. + valueFormat = "^[-A-Za-z0-9_./]*$" +) + +func validateFormat(s, format string) error { + isValid, _ := regexp.MatchString(format, s) + if !isValid { + return errors.Errorf("%s does not match format %q", s, format) + } + return nil +} + +// ValidateLabels checks the legality of the labels. +func ValidateLabels(labels []*metapb.StoreLabel) error { + for _, label := range labels { + if err := validateFormat(label.Key, keyFormat); err != nil { + return err + } + if err := validateFormat(label.Value, valueFormat); err != nil { + return err + } + } + return nil +} + +// ValidateLabelKey checks the legality of the label key. +func ValidateLabelKey(key string) error { + return validateFormat(key, keyFormat) +} diff --git a/pkg/schedule/config/util_test.go b/pkg/schedule/config/util_test.go new file mode 100644 index 00000000000..31ab3ccf2a6 --- /dev/null +++ b/pkg/schedule/config/util_test.go @@ -0,0 +1,56 @@ +// Copyright 2023 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import ( + "testing" + + "github.com/pingcap/kvproto/pkg/metapb" + "github.com/stretchr/testify/require" +) + +func TestValidateLabels(t *testing.T) { + re := require.New(t) + tests := []struct { + label string + hasErr bool + }{ + {"z1", false}, + {"z-1", false}, + {"h1;", true}, + {"z_1", false}, + {"z_1&", true}, + {"cn", false}, + {"Zo^ne", true}, + {"z_", true}, + {"hos&t-15", true}, + {"_test1", true}, + {"-test1", true}, + {"127.0.0.1", false}, + {"www.pingcap.com", false}, + {"h_127.0.0.1", false}, + {"a", false}, + {"a/b", false}, + {"ab/", true}, + {"/ab", true}, + {"$abc", false}, + {"$", true}, + {"a$b", true}, + {"$$", true}, + } + for _, test := range tests { + re.Equal(test.hasErr, ValidateLabels([]*metapb.StoreLabel{{Key: test.label}}) != nil) + } +} diff --git a/server/cluster/coordinator.go b/pkg/schedule/coordinator.go similarity index 50% rename from server/cluster/coordinator.go rename to pkg/schedule/coordinator.go index 932e1d185cd..16d72ff913b 100644 --- a/server/cluster/coordinator.go +++ b/pkg/schedule/coordinator.go @@ -12,15 +12,13 @@ // See the License for the specific language governing permissions and // limitations under the License. -package cluster +package schedule import ( "bytes" "context" - "net/http" "strconv" "sync" - "sync/atomic" "time" "github.com/pingcap/errors" @@ -29,17 +27,18 @@ import ( "github.com/tikv/pd/pkg/cache" "github.com/tikv/pd/pkg/core" "github.com/tikv/pd/pkg/errs" - "github.com/tikv/pd/pkg/schedule" "github.com/tikv/pd/pkg/schedule/checker" + sc "github.com/tikv/pd/pkg/schedule/config" + sche "github.com/tikv/pd/pkg/schedule/core" + "github.com/tikv/pd/pkg/schedule/diagnostic" "github.com/tikv/pd/pkg/schedule/hbstream" "github.com/tikv/pd/pkg/schedule/operator" - "github.com/tikv/pd/pkg/schedule/plan" + "github.com/tikv/pd/pkg/schedule/scatter" "github.com/tikv/pd/pkg/schedule/schedulers" + "github.com/tikv/pd/pkg/schedule/splitter" "github.com/tikv/pd/pkg/statistics" - "github.com/tikv/pd/pkg/storage" "github.com/tikv/pd/pkg/utils/logutil" "github.com/tikv/pd/pkg/utils/syncutil" - "github.com/tikv/pd/server/config" "go.uber.org/zap" ) @@ -48,8 +47,9 @@ const ( checkSuspectRangesInterval = 100 * time.Millisecond collectFactor = 0.9 collectTimeout = 5 * time.Minute - maxScheduleRetries = 10 maxLoadConfigRetries = 10 + // pushOperatorTickInterval is the interval try to push the operator. + pushOperatorTickInterval = 500 * time.Millisecond patrolScanRegionLimit = 128 // It takes about 14 minutes to iterate 1 million regions. // PluginLoad means action for load plugin @@ -64,64 +64,67 @@ var ( priorityListGauge = regionListGauge.WithLabelValues("priority_list") ) -// coordinator is used to manage all schedulers and checkers to decide if the region needs to be scheduled. -type coordinator struct { +// Coordinator is used to manage all schedulers and checkers to decide if the region needs to be scheduled. +type Coordinator struct { syncutil.RWMutex wg sync.WaitGroup ctx context.Context cancel context.CancelFunc - cluster *RaftCluster + cluster sche.ClusterInformer prepareChecker *prepareChecker checkers *checker.Controller - regionScatterer *schedule.RegionScatterer - regionSplitter *schedule.RegionSplitter - schedulers map[string]*scheduleController - opController *schedule.OperatorController + regionScatterer *scatter.RegionScatterer + regionSplitter *splitter.RegionSplitter + schedulers *schedulers.Controller + opController *operator.Controller hbStreams *hbstream.HeartbeatStreams - pluginInterface *schedule.PluginInterface - diagnosticManager *diagnosticManager + pluginInterface *PluginInterface + diagnosticManager *diagnostic.Manager } -// newCoordinator creates a new coordinator. -func newCoordinator(ctx context.Context, cluster *RaftCluster, hbStreams *hbstream.HeartbeatStreams) *coordinator { +// NewCoordinator creates a new Coordinator. +func NewCoordinator(ctx context.Context, cluster sche.ClusterInformer, hbStreams *hbstream.HeartbeatStreams) *Coordinator { ctx, cancel := context.WithCancel(ctx) - opController := schedule.NewOperatorController(ctx, cluster, hbStreams) - schedulers := make(map[string]*scheduleController) - return &coordinator{ + opController := operator.NewController(ctx, cluster.GetBasicCluster(), cluster.GetSharedConfig(), hbStreams) + schedulers := schedulers.NewController(ctx, cluster, cluster.GetStorage(), opController) + return &Coordinator{ ctx: ctx, cancel: cancel, cluster: cluster, prepareChecker: newPrepareChecker(), - checkers: checker.NewController(ctx, cluster, cluster.opt, cluster.ruleManager, cluster.regionLabeler, opController), - regionScatterer: schedule.NewRegionScatterer(ctx, cluster, opController), - regionSplitter: schedule.NewRegionSplitter(cluster, schedule.NewSplitRegionsHandler(cluster, opController)), + checkers: checker.NewController(ctx, cluster, cluster.GetCheckerConfig(), cluster.GetRuleManager(), cluster.GetRegionLabeler(), opController), + regionScatterer: scatter.NewRegionScatterer(ctx, cluster, opController), + regionSplitter: splitter.NewRegionSplitter(cluster, splitter.NewSplitRegionsHandler(cluster, opController)), schedulers: schedulers, opController: opController, hbStreams: hbStreams, - pluginInterface: schedule.NewPluginInterface(), - diagnosticManager: newDiagnosticManager(cluster), + pluginInterface: NewPluginInterface(), + diagnosticManager: diagnostic.NewManager(schedulers, cluster.GetSchedulerConfig()), } } -func (c *coordinator) GetWaitingRegions() []*cache.Item { +// GetWaitingRegions returns the regions in the waiting list. +func (c *Coordinator) GetWaitingRegions() []*cache.Item { return c.checkers.GetWaitingRegions() } -func (c *coordinator) IsPendingRegion(region uint64) bool { +// IsPendingRegion returns if the region is in the pending list. +func (c *Coordinator) IsPendingRegion(region uint64) bool { return c.checkers.IsPendingRegion(region) } -// patrolRegions is used to scan regions. +// PatrolRegions is used to scan regions. // The checkers will check these regions to decide if they need to do some operations. -func (c *coordinator) patrolRegions() { +// The function is exposed for test purpose. +func (c *Coordinator) PatrolRegions() { defer logutil.LogPanic() defer c.wg.Done() - timer := time.NewTimer(c.cluster.GetOpts().GetPatrolRegionInterval()) - defer timer.Stop() + ticker := time.NewTicker(c.cluster.GetCheckerConfig().GetPatrolRegionInterval()) + defer ticker.Stop() - log.Info("coordinator starts patrol regions") + log.Info("Coordinator starts patrol regions") start := time.Now() var ( key []byte @@ -129,14 +132,14 @@ func (c *coordinator) patrolRegions() { ) for { select { - case <-timer.C: - timer.Reset(c.cluster.GetOpts().GetPatrolRegionInterval()) + case <-ticker.C: + // Note: we reset the ticker here to support updating configuration dynamically. + ticker.Reset(c.cluster.GetCheckerConfig().GetPatrolRegionInterval()) case <-c.ctx.Done(): log.Info("patrol regions has been stopped") return } - if c.cluster.GetUnsafeRecoveryController().IsRunning() { - // Skip patrolling regions during unsafe recovery. + if c.isSchedulingHalted() { continue } @@ -152,7 +155,7 @@ func (c *coordinator) patrolRegions() { continue } // Updates the label level isolation statistics. - c.cluster.updateRegionsLabelLevelStats(regions) + c.cluster.UpdateRegionsLabelLevelStats(regions) if len(key) == 0 { patrolCheckRegionsGauge.Set(time.Since(start).Seconds()) start = time.Now() @@ -163,7 +166,11 @@ func (c *coordinator) patrolRegions() { } } -func (c *coordinator) checkRegions(startKey []byte) (key []byte, regions []*core.RegionInfo) { +func (c *Coordinator) isSchedulingHalted() bool { + return c.cluster.GetPersistOptions().IsSchedulingHalted() +} + +func (c *Coordinator) checkRegions(startKey []byte) (key []byte, regions []*core.RegionInfo) { regions = c.cluster.ScanRegions(startKey, nil, patrolScanRegionLimit) if len(regions) == 0 { // Resets the scan key. @@ -178,14 +185,14 @@ func (c *coordinator) checkRegions(startKey []byte) (key []byte, regions []*core return } -func (c *coordinator) checkSuspectRegions() { +func (c *Coordinator) checkSuspectRegions() { for _, id := range c.checkers.GetSuspectRegions() { region := c.cluster.GetRegion(id) c.tryAddOperators(region) } } -func (c *coordinator) checkWaitingRegions() { +func (c *Coordinator) checkWaitingRegions() { items := c.checkers.GetWaitingRegions() waitingListGauge.Set(float64(len(items))) for _, item := range items { @@ -195,7 +202,7 @@ func (c *coordinator) checkWaitingRegions() { } // checkPriorityRegions checks priority regions -func (c *coordinator) checkPriorityRegions() { +func (c *Coordinator) checkPriorityRegions() { items := c.checkers.GetPriorityRegions() removes := make([]uint64, 0) priorityListGauge.Set(float64(len(items))) @@ -222,9 +229,10 @@ func (c *coordinator) checkPriorityRegions() { // checkSuspectRanges would pop one suspect key range group // The regions of new version key range and old version key range would be placed into // the suspect regions map -func (c *coordinator) checkSuspectRanges() { +func (c *Coordinator) checkSuspectRanges() { + defer logutil.LogPanic() defer c.wg.Done() - log.Info("coordinator begins to check suspect key ranges") + log.Info("Coordinator begins to check suspect key ranges") ticker := time.NewTicker(checkSuspectRangesInterval) defer ticker.Stop() for { @@ -258,7 +266,7 @@ func (c *coordinator) checkSuspectRanges() { } } -func (c *coordinator) tryAddOperators(region *core.RegionInfo) { +func (c *Coordinator) tryAddOperators(region *core.RegionInfo) { if region == nil { // the region could be recent split, continue to wait. return @@ -284,12 +292,12 @@ func (c *coordinator) tryAddOperators(region *core.RegionInfo) { } // drivePushOperator is used to push the unfinished operator to the executor. -func (c *coordinator) drivePushOperator() { +func (c *Coordinator) drivePushOperator() { defer logutil.LogPanic() defer c.wg.Done() - log.Info("coordinator begins to actively drive push operator") - ticker := time.NewTicker(schedule.PushOperatorTickInterval) + log.Info("Coordinator begins to actively drive push operator") + ticker := time.NewTicker(pushOperatorTickInterval) defer ticker.Stop() for { select { @@ -297,49 +305,63 @@ func (c *coordinator) drivePushOperator() { log.Info("drive push operator has been stopped") return case <-ticker.C: - c.opController.PushOperators() + c.opController.PushOperators(c.RecordOpStepWithTTL) } } } -func (c *coordinator) runUntilStop() { - c.run() +// RunUntilStop runs the coordinator until receiving the stop signal. +func (c *Coordinator) RunUntilStop() { + c.Run() <-c.ctx.Done() - log.Info("coordinator is stopping") + log.Info("Coordinator is stopping") + c.GetSchedulersController().Wait() c.wg.Wait() - log.Info("coordinator has been stopped") + log.Info("Coordinator has been stopped") } -func (c *coordinator) run() { +// Run starts coordinator. +func (c *Coordinator) Run() { ticker := time.NewTicker(runSchedulerCheckInterval) failpoint.Inject("changeCoordinatorTicker", func() { ticker = time.NewTicker(100 * time.Millisecond) }) defer ticker.Stop() - log.Info("coordinator starts to collect cluster information") + log.Info("Coordinator starts to collect cluster information") for { - if c.shouldRun() { - log.Info("coordinator has finished cluster information preparation") + if c.ShouldRun() { + log.Info("Coordinator has finished cluster information preparation") break } select { case <-ticker.C: case <-c.ctx.Done(): - log.Info("coordinator stops running") + log.Info("Coordinator stops running") return } } - log.Info("coordinator starts to run schedulers") + log.Info("Coordinator starts to run schedulers") + c.initSchedulers() + + c.wg.Add(3) + // Starts to patrol regions. + go c.PatrolRegions() + // Checks suspect key ranges + go c.checkSuspectRanges() + go c.drivePushOperator() +} + +func (c *Coordinator) initSchedulers() { var ( scheduleNames []string configs []string err error ) for i := 0; i < maxLoadConfigRetries; i++ { - scheduleNames, configs, err = c.cluster.storage.LoadAllScheduleConfig() + scheduleNames, configs, err = c.cluster.GetStorage().LoadAllScheduleConfig() select { case <-c.ctx.Done(): - log.Info("coordinator stops running") + log.Info("Coordinator stops running") return default: } @@ -352,12 +374,12 @@ func (c *coordinator) run() { log.Fatal("cannot load schedulers' config", errs.ZapError(err)) } - scheduleCfg := c.cluster.opt.GetScheduleConfig().Clone() + scheduleCfg := c.cluster.GetPersistOptions().GetScheduleConfig().Clone() // The new way to create scheduler with the independent configuration. for i, name := range scheduleNames { data := configs[i] - typ := schedule.FindSchedulerTypeByName(name) - var cfg config.SchedulerConfig + typ := schedulers.FindSchedulerTypeByName(name) + var cfg sc.SchedulerConfig for _, c := range scheduleCfg.Schedulers { if c.Type == typ { cfg = c @@ -372,13 +394,13 @@ func (c *coordinator) run() { log.Info("skip create scheduler with independent configuration", zap.String("scheduler-name", name), zap.String("scheduler-type", cfg.Type), zap.Strings("scheduler-args", cfg.Args)) continue } - s, err := schedule.CreateScheduler(cfg.Type, c.opController, c.cluster.storage, schedule.ConfigJSONDecoder([]byte(data))) + s, err := schedulers.CreateScheduler(cfg.Type, c.opController, c.cluster.GetStorage(), schedulers.ConfigJSONDecoder([]byte(data)), c.schedulers.RemoveScheduler) if err != nil { log.Error("can not create scheduler with independent configuration", zap.String("scheduler-name", name), zap.Strings("scheduler-args", cfg.Args), errs.ZapError(err)) continue } log.Info("create scheduler with independent configuration", zap.String("scheduler-name", s.GetName())) - if err = c.addScheduler(s); err != nil { + if err = c.schedulers.AddScheduler(s); err != nil { log.Error("can not add scheduler with independent configuration", zap.String("scheduler-name", s.GetName()), zap.Strings("scheduler-args", cfg.Args), errs.ZapError(err)) } } @@ -393,14 +415,14 @@ func (c *coordinator) run() { continue } - s, err := schedule.CreateScheduler(schedulerCfg.Type, c.opController, c.cluster.storage, schedule.ConfigSliceDecoder(schedulerCfg.Type, schedulerCfg.Args)) + s, err := schedulers.CreateScheduler(schedulerCfg.Type, c.opController, c.cluster.GetStorage(), schedulers.ConfigSliceDecoder(schedulerCfg.Type, schedulerCfg.Args), c.schedulers.RemoveScheduler) if err != nil { log.Error("can not create scheduler", zap.String("scheduler-type", schedulerCfg.Type), zap.Strings("scheduler-args", schedulerCfg.Args), errs.ZapError(err)) continue } log.Info("create scheduler", zap.String("scheduler-name", s.GetName()), zap.Strings("scheduler-args", schedulerCfg.Args)) - if err = c.addScheduler(s, schedulerCfg.Args...); err != nil && !errors.ErrorEqual(err, errs.ErrSchedulerExisted.FastGenByArgs()) { + if err = c.schedulers.AddScheduler(s, schedulerCfg.Args...); err != nil && !errors.ErrorEqual(err, errs.ErrSchedulerExisted.FastGenByArgs()) { log.Error("can not add scheduler", zap.String("scheduler-name", s.GetName()), zap.Strings("scheduler-args", schedulerCfg.Args), errs.ZapError(err)) } else { // Only records the valid scheduler config. @@ -411,21 +433,14 @@ func (c *coordinator) run() { // Removes the invalid scheduler config and persist. scheduleCfg.Schedulers = scheduleCfg.Schedulers[:k] - c.cluster.opt.SetScheduleConfig(scheduleCfg) - if err := c.cluster.opt.Persist(c.cluster.storage); err != nil { + c.cluster.GetPersistOptions().SetScheduleConfig(scheduleCfg) + if err := c.cluster.GetPersistOptions().Persist(c.cluster.GetStorage()); err != nil { log.Error("cannot persist schedule config", errs.ZapError(err)) } - - c.wg.Add(3) - // Starts to patrol regions. - go c.patrolRegions() - // Checks suspect key ranges - go c.checkSuspectRanges() - go c.drivePushOperator() } // LoadPlugin load user plugin -func (c *coordinator) LoadPlugin(pluginPath string, ch chan string) { +func (c *Coordinator) LoadPlugin(pluginPath string, ch chan string) { log.Info("load plugin", zap.String("plugin-path", pluginPath)) // get func: SchedulerType from plugin SchedulerType, err := c.pluginInterface.GetFunction(pluginPath, "SchedulerType") @@ -442,13 +457,13 @@ func (c *coordinator) LoadPlugin(pluginPath string, ch chan string) { } schedulerArgs := SchedulerArgs.(func() []string) // create and add user scheduler - s, err := schedule.CreateScheduler(schedulerType(), c.opController, c.cluster.storage, schedule.ConfigSliceDecoder(schedulerType(), schedulerArgs())) + s, err := schedulers.CreateScheduler(schedulerType(), c.opController, c.cluster.GetStorage(), schedulers.ConfigSliceDecoder(schedulerType(), schedulerArgs()), c.schedulers.RemoveScheduler) if err != nil { log.Error("can not create scheduler", zap.String("scheduler-type", schedulerType()), errs.ZapError(err)) return } log.Info("create scheduler", zap.String("scheduler-name", s.GetName())) - if err = c.addScheduler(s); err != nil { + if err = c.schedulers.AddScheduler(s); err != nil { log.Error("can't add scheduler", zap.String("scheduler-name", s.GetName()), errs.ZapError(err)) return } @@ -457,7 +472,7 @@ func (c *coordinator) LoadPlugin(pluginPath string, ch chan string) { go c.waitPluginUnload(pluginPath, s.GetName(), ch) } -func (c *coordinator) waitPluginUnload(pluginPath, schedulerName string, ch chan string) { +func (c *Coordinator) waitPluginUnload(pluginPath, schedulerName string, ch chan string) { defer logutil.LogPanic() defer c.wg.Done() // Get signal from channel which means user unload the plugin @@ -465,7 +480,7 @@ func (c *coordinator) waitPluginUnload(pluginPath, schedulerName string, ch chan select { case action := <-ch: if action == PluginUnload { - err := c.removeScheduler(schedulerName) + err := c.schedulers.RemoveScheduler(schedulerName) if err != nil { log.Error("can not remove scheduler", zap.String("scheduler-name", schedulerName), errs.ZapError(err)) } else { @@ -482,12 +497,14 @@ func (c *coordinator) waitPluginUnload(pluginPath, schedulerName string, ch chan } } -func (c *coordinator) stop() { +// Stop stops the coordinator. +func (c *Coordinator) Stop() { c.cancel() } -func (c *coordinator) getHotRegionsByType(typ statistics.RWType) *statistics.StoreHotPeersInfos { - isTraceFlow := c.cluster.GetOpts().IsTraceRegionFlow() +// GetHotRegionsByType gets hot regions' statistics by RWType. +func (c *Coordinator) GetHotRegionsByType(typ statistics.RWType) *statistics.StoreHotPeersInfos { + isTraceFlow := c.cluster.GetSchedulerConfig().IsTraceRegionFlow() storeLoads := c.cluster.GetStoresLoads() stores := c.cluster.GetStores() var infos *statistics.StoreHotPeersInfos @@ -501,56 +518,41 @@ func (c *coordinator) getHotRegionsByType(typ statistics.RWType) *statistics.Sto default: } // update params `IsLearner` and `LastUpdateTime` - for _, stores := range []statistics.StoreHotPeersStat{infos.AsLeader, infos.AsPeer} { - for _, store := range stores { - for _, hotPeer := range store.Stats { - region := c.cluster.GetRegion(hotPeer.RegionID) - hotPeer.UpdateHotPeerStatShow(region) + s := []statistics.StoreHotPeersStat{infos.AsLeader, infos.AsPeer} + for i, stores := range s { + for j, store := range stores { + for k := range store.Stats { + h := &s[i][j].Stats[k] + region := c.cluster.GetRegion(h.RegionID) + if region != nil { + h.IsLearner = core.IsLearner(region.GetPeer(h.StoreID)) + } + switch typ { + case statistics.Write: + if region != nil { + h.LastUpdateTime = time.Unix(int64(region.GetInterval().GetEndTimestamp()), 0) + } + case statistics.Read: + store := c.cluster.GetStore(h.StoreID) + if store != nil { + ts := store.GetMeta().GetLastHeartbeat() + h.LastUpdateTime = time.Unix(ts/1e9, ts%1e9) + } + default: + } } } } return infos } -func (c *coordinator) getSchedulers() []string { - c.RLock() - defer c.RUnlock() - names := make([]string, 0, len(c.schedulers)) - for name := range c.schedulers { - names = append(names, name) - } - return names +// GetWaitGroup returns the wait group. Only for test purpose. +func (c *Coordinator) GetWaitGroup() *sync.WaitGroup { + return &c.wg } -func (c *coordinator) getSchedulerHandlers() map[string]http.Handler { - c.RLock() - defer c.RUnlock() - handlers := make(map[string]http.Handler, len(c.schedulers)) - for name, scheduler := range c.schedulers { - handlers[name] = scheduler.Scheduler - } - return handlers -} - -func (c *coordinator) collectSchedulerMetrics() { - c.RLock() - defer c.RUnlock() - for _, s := range c.schedulers { - var allowScheduler float64 - // If the scheduler is not allowed to schedule, it will disappear in Grafana panel. - // See issue #1341. - if !s.IsPaused() && !s.cluster.GetUnsafeRecoveryController().IsRunning() { - allowScheduler = 1 - } - schedulerStatusGauge.WithLabelValues(s.GetName(), "allow").Set(allowScheduler) - } -} - -func (c *coordinator) resetSchedulerMetrics() { - schedulerStatusGauge.Reset() -} - -func (c *coordinator) collectHotSpotMetrics() { +// CollectHotSpotMetrics collects hot spot metrics. +func (c *Coordinator) CollectHotSpotMetrics() { stores := c.cluster.GetStores() // Collects hot write region metrics. collectHotMetrics(c.cluster, stores, statistics.Write) @@ -558,7 +560,7 @@ func (c *coordinator) collectHotSpotMetrics() { collectHotMetrics(c.cluster, stores, statistics.Read) } -func collectHotMetrics(cluster *RaftCluster, stores []*core.StoreInfo, typ statistics.RWType) { +func collectHotMetrics(cluster sche.ClusterInformer, stores []*core.StoreInfo, typ statistics.RWType) { var ( kind string regionStats map[uint64][]*statistics.HotPeerStat @@ -607,219 +609,30 @@ func collectHotMetrics(cluster *RaftCluster, stores []*core.StoreInfo, typ stati if !hasHotLeader && !hasHotPeer { statistics.ForeachRegionStats(func(rwTy statistics.RWType, dim int, _ statistics.RegionStatKind) { - hotPendingSum.DeleteLabelValues(storeLabel, rwTy.String(), statistics.DimToString(dim)) + schedulers.HotPendingSum.DeleteLabelValues(storeLabel, rwTy.String(), statistics.DimToString(dim)) }) } } } -func (c *coordinator) resetHotSpotMetrics() { +// ResetHotSpotMetrics resets hot spot metrics. +func (c *Coordinator) ResetHotSpotMetrics() { hotSpotStatusGauge.Reset() - hotPendingSum.Reset() + schedulers.HotPendingSum.Reset() } -func (c *coordinator) shouldRun() bool { +// ShouldRun returns true if the coordinator should run. +func (c *Coordinator) ShouldRun() bool { return c.prepareChecker.check(c.cluster.GetBasicCluster()) } -func (c *coordinator) addScheduler(scheduler schedule.Scheduler, args ...string) error { - c.Lock() - defer c.Unlock() - - if _, ok := c.schedulers[scheduler.GetName()]; ok { - return errs.ErrSchedulerExisted.FastGenByArgs() - } - - s := newScheduleController(c, scheduler) - if err := s.Prepare(c.cluster); err != nil { - return err - } - - c.wg.Add(1) - go c.runScheduler(s) - c.schedulers[s.GetName()] = s - c.cluster.opt.AddSchedulerCfg(s.GetType(), args) - return nil -} - -func (c *coordinator) removeScheduler(name string) error { - c.Lock() - defer c.Unlock() - if c.cluster == nil { - return errs.ErrNotBootstrapped.FastGenByArgs() - } - s, ok := c.schedulers[name] - if !ok { - return errs.ErrSchedulerNotFound.FastGenByArgs() - } - - opt := c.cluster.opt - if err := c.removeOptScheduler(opt, name); err != nil { - log.Error("can not remove scheduler", zap.String("scheduler-name", name), errs.ZapError(err)) - return err - } - - if err := opt.Persist(c.cluster.storage); err != nil { - log.Error("the option can not persist scheduler config", errs.ZapError(err)) - return err - } - - if err := c.cluster.storage.RemoveScheduleConfig(name); err != nil { - log.Error("can not remove the scheduler config", errs.ZapError(err)) - return err - } - - s.Stop() - schedulerStatusGauge.DeleteLabelValues(name, "allow") - delete(c.schedulers, name) - - return nil -} - -func (c *coordinator) removeOptScheduler(o *config.PersistOptions, name string) error { - v := o.GetScheduleConfig().Clone() - for i, schedulerCfg := range v.Schedulers { - // To create a temporary scheduler is just used to get scheduler's name - decoder := schedule.ConfigSliceDecoder(schedulerCfg.Type, schedulerCfg.Args) - tmp, err := schedule.CreateScheduler(schedulerCfg.Type, schedule.NewOperatorController(c.ctx, nil, nil), storage.NewStorageWithMemoryBackend(), decoder) - if err != nil { - return err - } - if tmp.GetName() == name { - if config.IsDefaultScheduler(tmp.GetType()) { - schedulerCfg.Disable = true - v.Schedulers[i] = schedulerCfg - } else { - v.Schedulers = append(v.Schedulers[:i], v.Schedulers[i+1:]...) - } - o.SetScheduleConfig(v) - return nil - } - } - return nil -} - -func (c *coordinator) pauseOrResumeScheduler(name string, t int64) error { - c.Lock() - defer c.Unlock() - if c.cluster == nil { - return errs.ErrNotBootstrapped.FastGenByArgs() - } - var s []*scheduleController - if name != "all" { - sc, ok := c.schedulers[name] - if !ok { - return errs.ErrSchedulerNotFound.FastGenByArgs() - } - s = append(s, sc) - } else { - for _, sc := range c.schedulers { - s = append(s, sc) - } - } - var err error - for _, sc := range s { - var delayAt, delayUntil int64 - if t > 0 { - delayAt = time.Now().Unix() - delayUntil = delayAt + t - } - atomic.StoreInt64(&sc.delayAt, delayAt) - atomic.StoreInt64(&sc.delayUntil, delayUntil) - } - return err -} - -// isSchedulerAllowed returns whether a scheduler is allowed to schedule, a scheduler is not allowed to schedule if it is paused or blocked by unsafe recovery. -func (c *coordinator) isSchedulerAllowed(name string) (bool, error) { - c.RLock() - defer c.RUnlock() - if c.cluster == nil { - return false, errs.ErrNotBootstrapped.FastGenByArgs() - } - s, ok := c.schedulers[name] - if !ok { - return false, errs.ErrSchedulerNotFound.FastGenByArgs() - } - return s.AllowSchedule(false), nil -} - -func (c *coordinator) isSchedulerPaused(name string) (bool, error) { - c.RLock() - defer c.RUnlock() - if c.cluster == nil { - return false, errs.ErrNotBootstrapped.FastGenByArgs() - } - s, ok := c.schedulers[name] - if !ok { - return false, errs.ErrSchedulerNotFound.FastGenByArgs() - } - return s.IsPaused(), nil -} - -func (c *coordinator) isSchedulerDisabled(name string) (bool, error) { - c.RLock() - defer c.RUnlock() - if c.cluster == nil { - return false, errs.ErrNotBootstrapped.FastGenByArgs() - } - s, ok := c.schedulers[name] - if !ok { - return false, errs.ErrSchedulerNotFound.FastGenByArgs() - } - t := s.GetType() - scheduleConfig := c.cluster.GetScheduleConfig() - for _, s := range scheduleConfig.Schedulers { - if t == s.Type { - return s.Disable, nil - } - } - return false, nil -} - -func (c *coordinator) isSchedulerExisted(name string) (bool, error) { - c.RLock() - defer c.RUnlock() - if c.cluster == nil { - return false, errs.ErrNotBootstrapped.FastGenByArgs() - } - _, ok := c.schedulers[name] - if !ok { - return false, errs.ErrSchedulerNotFound.FastGenByArgs() - } - return true, nil -} - -func (c *coordinator) runScheduler(s *scheduleController) { - defer logutil.LogPanic() - defer c.wg.Done() - defer s.Cleanup(c.cluster) - - timer := time.NewTimer(s.GetInterval()) - defer timer.Stop() - for { - select { - case <-timer.C: - timer.Reset(s.GetInterval()) - diagnosable := s.diagnosticRecorder.isAllowed() - if !s.AllowSchedule(diagnosable) { - continue - } - if op := s.Schedule(diagnosable); len(op) > 0 { - added := c.opController.AddWaitingOperator(op...) - log.Debug("add operator", zap.Int("added", added), zap.Int("total", len(op)), zap.String("scheduler", s.GetName())) - } - - case <-s.Ctx().Done(): - log.Info("scheduler has been stopped", - zap.String("scheduler-name", s.GetName()), - errs.ZapError(s.Ctx().Err())) - return - } - } +// GetSchedulersController returns the schedulers controller. +func (c *Coordinator) GetSchedulersController() *schedulers.Controller { + return c.schedulers } -func (c *coordinator) pauseOrResumeChecker(name string, t int64) error { +// PauseOrResumeChecker pauses or resumes a checker by name. +func (c *Coordinator) PauseOrResumeChecker(name string, t int64) error { c.Lock() defer c.Unlock() if c.cluster == nil { @@ -833,7 +646,8 @@ func (c *coordinator) pauseOrResumeChecker(name string, t int64) error { return nil } -func (c *coordinator) isCheckerPaused(name string) (bool, error) { +// IsCheckerPaused returns whether a checker is paused. +func (c *Coordinator) IsCheckerPaused(name string) (bool, error) { c.RLock() defer c.RUnlock() if c.cluster == nil { @@ -846,157 +660,57 @@ func (c *coordinator) isCheckerPaused(name string) (bool, error) { return p.IsPaused(), nil } -func (c *coordinator) GetDiagnosticResult(name string) (*DiagnosticResult, error) { - return c.diagnosticManager.getDiagnosticResult(name) +// GetRegionScatterer returns the region scatterer. +func (c *Coordinator) GetRegionScatterer() *scatter.RegionScatterer { + return c.regionScatterer } -// scheduleController is used to manage a scheduler to schedule. -type scheduleController struct { - schedule.Scheduler - cluster *RaftCluster - opController *schedule.OperatorController - nextInterval time.Duration - ctx context.Context - cancel context.CancelFunc - delayAt int64 - delayUntil int64 - diagnosticRecorder *diagnosticRecorder +// GetRegionSplitter returns the region splitter. +func (c *Coordinator) GetRegionSplitter() *splitter.RegionSplitter { + return c.regionSplitter } -// newScheduleController creates a new scheduleController. -func newScheduleController(c *coordinator, s schedule.Scheduler) *scheduleController { - ctx, cancel := context.WithCancel(c.ctx) - return &scheduleController{ - Scheduler: s, - cluster: c.cluster, - opController: c.opController, - nextInterval: s.GetMinInterval(), - ctx: ctx, - cancel: cancel, - diagnosticRecorder: c.diagnosticManager.getRecorder(s.GetName()), - } -} - -func (s *scheduleController) Ctx() context.Context { - return s.ctx -} - -func (s *scheduleController) Stop() { - s.cancel() -} - -func (s *scheduleController) Schedule(diagnosable bool) []*operator.Operator { - for i := 0; i < maxScheduleRetries; i++ { - // no need to retry if schedule should stop to speed exit - select { - case <-s.ctx.Done(): - return nil - default: - } - cacheCluster := newCacheCluster(s.cluster) - // we need only process diagnostic once in the retry loop - diagnosable = diagnosable && i == 0 - ops, plans := s.Scheduler.Schedule(cacheCluster, diagnosable) - if diagnosable { - s.diagnosticRecorder.setResultFromPlans(ops, plans) - } - if len(ops) > 0 { - // If we have schedule, reset interval to the minimal interval. - s.nextInterval = s.Scheduler.GetMinInterval() - return ops - } - } - s.nextInterval = s.Scheduler.GetNextInterval(s.nextInterval) - return nil +// GetOperatorController returns the operator controller. +func (c *Coordinator) GetOperatorController() *operator.Controller { + return c.opController } -func (s *scheduleController) DiagnoseDryRun() ([]*operator.Operator, []plan.Plan) { - cacheCluster := newCacheCluster(s.cluster) - return s.Scheduler.Schedule(cacheCluster, true) +// GetCheckerController returns the checker controller. +func (c *Coordinator) GetCheckerController() *checker.Controller { + return c.checkers } -// GetInterval returns the interval of scheduling for a scheduler. -func (s *scheduleController) GetInterval() time.Duration { - return s.nextInterval +// GetMergeChecker returns the merge checker. +func (c *Coordinator) GetMergeChecker() *checker.MergeChecker { + return c.checkers.GetMergeChecker() } -// AllowSchedule returns if a scheduler is allowed to schedule. -func (s *scheduleController) AllowSchedule(diagnosable bool) bool { - if !s.Scheduler.IsScheduleAllowed(s.cluster) { - if diagnosable { - s.diagnosticRecorder.setResultFromStatus(pending) - } - return false - } - if s.IsPaused() || s.cluster.GetUnsafeRecoveryController().IsRunning() { - if diagnosable { - s.diagnosticRecorder.setResultFromStatus(paused) - } - return false - } - return true +// GetRuleChecker returns the rule checker. +func (c *Coordinator) GetRuleChecker() *checker.RuleChecker { + return c.checkers.GetRuleChecker() } -// isPaused returns if a scheduler is paused. -func (s *scheduleController) IsPaused() bool { - delayUntil := atomic.LoadInt64(&s.delayUntil) - return time.Now().Unix() < delayUntil +// GetPrepareChecker returns the prepare checker. +func (c *Coordinator) GetPrepareChecker() *prepareChecker { + return c.prepareChecker } -// GetPausedSchedulerDelayAt returns paused timestamp of a paused scheduler -func (s *scheduleController) GetDelayAt() int64 { - if s.IsPaused() { - return atomic.LoadInt64(&s.delayAt) - } - return 0 +// GetHeartbeatStreams returns the heartbeat streams. Only for test purpose. +func (c *Coordinator) GetHeartbeatStreams() *hbstream.HeartbeatStreams { + return c.hbStreams } -// GetPausedSchedulerDelayUntil returns resume timestamp of a paused scheduler -func (s *scheduleController) GetDelayUntil() int64 { - if s.IsPaused() { - return atomic.LoadInt64(&s.delayUntil) - } - return 0 +// GetCluster returns the cluster. Only for test purpose. +func (c *Coordinator) GetCluster() sche.ClusterInformer { + return c.cluster } -func (c *coordinator) getPausedSchedulerDelayAt(name string) (int64, error) { - c.RLock() - defer c.RUnlock() - if c.cluster == nil { - return -1, errs.ErrNotBootstrapped.FastGenByArgs() - } - s, ok := c.schedulers[name] - if !ok { - return -1, errs.ErrSchedulerNotFound.FastGenByArgs() - } - return s.GetDelayAt(), nil +// GetDiagnosticResult returns the diagnostic result. +func (c *Coordinator) GetDiagnosticResult(name string) (*schedulers.DiagnosticResult, error) { + return c.diagnosticManager.GetDiagnosticResult(name) } -func (c *coordinator) getPausedSchedulerDelayUntil(name string) (int64, error) { - c.RLock() - defer c.RUnlock() - if c.cluster == nil { - return -1, errs.ErrNotBootstrapped.FastGenByArgs() - } - s, ok := c.schedulers[name] - if !ok { - return -1, errs.ErrSchedulerNotFound.FastGenByArgs() - } - return s.GetDelayUntil(), nil -} - -// CheckTransferWitnessLeader determines if transfer leader is required, then sends to the scheduler if needed -func (c *coordinator) CheckTransferWitnessLeader(region *core.RegionInfo) { - if core.NeedTransferWitnessLeader(region) { - c.RLock() - s, ok := c.schedulers[schedulers.TransferWitnessLeaderName] - c.RUnlock() - if ok { - select { - case schedulers.RecvRegionInfo(s.Scheduler) <- region: - default: - log.Warn("drop transfer witness leader due to recv region channel full", zap.Uint64("region-id", region.GetID())) - } - } - } +// RecordOpStepWithTTL records OpStep with TTL +func (c *Coordinator) RecordOpStepWithTTL(regionID uint64) { + c.GetRuleChecker().RecordRegionPromoteToNonWitness(regionID) } diff --git a/pkg/schedule/core/cluster_informer.go b/pkg/schedule/core/cluster_informer.go new file mode 100644 index 00000000000..388d9dcf3cd --- /dev/null +++ b/pkg/schedule/core/cluster_informer.go @@ -0,0 +1,82 @@ +// Copyright 2017 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package core + +import ( + "github.com/tikv/pd/pkg/core" + sc "github.com/tikv/pd/pkg/schedule/config" + "github.com/tikv/pd/pkg/schedule/labeler" + "github.com/tikv/pd/pkg/schedule/placement" + "github.com/tikv/pd/pkg/statistics" + "github.com/tikv/pd/pkg/statistics/buckets" + "github.com/tikv/pd/pkg/storage" + "github.com/tikv/pd/server/config" +) + +// ClusterInformer provides the necessary information of a cluster. +type ClusterInformer interface { + SchedulerCluster + CheckerCluster + ScatterCluster + + GetStorage() storage.Storage + UpdateRegionsLabelLevelStats(regions []*core.RegionInfo) + GetPersistOptions() *config.PersistOptions +} + +// SchedulerCluster is an aggregate interface that wraps multiple interfaces +type SchedulerCluster interface { + SharedCluster + + statistics.StoreStatInformer + buckets.BucketStatInformer + + GetSchedulerConfig() sc.SchedulerConfigProvider + GetRegionLabeler() *labeler.RegionLabeler + GetStoreConfig() sc.StoreConfigProvider +} + +// CheckerCluster is an aggregate interface that wraps multiple interfaces +type CheckerCluster interface { + SharedCluster + + GetCheckerConfig() sc.CheckerConfigProvider + GetStoreConfig() sc.StoreConfigProvider +} + +// ScatterCluster is an aggregate interface that wraps multiple interfaces +type ScatterCluster interface { + SharedCluster + + AddSuspectRegions(ids ...uint64) +} + +// SharedCluster is an aggregate interface that wraps multiple interfaces +type SharedCluster interface { + BasicCluster + statistics.RegionStatInformer + + GetBasicCluster() *core.BasicCluster + GetSharedConfig() sc.SharedConfigProvider + GetRuleManager() *placement.RuleManager + AllocID() (uint64, error) +} + +// BasicCluster is an aggregate interface that wraps multiple interfaces +type BasicCluster interface { + core.StoreSetInformer + core.StoreSetController + core.RegionSetInformer +} diff --git a/pkg/schedule/diagnostic/diagnostic_manager.go b/pkg/schedule/diagnostic/diagnostic_manager.go new file mode 100644 index 00000000000..5a56e849c3b --- /dev/null +++ b/pkg/schedule/diagnostic/diagnostic_manager.go @@ -0,0 +1,71 @@ +// Copyright 2022 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package diagnostic + +import ( + "time" + + "github.com/tikv/pd/pkg/errs" + "github.com/tikv/pd/pkg/schedule/config" + "github.com/tikv/pd/pkg/schedule/schedulers" +) + +// Manager is used to manage the diagnostic result of schedulers for now. +type Manager struct { + config config.SchedulerConfigProvider + schedulerController *schedulers.Controller +} + +// NewManager creates a new Manager. +func NewManager(schedulerController *schedulers.Controller, config config.SchedulerConfigProvider) *Manager { + return &Manager{ + config: config, + schedulerController: schedulerController, + } +} + +// GetDiagnosticResult gets the diagnostic result of the scheduler. +func (d *Manager) GetDiagnosticResult(name string) (*schedulers.DiagnosticResult, error) { + if !d.config.IsDiagnosticAllowed() { + return nil, errs.ErrDiagnosticDisabled + } + + scheduler := d.schedulerController.GetScheduler(name) + if scheduler == nil { + ts := uint64(time.Now().Unix()) + res := &schedulers.DiagnosticResult{Name: name, Timestamp: ts, Status: schedulers.Disabled} + return res, nil + } + isDisabled := d.config.IsSchedulerDisabled(scheduler.Scheduler.GetType()) + if isDisabled { + ts := uint64(time.Now().Unix()) + res := &schedulers.DiagnosticResult{Name: name, Timestamp: ts, Status: schedulers.Disabled} + return res, nil + } + + recorder := d.getSchedulerRecorder(name) + if recorder == nil { + return nil, errs.ErrSchedulerUndiagnosable.FastGenByArgs(name) + } + result := recorder.GetLastResult() + if result == nil { + return nil, errs.ErrNoDiagnosticResult.FastGenByArgs(name) + } + return result, nil +} + +func (d *Manager) getSchedulerRecorder(name string) *schedulers.DiagnosticRecorder { + return d.schedulerController.GetScheduler(name).GetDiagnosticRecorder() +} diff --git a/pkg/schedule/filter/candidates.go b/pkg/schedule/filter/candidates.go index 227d9c97d82..35ca4a2e284 100644 --- a/pkg/schedule/filter/candidates.go +++ b/pkg/schedule/filter/candidates.go @@ -17,6 +17,7 @@ package filter import ( "math/rand" "sort" + "time" "github.com/tikv/pd/pkg/core" "github.com/tikv/pd/pkg/schedule/config" @@ -26,22 +27,23 @@ import ( // StoreCandidates wraps store list and provide utilities to select source or // target store to schedule. type StoreCandidates struct { + r *rand.Rand Stores []*core.StoreInfo } // NewCandidates creates StoreCandidates with store list. func NewCandidates(stores []*core.StoreInfo) *StoreCandidates { - return &StoreCandidates{Stores: stores} + return &StoreCandidates{r: rand.New(rand.NewSource(time.Now().UnixNano())), Stores: stores} } // FilterSource keeps stores that can pass all source filters. -func (c *StoreCandidates) FilterSource(conf config.Config, collector *plan.Collector, counter *Counter, filters ...Filter) *StoreCandidates { +func (c *StoreCandidates) FilterSource(conf config.SharedConfigProvider, collector *plan.Collector, counter *Counter, filters ...Filter) *StoreCandidates { c.Stores = SelectSourceStores(c.Stores, filters, conf, collector, counter) return c } // FilterTarget keeps stores that can pass all target filters. -func (c *StoreCandidates) FilterTarget(conf config.Config, collector *plan.Collector, counter *Counter, filters ...Filter) *StoreCandidates { +func (c *StoreCandidates) FilterTarget(conf config.SharedConfigProvider, collector *plan.Collector, counter *Counter, filters ...Filter) *StoreCandidates { c.Stores = SelectTargetStores(c.Stores, filters, conf, collector, counter) return c } @@ -54,7 +56,7 @@ func (c *StoreCandidates) Sort(less StoreComparer) *StoreCandidates { // Shuffle reorders all candidates randomly. func (c *StoreCandidates) Shuffle() *StoreCandidates { - rand.Shuffle(len(c.Stores), func(i, j int) { c.Stores[i], c.Stores[j] = c.Stores[j], c.Stores[i] }) + c.r.Shuffle(len(c.Stores), func(i, j int) { c.Stores[i], c.Stores[j] = c.Stores[j], c.Stores[i] }) return c } @@ -108,7 +110,7 @@ func (c *StoreCandidates) RandomPick() *core.StoreInfo { if len(c.Stores) == 0 { return nil } - return c.Stores[rand.Intn(len(c.Stores))] + return c.Stores[c.r.Intn(len(c.Stores))] } // PickAll return all stores in candidate list. diff --git a/pkg/schedule/filter/candidates_test.go b/pkg/schedule/filter/candidates_test.go index be30a874468..13e8ed661cc 100644 --- a/pkg/schedule/filter/candidates_test.go +++ b/pkg/schedule/filter/candidates_test.go @@ -50,7 +50,7 @@ type idFilter func(uint64) bool func (f idFilter) Scope() string { return "idFilter" } func (f idFilter) Type() filterType { return filterType(0) } -func (f idFilter) Source(conf config.Config, store *core.StoreInfo) *plan.Status { +func (f idFilter) Source(conf config.SharedConfigProvider, store *core.StoreInfo) *plan.Status { if f(store.GetID()) { return statusOK } @@ -58,7 +58,7 @@ func (f idFilter) Source(conf config.Config, store *core.StoreInfo) *plan.Status return statusStoreScoreDisallowed } -func (f idFilter) Target(conf config.Config, store *core.StoreInfo) *plan.Status { +func (f idFilter) Target(conf config.SharedConfigProvider, store *core.StoreInfo) *plan.Status { if f(store.GetID()) { return statusOK } diff --git a/pkg/schedule/filter/comparer.go b/pkg/schedule/filter/comparer.go index eb319105f87..58d3032f36d 100644 --- a/pkg/schedule/filter/comparer.go +++ b/pkg/schedule/filter/comparer.go @@ -25,7 +25,7 @@ type StoreComparer func(a, b *core.StoreInfo) int // RegionScoreComparer creates a StoreComparer to sort store by region // score. -func RegionScoreComparer(conf config.Config) StoreComparer { +func RegionScoreComparer(conf config.SharedConfigProvider) StoreComparer { return func(a, b *core.StoreInfo) int { sa := a.RegionScore(conf.GetRegionScoreFormulaVersion(), conf.GetHighSpaceRatio(), conf.GetLowSpaceRatio(), 0) sb := b.RegionScore(conf.GetRegionScoreFormulaVersion(), conf.GetHighSpaceRatio(), conf.GetLowSpaceRatio(), 0) diff --git a/pkg/schedule/filter/filters.go b/pkg/schedule/filter/filters.go index 2bda9bee9c0..0d188e69180 100644 --- a/pkg/schedule/filter/filters.go +++ b/pkg/schedule/filter/filters.go @@ -20,6 +20,7 @@ import ( "github.com/pingcap/kvproto/pkg/metapb" "github.com/pingcap/log" "github.com/tikv/pd/pkg/core" + "github.com/tikv/pd/pkg/core/constant" "github.com/tikv/pd/pkg/core/storelimit" "github.com/tikv/pd/pkg/schedule/config" "github.com/tikv/pd/pkg/schedule/placement" @@ -30,7 +31,7 @@ import ( ) // SelectSourceStores selects stores that be selected as source store from the list. -func SelectSourceStores(stores []*core.StoreInfo, filters []Filter, conf config.Config, collector *plan.Collector, +func SelectSourceStores(stores []*core.StoreInfo, filters []Filter, conf config.SharedConfigProvider, collector *plan.Collector, counter *Counter) []*core.StoreInfo { return filterStoresBy(stores, func(s *core.StoreInfo) bool { return slice.AllOf(filters, func(i int) bool { @@ -54,7 +55,7 @@ func SelectSourceStores(stores []*core.StoreInfo, filters []Filter, conf config. } // SelectUnavailableTargetStores selects unavailable stores that can't be selected as target store from the list. -func SelectUnavailableTargetStores(stores []*core.StoreInfo, filters []Filter, conf config.Config, +func SelectUnavailableTargetStores(stores []*core.StoreInfo, filters []Filter, conf config.SharedConfigProvider, collector *plan.Collector, counter *Counter) []*core.StoreInfo { return filterStoresBy(stores, func(s *core.StoreInfo) bool { targetID := strconv.FormatUint(s.GetID(), 10) @@ -84,7 +85,7 @@ func SelectUnavailableTargetStores(stores []*core.StoreInfo, filters []Filter, c } // SelectTargetStores selects stores that be selected as target store from the list. -func SelectTargetStores(stores []*core.StoreInfo, filters []Filter, conf config.Config, collector *plan.Collector, +func SelectTargetStores(stores []*core.StoreInfo, filters []Filter, conf config.SharedConfigProvider, collector *plan.Collector, counter *Counter) []*core.StoreInfo { if len(filters) == 0 { return stores @@ -131,10 +132,10 @@ type Filter interface { // Scope is used to indicate where the filter will act on. Scope() string Type() filterType - // Return plan.Status to show whether be filtered as source - Source(conf config.Config, store *core.StoreInfo) *plan.Status - // Return plan.Status to show whether be filtered as target - Target(conf config.Config, store *core.StoreInfo) *plan.Status + // Source Return plan.Status to show whether be filtered as source + Source(conf config.SharedConfigProvider, store *core.StoreInfo) *plan.Status + // Target Return plan.Status to show whether be filtered as target + Target(conf config.SharedConfigProvider, store *core.StoreInfo) *plan.Status } // comparingFilter is an interface to filter target store by comparing source and target stores @@ -145,7 +146,7 @@ type comparingFilter interface { } // Target checks if store can pass all Filters as target store. -func Target(conf config.Config, store *core.StoreInfo, filters []Filter) bool { +func Target(conf config.SharedConfigProvider, store *core.StoreInfo, filters []Filter) bool { storeID := strconv.FormatUint(store.GetID(), 10) for _, filter := range filters { status := filter.Target(conf, store) @@ -188,14 +189,14 @@ func (f *excludedFilter) Type() filterType { return excluded } -func (f *excludedFilter) Source(conf config.Config, store *core.StoreInfo) *plan.Status { +func (f *excludedFilter) Source(conf config.SharedConfigProvider, store *core.StoreInfo) *plan.Status { if _, ok := f.sources[store.GetID()]; ok { return statusStoreAlreadyHasPeer } return statusOK } -func (f *excludedFilter) Target(conf config.Config, store *core.StoreInfo) *plan.Status { +func (f *excludedFilter) Target(conf config.SharedConfigProvider, store *core.StoreInfo) *plan.Status { if _, ok := f.targets[store.GetID()]; ok { return statusStoreAlreadyHasPeer } @@ -218,11 +219,11 @@ func (f *storageThresholdFilter) Type() filterType { return storageThreshold } -func (f *storageThresholdFilter) Source(conf config.Config, store *core.StoreInfo) *plan.Status { +func (f *storageThresholdFilter) Source(conf config.SharedConfigProvider, store *core.StoreInfo) *plan.Status { return statusOK } -func (f *storageThresholdFilter) Target(conf config.Config, store *core.StoreInfo) *plan.Status { +func (f *storageThresholdFilter) Target(conf config.SharedConfigProvider, store *core.StoreInfo) *plan.Status { if !store.IsLowSpace(conf.GetLowSpaceRatio()) { return statusOK } @@ -286,11 +287,11 @@ func (f *distinctScoreFilter) Type() filterType { return distinctScore } -func (f *distinctScoreFilter) Source(_ config.Config, _ *core.StoreInfo) *plan.Status { +func (f *distinctScoreFilter) Source(_ config.SharedConfigProvider, _ *core.StoreInfo) *plan.Status { return statusOK } -func (f *distinctScoreFilter) Target(_ config.Config, store *core.StoreInfo) *plan.Status { +func (f *distinctScoreFilter) Target(_ config.SharedConfigProvider, store *core.StoreInfo) *plan.Status { score := core.DistinctScore(f.labels, f.stores, store) switch f.policy { case locationSafeguard: @@ -325,6 +326,12 @@ type StoreStateFilter struct { AllowFastFailover bool // Set true if allows temporary states. AllowTemporaryStates bool + // Set the priority level of the filter, it should be same with the operator level. + // The priority level can be higher than the operator level in checker, + // the operator controller should check it again by using the actual operator level. + // If it checks failed, the operator will be put back to the waiting queue util the limit is available. + // But the scheduler should keep the same with the operator level. + OperatorLevel constant.PriorityLevel // Reason is used to distinguish the reason of store state filter Reason filterType } @@ -341,9 +348,9 @@ func (f *StoreStateFilter) Type() filterType { // conditionFunc defines condition to determine a store should be selected. // It should consider if the filter allows temporary states. -type conditionFunc func(config.Config, *core.StoreInfo) *plan.Status +type conditionFunc func(config.SharedConfigProvider, *core.StoreInfo) *plan.Status -func (f *StoreStateFilter) isRemoved(_ config.Config, store *core.StoreInfo) *plan.Status { +func (f *StoreStateFilter) isRemoved(_ config.SharedConfigProvider, store *core.StoreInfo) *plan.Status { if store.IsRemoved() { f.Reason = storeStateTombstone return statusStoreRemoved @@ -352,7 +359,7 @@ func (f *StoreStateFilter) isRemoved(_ config.Config, store *core.StoreInfo) *pl return statusOK } -func (f *StoreStateFilter) isDown(conf config.Config, store *core.StoreInfo) *plan.Status { +func (f *StoreStateFilter) isDown(conf config.SharedConfigProvider, store *core.StoreInfo) *plan.Status { if store.DownTime() > conf.GetMaxStoreDownTime() { f.Reason = storeStateDown return statusStoreDown @@ -362,7 +369,7 @@ func (f *StoreStateFilter) isDown(conf config.Config, store *core.StoreInfo) *pl return statusOK } -func (f *StoreStateFilter) isRemoving(_ config.Config, store *core.StoreInfo) *plan.Status { +func (f *StoreStateFilter) isRemoving(_ config.SharedConfigProvider, store *core.StoreInfo) *plan.Status { if store.IsRemoving() { f.Reason = storeStateOffline return statusStoresRemoving @@ -371,7 +378,7 @@ func (f *StoreStateFilter) isRemoving(_ config.Config, store *core.StoreInfo) *p return statusOK } -func (f *StoreStateFilter) pauseLeaderTransfer(_ config.Config, store *core.StoreInfo) *plan.Status { +func (f *StoreStateFilter) pauseLeaderTransfer(_ config.SharedConfigProvider, store *core.StoreInfo) *plan.Status { if !store.AllowLeaderTransfer() { f.Reason = storeStatePauseLeader return statusStoreRejectLeader @@ -380,7 +387,7 @@ func (f *StoreStateFilter) pauseLeaderTransfer(_ config.Config, store *core.Stor return statusOK } -func (f *StoreStateFilter) slowStoreEvicted(conf config.Config, store *core.StoreInfo) *plan.Status { +func (f *StoreStateFilter) slowStoreEvicted(conf config.SharedConfigProvider, store *core.StoreInfo) *plan.Status { if store.EvictedAsSlowStore() { f.Reason = storeStateSlow return statusStoreRejectLeader @@ -389,7 +396,7 @@ func (f *StoreStateFilter) slowStoreEvicted(conf config.Config, store *core.Stor return statusOK } -func (f *StoreStateFilter) slowTrendEvicted(_ config.Config, store *core.StoreInfo) *plan.Status { +func (f *StoreStateFilter) slowTrendEvicted(_ config.SharedConfigProvider, store *core.StoreInfo) *plan.Status { if store.IsEvictedAsSlowTrend() { f.Reason = storeStateSlowTrend return statusStoreRejectLeader @@ -398,7 +405,7 @@ func (f *StoreStateFilter) slowTrendEvicted(_ config.Config, store *core.StoreIn return statusOK } -func (f *StoreStateFilter) isDisconnected(_ config.Config, store *core.StoreInfo) *plan.Status { +func (f *StoreStateFilter) isDisconnected(_ config.SharedConfigProvider, store *core.StoreInfo) *plan.Status { if !f.AllowTemporaryStates && store.IsDisconnected() { f.Reason = storeStateDisconnected return statusStoreDisconnected @@ -407,7 +414,7 @@ func (f *StoreStateFilter) isDisconnected(_ config.Config, store *core.StoreInfo return statusOK } -func (f *StoreStateFilter) isBusy(_ config.Config, store *core.StoreInfo) *plan.Status { +func (f *StoreStateFilter) isBusy(_ config.SharedConfigProvider, store *core.StoreInfo) *plan.Status { if !f.AllowTemporaryStates && store.IsBusy() { f.Reason = storeStateBusy return statusStoreBusy @@ -416,8 +423,8 @@ func (f *StoreStateFilter) isBusy(_ config.Config, store *core.StoreInfo) *plan. return statusOK } -func (f *StoreStateFilter) exceedRemoveLimit(_ config.Config, store *core.StoreInfo) *plan.Status { - if !f.AllowTemporaryStates && !store.IsAvailable(storelimit.RemovePeer) { +func (f *StoreStateFilter) exceedRemoveLimit(_ config.SharedConfigProvider, store *core.StoreInfo) *plan.Status { + if !f.AllowTemporaryStates && !store.IsAvailable(storelimit.RemovePeer, f.OperatorLevel) { f.Reason = storeStateExceedRemoveLimit return statusStoreRemoveLimit } @@ -425,8 +432,8 @@ func (f *StoreStateFilter) exceedRemoveLimit(_ config.Config, store *core.StoreI return statusOK } -func (f *StoreStateFilter) exceedAddLimit(_ config.Config, store *core.StoreInfo) *plan.Status { - if !f.AllowTemporaryStates && !store.IsAvailable(storelimit.AddPeer) { +func (f *StoreStateFilter) exceedAddLimit(_ config.SharedConfigProvider, store *core.StoreInfo) *plan.Status { + if !f.AllowTemporaryStates && !store.IsAvailable(storelimit.AddPeer, f.OperatorLevel) { f.Reason = storeStateExceedAddLimit return statusStoreAddLimit } @@ -434,7 +441,7 @@ func (f *StoreStateFilter) exceedAddLimit(_ config.Config, store *core.StoreInfo return statusOK } -func (f *StoreStateFilter) tooManySnapshots(conf config.Config, store *core.StoreInfo) *plan.Status { +func (f *StoreStateFilter) tooManySnapshots(conf config.SharedConfigProvider, store *core.StoreInfo) *plan.Status { if !f.AllowTemporaryStates && (uint64(store.GetSendingSnapCount()) > conf.GetMaxSnapshotCount() || uint64(store.GetReceivingSnapCount()) > conf.GetMaxSnapshotCount()) { f.Reason = storeStateTooManySnapshot @@ -444,7 +451,7 @@ func (f *StoreStateFilter) tooManySnapshots(conf config.Config, store *core.Stor return statusOK } -func (f *StoreStateFilter) tooManyPendingPeers(conf config.Config, store *core.StoreInfo) *plan.Status { +func (f *StoreStateFilter) tooManyPendingPeers(conf config.SharedConfigProvider, store *core.StoreInfo) *plan.Status { if !f.AllowTemporaryStates && conf.GetMaxPendingPeerCount() > 0 && store.GetPendingPeerCount() > int(conf.GetMaxPendingPeerCount()) { @@ -455,7 +462,7 @@ func (f *StoreStateFilter) tooManyPendingPeers(conf config.Config, store *core.S return statusOK } -func (f *StoreStateFilter) hasRejectLeaderProperty(conf config.Config, store *core.StoreInfo) *plan.Status { +func (f *StoreStateFilter) hasRejectLeaderProperty(conf config.SharedConfigProvider, store *core.StoreInfo) *plan.Status { if conf.CheckLabelProperty(config.RejectLeader, store.GetLabels()) { f.Reason = storeStateRejectLeader return statusStoreRejectLeader @@ -488,7 +495,7 @@ const ( fastFailoverTarget ) -func (f *StoreStateFilter) anyConditionMatch(typ int, conf config.Config, store *core.StoreInfo) *plan.Status { +func (f *StoreStateFilter) anyConditionMatch(typ int, conf config.SharedConfigProvider, store *core.StoreInfo) *plan.Status { var funcs []conditionFunc switch typ { case leaderSource: @@ -520,7 +527,7 @@ func (f *StoreStateFilter) anyConditionMatch(typ int, conf config.Config, store // Source returns true when the store can be selected as the schedule // source. -func (f *StoreStateFilter) Source(conf config.Config, store *core.StoreInfo) (status *plan.Status) { +func (f *StoreStateFilter) Source(conf config.SharedConfigProvider, store *core.StoreInfo) (status *plan.Status) { if f.TransferLeader { if status = f.anyConditionMatch(leaderSource, conf, store); !status.IsOK() { return @@ -531,12 +538,13 @@ func (f *StoreStateFilter) Source(conf config.Config, store *core.StoreInfo) (st return } } + return statusOK } // Target returns true when the store can be selected as the schedule // target. -func (f *StoreStateFilter) Target(conf config.Config, store *core.StoreInfo) (status *plan.Status) { +func (f *StoreStateFilter) Target(conf config.SharedConfigProvider, store *core.StoreInfo) (status *plan.Status) { if f.TransferLeader { if status = f.anyConditionMatch(leaderTarget, conf, store); !status.IsOK() { return @@ -580,7 +588,7 @@ func (f labelConstraintFilter) Type() filterType { } // Source filters stores when select them as schedule source. -func (f labelConstraintFilter) Source(conf config.Config, store *core.StoreInfo) *plan.Status { +func (f labelConstraintFilter) Source(conf config.SharedConfigProvider, store *core.StoreInfo) *plan.Status { if placement.MatchLabelConstraints(store, f.constraints) { return statusOK } @@ -588,7 +596,7 @@ func (f labelConstraintFilter) Source(conf config.Config, store *core.StoreInfo) } // Target filters stores when select them as schedule target. -func (f labelConstraintFilter) Target(_ config.Config, store *core.StoreInfo) *plan.Status { +func (f labelConstraintFilter) Target(_ config.SharedConfigProvider, store *core.StoreInfo) *plan.Status { if placement.MatchLabelConstraints(store, f.constraints) { return statusOK } @@ -630,7 +638,7 @@ func (f *ruleFitFilter) Type() filterType { return ruleFit } -func (f *ruleFitFilter) Source(_ config.Config, _ *core.StoreInfo) *plan.Status { +func (f *ruleFitFilter) Source(_ config.SharedConfigProvider, _ *core.StoreInfo) *plan.Status { return statusOK } @@ -639,7 +647,7 @@ func (f *ruleFitFilter) Source(_ config.Config, _ *core.StoreInfo) *plan.Status // the replaced store can match the source rule. // RegionA:[1,2,3], move peer1 --> peer2 will not allow, because it's count not match the rule. // but transfer role peer1 --> peer2, it will support. -func (f *ruleFitFilter) Target(options config.Config, store *core.StoreInfo) *plan.Status { +func (f *ruleFitFilter) Target(_ config.SharedConfigProvider, store *core.StoreInfo) *plan.Status { if f.oldFit.Replace(f.srcStore, store) { return statusOK } @@ -683,11 +691,11 @@ func (f *ruleLeaderFitFilter) Type() filterType { return ruleLeader } -func (f *ruleLeaderFitFilter) Source(_ config.Config, _ *core.StoreInfo) *plan.Status { +func (f *ruleLeaderFitFilter) Source(_ config.SharedConfigProvider, _ *core.StoreInfo) *plan.Status { return statusOK } -func (f *ruleLeaderFitFilter) Target(_ config.Config, store *core.StoreInfo) *plan.Status { +func (f *ruleLeaderFitFilter) Target(_ config.SharedConfigProvider, store *core.StoreInfo) *plan.Status { targetStoreID := store.GetID() targetPeer := f.region.GetStorePeer(targetStoreID) if targetPeer == nil && !f.allowMoveLeader { @@ -739,11 +747,11 @@ func (f *ruleWitnessFitFilter) Type() filterType { return ruleFit } -func (f *ruleWitnessFitFilter) Source(_ config.Config, _ *core.StoreInfo) *plan.Status { +func (f *ruleWitnessFitFilter) Source(_ config.SharedConfigProvider, _ *core.StoreInfo) *plan.Status { return statusOK } -func (f *ruleWitnessFitFilter) Target(_ config.Config, store *core.StoreInfo) *plan.Status { +func (f *ruleWitnessFitFilter) Target(_ config.SharedConfigProvider, store *core.StoreInfo) *plan.Status { targetStoreID := store.GetID() targetPeer := f.region.GetStorePeer(targetStoreID) if targetPeer == nil { @@ -761,7 +769,7 @@ func (f *ruleWitnessFitFilter) Target(_ config.Config, store *core.StoreInfo) *p // NewPlacementSafeguard creates a filter that ensures after replace a peer with new // peer, the placement restriction will not become worse. -func NewPlacementSafeguard(scope string, conf config.Config, cluster *core.BasicCluster, ruleManager *placement.RuleManager, +func NewPlacementSafeguard(scope string, conf config.SharedConfigProvider, cluster *core.BasicCluster, ruleManager *placement.RuleManager, region *core.RegionInfo, sourceStore *core.StoreInfo, oldFit *placement.RegionFit) Filter { if conf.IsPlacementRulesEnabled() { return newRuleFitFilter(scope, cluster, ruleManager, region, oldFit, sourceStore.GetID()) @@ -772,7 +780,7 @@ func NewPlacementSafeguard(scope string, conf config.Config, cluster *core.Basic // NewPlacementLeaderSafeguard creates a filter that ensures after transfer a leader with // existed peer, the placement restriction will not become worse. // Note that it only worked when PlacementRules enabled otherwise it will always permit the sourceStore. -func NewPlacementLeaderSafeguard(scope string, conf config.Config, cluster *core.BasicCluster, ruleManager *placement.RuleManager, region *core.RegionInfo, sourceStore *core.StoreInfo, allowMoveLeader bool) Filter { +func NewPlacementLeaderSafeguard(scope string, conf config.SharedConfigProvider, cluster *core.BasicCluster, ruleManager *placement.RuleManager, region *core.RegionInfo, sourceStore *core.StoreInfo, allowMoveLeader bool) Filter { if conf.IsPlacementRulesEnabled() { return newRuleLeaderFitFilter(scope, cluster, ruleManager, region, sourceStore.GetID(), allowMoveLeader) } @@ -782,7 +790,7 @@ func NewPlacementLeaderSafeguard(scope string, conf config.Config, cluster *core // NewPlacementWitnessSafeguard creates a filter that ensures after transfer a witness with // existed peer, the placement restriction will not become worse. // Note that it only worked when PlacementRules enabled otherwise it will always permit the sourceStore. -func NewPlacementWitnessSafeguard(scope string, conf config.Config, cluster *core.BasicCluster, ruleManager *placement.RuleManager, +func NewPlacementWitnessSafeguard(scope string, conf config.SharedConfigProvider, cluster *core.BasicCluster, ruleManager *placement.RuleManager, region *core.RegionInfo, sourceStore *core.StoreInfo, oldFit *placement.RegionFit) Filter { if conf.IsPlacementRulesEnabled() { return newRuleWitnessFitFilter(scope, cluster, ruleManager, region, oldFit, sourceStore.GetID()) @@ -811,14 +819,14 @@ func (f *engineFilter) Type() filterType { return engine } -func (f *engineFilter) Source(_ config.Config, store *core.StoreInfo) *plan.Status { +func (f *engineFilter) Source(_ config.SharedConfigProvider, store *core.StoreInfo) *plan.Status { if f.constraint.MatchStore(store) { return statusOK } return statusStoreNotMatchRule } -func (f *engineFilter) Target(_ config.Config, store *core.StoreInfo) *plan.Status { +func (f *engineFilter) Target(_ config.SharedConfigProvider, store *core.StoreInfo) *plan.Status { if f.constraint.MatchStore(store) { return statusOK } @@ -854,14 +862,14 @@ func (f *specialUseFilter) Type() filterType { return specialUse } -func (f *specialUseFilter) Source(conf config.Config, store *core.StoreInfo) *plan.Status { +func (f *specialUseFilter) Source(conf config.SharedConfigProvider, store *core.StoreInfo) *plan.Status { if store.IsLowSpace(conf.GetLowSpaceRatio()) || !f.constraint.MatchStore(store) { return statusOK } return statusStoreNotMatchRule } -func (f *specialUseFilter) Target(conf config.Config, store *core.StoreInfo) *plan.Status { +func (f *specialUseFilter) Target(conf config.SharedConfigProvider, store *core.StoreInfo) *plan.Status { if !f.constraint.MatchStore(store) { return statusOK } @@ -928,11 +936,11 @@ func (f *isolationFilter) Type() filterType { return isolation } -func (f *isolationFilter) Source(conf config.Config, store *core.StoreInfo) *plan.Status { +func (f *isolationFilter) Source(conf config.SharedConfigProvider, store *core.StoreInfo) *plan.Status { return statusOK } -func (f *isolationFilter) Target(_ config.Config, store *core.StoreInfo) *plan.Status { +func (f *isolationFilter) Target(_ config.SharedConfigProvider, store *core.StoreInfo) *plan.Status { // No isolation constraint to fit if len(f.constraintSet) == 0 { return statusStoreNotMatchIsolation diff --git a/pkg/schedule/filter/filters_test.go b/pkg/schedule/filter/filters_test.go index 75b6b95e819..fa085890694 100644 --- a/pkg/schedule/filter/filters_test.go +++ b/pkg/schedule/filter/filters_test.go @@ -23,6 +23,8 @@ import ( "github.com/pingcap/kvproto/pkg/pdpb" "github.com/stretchr/testify/require" "github.com/tikv/pd/pkg/core" + "github.com/tikv/pd/pkg/core/constant" + "github.com/tikv/pd/pkg/core/storelimit" "github.com/tikv/pd/pkg/mock/mockcluster" "github.com/tikv/pd/pkg/mock/mockconfig" "github.com/tikv/pd/pkg/schedule/placement" @@ -91,7 +93,7 @@ func TestLabelConstraintsFilter(t *testing.T) { } for _, testCase := range testCases { filter := NewLabelConstraintFilter("", []placement.LabelConstraint{{Key: testCase.key, Op: placement.LabelConstraintOp(testCase.op), Values: testCase.values}}) - re.Equal(testCase.res, filter.Source(testCluster.GetOpts(), store).StatusCode) + re.Equal(testCase.res, filter.Source(testCluster.GetSharedConfig(), store).StatusCode) } } @@ -137,15 +139,100 @@ func TestRuleFitFilter(t *testing.T) { } for _, testCase := range testCases { filter := newRuleFitFilter("", testCluster.GetBasicCluster(), testCluster.GetRuleManager(), region, nil, 1) - re.Equal(testCase.sourceRes, filter.Source(testCluster.GetOpts(), testCluster.GetStore(testCase.storeID)).StatusCode) - re.Equal(testCase.targetRes, filter.Target(testCluster.GetOpts(), testCluster.GetStore(testCase.storeID)).StatusCode) + re.Equal(testCase.sourceRes, filter.Source(testCluster.GetSharedConfig(), testCluster.GetStore(testCase.storeID)).StatusCode) + re.Equal(testCase.targetRes, filter.Target(testCluster.GetSharedConfig(), testCluster.GetStore(testCase.storeID)).StatusCode) leaderFilter := newRuleLeaderFitFilter("", testCluster.GetBasicCluster(), testCluster.GetRuleManager(), region, 1, true) - re.Equal(testCase.targetRes, leaderFilter.Target(testCluster.GetOpts(), testCluster.GetStore(testCase.storeID)).StatusCode) + re.Equal(testCase.targetRes, leaderFilter.Target(testCluster.GetSharedConfig(), testCluster.GetStore(testCase.storeID)).StatusCode) } // store-6 is not exist in the peers, so it will not allow transferring leader to store 6. leaderFilter := newRuleLeaderFitFilter("", testCluster.GetBasicCluster(), testCluster.GetRuleManager(), region, 1, false) - re.False(leaderFilter.Target(testCluster.GetOpts(), testCluster.GetStore(6)).IsOK()) + re.False(leaderFilter.Target(testCluster.GetSharedConfig(), testCluster.GetStore(6)).IsOK()) +} + +func TestRuleFitFilterWithPlacementRule(t *testing.T) { + re := require.New(t) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + opt := mockconfig.NewTestOptions() + testCluster := mockcluster.NewCluster(ctx, opt) + testCluster.SetEnablePlacementRules(true) + ruleManager := testCluster.RuleManager + ruleManager.DeleteRule("pd", "default") + err := ruleManager.SetRules([]*placement.Rule{ + { + GroupID: "test", + ID: "r1", + Index: 100, + Role: placement.Leader, + Count: 1, + LabelConstraints: []placement.LabelConstraint{ + {Key: "dc", Op: "in", Values: []string{"dc1"}}, + }, + LocationLabels: []string{"dc", "zone", "host"}, + }, + { + GroupID: "test", + ID: "r2", + Index: 100, + Role: placement.Voter, + Count: 2, + LabelConstraints: []placement.LabelConstraint{ + {Key: "dc", Op: "in", Values: []string{"dc1"}}, + }, + LocationLabels: []string{"dc", "zone", "host"}, + }, + { + GroupID: "test", + ID: "r3", + Index: 100, + Role: placement.Voter, + Count: 2, + LabelConstraints: []placement.LabelConstraint{ + {Key: "dc", Op: "in", Values: []string{"dc2"}}, + }, + LocationLabels: []string{"dc", "zone", "host"}, + }, + }) + re.NoError(err) + stores := []struct { + storeID uint64 + regionCount int + labels map[string]string + }{ + {1, 1, map[string]string{"dc": "dc1", "zone": "z1", "host": "h1"}}, + {2, 1, map[string]string{"dc": "dc1", "zone": "z2", "host": "h2"}}, + {3, 1, map[string]string{"dc": "dc1", "zone": "z3", "host": "h3"}}, + {4, 1, map[string]string{"dc": "dc1", "zone": "z4", "host": "h4"}}, + {5, 1, map[string]string{"dc": "dc2", "zone": "z5", "host": "h5"}}, + {6, 1, map[string]string{"dc": "dc2", "zone": "z6", "host": "h6"}}, + } + // Init cluster + for _, store := range stores { + testCluster.AddLabelsStore(store.storeID, store.regionCount, store.labels) + } + region := core.NewRegionInfo(&metapb.Region{Peers: []*metapb.Peer{ + {StoreId: 1, Id: 1}, + {StoreId: 2, Id: 2}, + {StoreId: 3, Id: 3}, + {StoreId: 5, Id: 4}, + {StoreId: 6, Id: 5}, + }}, &metapb.Peer{StoreId: 1, Id: 1}) + leaderFilter := newRuleLeaderFitFilter("", testCluster.GetBasicCluster(), testCluster.GetRuleManager(), region, 1, true) + re.Equal(plan.StatusText(plan.StatusStoreNotMatchRule), leaderFilter.Target(testCluster.GetSharedConfig(), testCluster.GetStore(6)).String()) +} + +func TestSendStateFilter(t *testing.T) { + re := require.New(t) + store := core.NewStoreInfoWithLabel(1, map[string]string{}).Clone(core.SetStoreLimit(storelimit.NewSlidingWindows())) + region := core.NewTestRegionInfo(1, 1, []byte(""), []byte("")) + snapshotFilter := NewSnapshotSendFilter([]*core.StoreInfo{store}, constant.Medium) + re.NotNil(SelectOneRegion([]*core.RegionInfo{region}, nil, snapshotFilter)) + re.True(store.GetStoreLimit().Take(1000, storelimit.SendSnapshot, constant.Medium)) + re.True(store.GetStoreLimit().Take(1000, storelimit.SendSnapshot, constant.Medium)) + snapshotFilter = NewSnapshotSendFilter([]*core.StoreInfo{store}, constant.Medium) + re.Nil(SelectOneRegion([]*core.RegionInfo{region}, nil, snapshotFilter)) } func TestStoreStateFilter(t *testing.T) { @@ -320,8 +407,8 @@ func TestIsolationFilter(t *testing.T) { for _, testCase := range testCases { filter := NewIsolationFilter("", testCase.isolationLevel, testCluster.GetLocationLabels(), testCluster.GetRegionStores(testCase.region)) for idx, store := range allStores { - re.Equal(testCase.sourceRes[idx], filter.Source(testCluster.GetOpts(), testCluster.GetStore(store.storeID)).StatusCode) - re.Equal(testCase.targetRes[idx], filter.Target(testCluster.GetOpts(), testCluster.GetStore(store.storeID)).StatusCode) + re.Equal(testCase.sourceRes[idx], filter.Source(testCluster.GetSharedConfig(), testCluster.GetStore(store.storeID)).StatusCode) + re.Equal(testCase.targetRes[idx], filter.Target(testCluster.GetSharedConfig(), testCluster.GetStore(store.storeID)).StatusCode) } } } @@ -348,10 +435,10 @@ func TestPlacementGuard(t *testing.T) { store := testCluster.GetStore(1) re.IsType(NewLocationSafeguard("", []string{"zone"}, testCluster.GetRegionStores(region), store), - NewPlacementSafeguard("", testCluster.GetOpts(), testCluster.GetBasicCluster(), testCluster.GetRuleManager(), region, store, nil)) + NewPlacementSafeguard("", testCluster.GetSharedConfig(), testCluster.GetBasicCluster(), testCluster.GetRuleManager(), region, store, nil)) testCluster.SetEnablePlacementRules(true) re.IsType(newRuleFitFilter("", testCluster.GetBasicCluster(), testCluster.GetRuleManager(), region, nil, 1), - NewPlacementSafeguard("", testCluster.GetOpts(), testCluster.GetBasicCluster(), testCluster.GetRuleManager(), region, store, nil)) + NewPlacementSafeguard("", testCluster.GetSharedConfig(), testCluster.GetBasicCluster(), testCluster.GetRuleManager(), region, store, nil)) } func TestSpecialUseFilter(t *testing.T) { @@ -379,8 +466,8 @@ func TestSpecialUseFilter(t *testing.T) { store := core.NewStoreInfoWithLabel(1, testCase.label) store = store.Clone(core.SetStoreStats(&pdpb.StoreStats{StoreId: 1, Capacity: 100 * units.GiB, Available: 100 * units.GiB})) filter := NewSpecialUseFilter("", testCase.allowUse...) - re.Equal(testCase.sourceRes, filter.Source(testCluster.GetOpts(), store).StatusCode) - re.Equal(testCase.targetRes, filter.Target(testCluster.GetOpts(), store).StatusCode) + re.Equal(testCase.sourceRes, filter.Source(testCluster.GetSharedConfig(), store).StatusCode) + re.Equal(testCase.targetRes, filter.Target(testCluster.GetSharedConfig(), store).StatusCode) } } diff --git a/pkg/schedule/filter/healthy.go b/pkg/schedule/filter/healthy.go index 905aadca1c8..adc4085048b 100644 --- a/pkg/schedule/filter/healthy.go +++ b/pkg/schedule/filter/healthy.go @@ -16,8 +16,7 @@ package filter import ( "github.com/tikv/pd/pkg/core" - "github.com/tikv/pd/pkg/schedule/config" - "github.com/tikv/pd/pkg/schedule/placement" + sche "github.com/tikv/pd/pkg/schedule/core" ) // IsRegionHealthy checks if a region is healthy for scheduling. It requires the @@ -43,30 +42,17 @@ func hasDownPeers(region *core.RegionInfo) bool { // IsRegionReplicated checks if a region is fully replicated. When placement // rules is enabled, its peers should fit corresponding rules. When placement // rules is disabled, it should have enough replicas and no any learner peer. -func IsRegionReplicated(cluster regionHealthCluster, region *core.RegionInfo) bool { - if cluster.GetOpts().IsPlacementRulesEnabled() { +func IsRegionReplicated(cluster sche.SharedCluster, region *core.RegionInfo) bool { + if cluster.GetSharedConfig().IsPlacementRulesEnabled() { return isRegionPlacementRuleSatisfied(cluster, region) } return isRegionReplicasSatisfied(cluster, region) } -func isRegionPlacementRuleSatisfied(cluster regionHealthCluster, region *core.RegionInfo) bool { +func isRegionPlacementRuleSatisfied(cluster sche.SharedCluster, region *core.RegionInfo) bool { return cluster.GetRuleManager().FitRegion(cluster, region).IsSatisfied() } -func isRegionReplicasSatisfied(cluster regionHealthCluster, region *core.RegionInfo) bool { - return len(region.GetLearners()) == 0 && len(region.GetPeers()) == cluster.GetOpts().GetMaxReplicas() -} - -// ReplicatedRegion returns a function that checks if a region is fully replicated. -func ReplicatedRegion(cluster regionHealthCluster) func(*core.RegionInfo) bool { - return func(region *core.RegionInfo) bool { return IsRegionReplicated(cluster, region) } -} - -// cluster provides an overview of a cluster's regions distribution. -type regionHealthCluster interface { - core.StoreSetInformer - core.RegionSetInformer - GetOpts() config.Config - GetRuleManager() *placement.RuleManager +func isRegionReplicasSatisfied(cluster sche.SharedCluster, region *core.RegionInfo) bool { + return len(region.GetLearners()) == 0 && len(region.GetPeers()) == cluster.GetSharedConfig().GetMaxReplicas() } diff --git a/pkg/schedule/filter/region_filters.go b/pkg/schedule/filter/region_filters.go index 2a390d4d9c0..799cee7d90c 100644 --- a/pkg/schedule/filter/region_filters.go +++ b/pkg/schedule/filter/region_filters.go @@ -16,6 +16,9 @@ package filter import ( "github.com/tikv/pd/pkg/core" + "github.com/tikv/pd/pkg/core/constant" + "github.com/tikv/pd/pkg/core/storelimit" + sche "github.com/tikv/pd/pkg/schedule/core" "github.com/tikv/pd/pkg/schedule/placement" "github.com/tikv/pd/pkg/schedule/plan" "github.com/tikv/pd/pkg/slice" @@ -97,12 +100,12 @@ func (f *regionDownFilter) Select(region *core.RegionInfo) *plan.Status { // RegionReplicatedFilter filters all unreplicated regions. type RegionReplicatedFilter struct { - cluster regionHealthCluster + cluster sche.SharedCluster fit *placement.RegionFit } // NewRegionReplicatedFilter creates a RegionFilter that filters all unreplicated regions. -func NewRegionReplicatedFilter(cluster regionHealthCluster) RegionFilter { +func NewRegionReplicatedFilter(cluster sche.SharedCluster) RegionFilter { return &RegionReplicatedFilter{cluster: cluster} } @@ -114,7 +117,7 @@ func (f *RegionReplicatedFilter) GetFit() *placement.RegionFit { // Select returns Ok if the given region satisfy the replication. // it will cache the lasted region fit if the region satisfy the replication. func (f *RegionReplicatedFilter) Select(region *core.RegionInfo) *plan.Status { - if f.cluster.GetOpts().IsPlacementRulesEnabled() { + if f.cluster.GetSharedConfig().IsPlacementRulesEnabled() { fit := f.cluster.GetRuleManager().FitRegion(f.cluster, region) if !fit.IsSatisfied() { return statusRegionNotMatchRule @@ -129,11 +132,11 @@ func (f *RegionReplicatedFilter) Select(region *core.RegionInfo) *plan.Status { } type regionEmptyFilter struct { - cluster regionHealthCluster + cluster sche.SharedCluster } // NewRegionEmptyFilter returns creates a RegionFilter that filters all empty regions. -func NewRegionEmptyFilter(cluster regionHealthCluster) RegionFilter { +func NewRegionEmptyFilter(cluster sche.SharedCluster) RegionFilter { return ®ionEmptyFilter{cluster: cluster} } @@ -145,8 +148,8 @@ func (f *regionEmptyFilter) Select(region *core.RegionInfo) *plan.Status { } // isEmptyRegionAllowBalance returns true if the region is not empty or the number of regions is too small. -func isEmptyRegionAllowBalance(cluster regionHealthCluster, region *core.RegionInfo) bool { - return region.GetApproximateSize() > core.EmptyRegionApproximateSize || cluster.GetRegionCount() < core.InitClusterRegionThreshold +func isEmptyRegionAllowBalance(cluster sche.SharedCluster, region *core.RegionInfo) bool { + return region.GetApproximateSize() > core.EmptyRegionApproximateSize || cluster.GetTotalRegionCount() < core.InitClusterRegionThreshold } type regionWitnessFilter struct { @@ -164,3 +167,29 @@ func (f *regionWitnessFilter) Select(region *core.RegionInfo) *plan.Status { } return statusOK } + +// SnapshotSenderFilter filer the region who's leader store reaches the limit. +type SnapshotSenderFilter struct { + senders map[uint64]struct{} +} + +// NewSnapshotSendFilter returns creates a RegionFilter that filters regions with witness peer on the specific store. +// level should be set as same with the operator priority level. +func NewSnapshotSendFilter(stores []*core.StoreInfo, level constant.PriorityLevel) RegionFilter { + senders := make(map[uint64]struct{}) + for _, store := range stores { + if store.IsAvailable(storelimit.SendSnapshot, level) && !store.IsBusy() { + senders[store.GetID()] = struct{}{} + } + } + return &SnapshotSenderFilter{senders: senders} +} + +// Select returns ok if the region leader in the senders. +func (f *SnapshotSenderFilter) Select(region *core.RegionInfo) *plan.Status { + leaderStoreID := region.GetLeader().GetStoreId() + if _, ok := f.senders[leaderStoreID]; ok { + return statusOK + } + return statusRegionLeaderSendSnapshotThrottled +} diff --git a/pkg/schedule/filter/status.go b/pkg/schedule/filter/status.go index 95a596cefa6..930c59e3ba8 100644 --- a/pkg/schedule/filter/status.go +++ b/pkg/schedule/filter/status.go @@ -43,10 +43,11 @@ var ( statusStoreNotMatchIsolation = plan.NewStatus(plan.StatusStoreNotMatchIsolation) // region filter status - statusRegionPendingPeer = plan.NewStatus(plan.StatusRegionUnhealthy) - statusRegionDownPeer = plan.NewStatus(plan.StatusRegionUnhealthy) - statusRegionEmpty = plan.NewStatus(plan.StatusRegionEmpty) - statusRegionNotMatchRule = plan.NewStatus(plan.StatusRegionNotMatchRule) - statusRegionNotReplicated = plan.NewStatus(plan.StatusRegionNotReplicated) - statusRegionWitnessPeer = plan.NewStatus(plan.StatusRegionNotMatchRule) + statusRegionPendingPeer = plan.NewStatus(plan.StatusRegionUnhealthy) + statusRegionDownPeer = plan.NewStatus(plan.StatusRegionUnhealthy) + statusRegionEmpty = plan.NewStatus(plan.StatusRegionEmpty) + statusRegionNotMatchRule = plan.NewStatus(plan.StatusRegionNotMatchRule) + statusRegionNotReplicated = plan.NewStatus(plan.StatusRegionNotReplicated) + statusRegionWitnessPeer = plan.NewStatus(plan.StatusRegionNotMatchRule) + statusRegionLeaderSendSnapshotThrottled = plan.NewStatus(plan.StatusRegionSendSnapshotThrottled) ) diff --git a/pkg/schedule/labeler/labeler.go b/pkg/schedule/labeler/labeler.go index 01b701e59f8..e80a75fc904 100644 --- a/pkg/schedule/labeler/labeler.go +++ b/pkg/schedule/labeler/labeler.go @@ -25,6 +25,7 @@ import ( "github.com/tikv/pd/pkg/errs" "github.com/tikv/pd/pkg/schedule/rangelist" "github.com/tikv/pd/pkg/storage/endpoint" + "github.com/tikv/pd/pkg/utils/logutil" "github.com/tikv/pd/pkg/utils/syncutil" "go.uber.org/zap" ) @@ -56,6 +57,8 @@ func NewRegionLabeler(ctx context.Context, storage endpoint.RuleStorage, gcInter } func (l *RegionLabeler) doGC(gcInterval time.Duration) { + defer logutil.LogPanic() + ticker := time.NewTicker(gcInterval) defer ticker.Stop() for { diff --git a/pkg/schedule/labeler/metrics.go b/pkg/schedule/labeler/metrics.go new file mode 100644 index 00000000000..31148259c60 --- /dev/null +++ b/pkg/schedule/labeler/metrics.go @@ -0,0 +1,30 @@ +// Copyright 2023 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package labeler + +import "github.com/prometheus/client_golang/prometheus" + +// LabelerEventCounter is a counter of the scheduler labeler system. +var LabelerEventCounter = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "pd", + Subsystem: "schedule", + Name: "labeler_event_counter", + Help: "Counter of the scheduler label.", + }, []string{"type", "event"}) + +func init() { + prometheus.MustRegister(LabelerEventCounter) +} diff --git a/pkg/schedule/labeler/rules.go b/pkg/schedule/labeler/rules.go index c902fff8f66..3b50779d659 100644 --- a/pkg/schedule/labeler/rules.go +++ b/pkg/schedule/labeler/rules.go @@ -171,7 +171,7 @@ func (rule *LabelRule) expireBefore(t time.Time) bool { return rule.minExpire.Before(t) } -// initKeyRangeRulesFromLabelRuleData init and adjust []KeyRangeRule from `LabelRule.Data“ +// initKeyRangeRulesFromLabelRuleData init and adjust []KeyRangeRule from `LabelRule.Data` func initKeyRangeRulesFromLabelRuleData(data interface{}) ([]*KeyRangeRule, error) { rules, ok := data.([]interface{}) if !ok { diff --git a/pkg/schedule/metrics.go b/pkg/schedule/metrics.go index 1207b602adc..6927fb1f178 100644 --- a/pkg/schedule/metrics.go +++ b/pkg/schedule/metrics.go @@ -17,82 +17,33 @@ package schedule import "github.com/prometheus/client_golang/prometheus" var ( - // TODO: pre-allocate gauge metrics - operatorCounter = prometheus.NewCounterVec( - prometheus.CounterOpts{ + hotSpotStatusGauge = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ Namespace: "pd", - Subsystem: "schedule", - Name: "operators_count", - Help: "Counter of schedule operators.", - }, []string{"type", "event"}) + Subsystem: "hotspot", + Name: "status", + Help: "Status of the hotspot.", + }, []string{"address", "store", "type"}) - operatorDuration = prometheus.NewHistogramVec( - prometheus.HistogramOpts{ + regionListGauge = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ Namespace: "pd", - Subsystem: "schedule", - Name: "finish_operators_duration_seconds", - Help: "Bucketed histogram of processing time (s) of finished operator.", - Buckets: []float64{0.5, 1, 2, 4, 8, 16, 20, 40, 60, 90, 120, 180, 240, 300, 480, 600, 720, 900, 1200, 1800, 3600}, + Subsystem: "checker", + Name: "region_list", + Help: "Number of region in waiting list", }, []string{"type"}) - operatorSizeHist = prometheus.NewHistogramVec( - prometheus.HistogramOpts{ + patrolCheckRegionsGauge = prometheus.NewGauge( + prometheus.GaugeOpts{ Namespace: "pd", - Subsystem: "schedule", - Name: "operator_region_size", - Help: "Bucketed histogram of the operator region size.", - Buckets: prometheus.ExponentialBuckets(1, 2, 20), // 1MB~1TB - }, []string{"type"}) - - operatorWaitCounter = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Namespace: "pd", - Subsystem: "schedule", - Name: "operators_waiting_count", - Help: "Counter of schedule waiting operators.", - }, []string{"type", "event"}) - - operatorWaitDuration = prometheus.NewHistogramVec( - prometheus.HistogramOpts{ - Namespace: "pd", - Subsystem: "schedule", - Name: "waiting_operators_duration_seconds", - Help: "Bucketed histogram of waiting time (s) of operator for being promoted.", - Buckets: prometheus.ExponentialBuckets(0.01, 2, 16), - }, []string{"type"}) - - storeLimitCostCounter = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Namespace: "pd", - Subsystem: "schedule", - Name: "store_limit_cost", - Help: "limit rate cost of store.", - }, []string{"store", "limit_type"}) - - scatterCounter = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Namespace: "pd", - Subsystem: "schedule", - Name: "scatter_operators_count", - Help: "Counter of region scatter operators.", - }, []string{"type", "event"}) - - scatterDistributionCounter = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Namespace: "pd", - Subsystem: "schedule", - Name: "scatter_distribution", - Help: "Counter of the distribution in scatter.", - }, []string{"store", "is_leader", "engine"}) + Subsystem: "checker", + Name: "patrol_regions_time", + Help: "Time spent of patrol checks region.", + }) ) func init() { - prometheus.MustRegister(operatorCounter) - prometheus.MustRegister(operatorDuration) - prometheus.MustRegister(operatorWaitDuration) - prometheus.MustRegister(storeLimitCostCounter) - prometheus.MustRegister(operatorWaitCounter) - prometheus.MustRegister(scatterCounter) - prometheus.MustRegister(scatterDistributionCounter) - prometheus.MustRegister(operatorSizeHist) + prometheus.MustRegister(hotSpotStatusGauge) + prometheus.MustRegister(regionListGauge) + prometheus.MustRegister(patrolCheckRegionsGauge) } diff --git a/pkg/schedule/operator/builder.go b/pkg/schedule/operator/builder.go index 88ab6e551d9..1852f292db0 100644 --- a/pkg/schedule/operator/builder.go +++ b/pkg/schedule/operator/builder.go @@ -21,23 +21,13 @@ import ( "github.com/pingcap/errors" "github.com/pingcap/kvproto/pkg/metapb" "github.com/tikv/pd/pkg/core" - "github.com/tikv/pd/pkg/id" - "github.com/tikv/pd/pkg/schedule/config" + sche "github.com/tikv/pd/pkg/schedule/core" "github.com/tikv/pd/pkg/schedule/filter" "github.com/tikv/pd/pkg/schedule/placement" "github.com/tikv/pd/pkg/utils/typeutil" "github.com/tikv/pd/pkg/versioninfo" ) -// ClusterInformer provides the necessary information for building operator. -type ClusterInformer interface { - GetBasicCluster() *core.BasicCluster - GetOpts() config.Config - GetStoreConfig() config.StoreConfig - GetRuleManager() *placement.RuleManager - GetAllocator() id.Allocator -} - // Builder is used to create operators. Usage: // // op, err := NewBuilder(desc, cluster, region). @@ -50,7 +40,7 @@ type ClusterInformer interface { // according to various constraints. type Builder struct { // basic info - ClusterInformer + sche.SharedCluster desc string regionID uint64 regionEpoch *metapb.RegionEpoch @@ -73,7 +63,8 @@ type Builder struct { // build flags useJointConsensus bool - lightWeight bool + removeLightPeer bool + addLightPeer bool forceTargetLeader bool // intermediate states @@ -102,10 +93,10 @@ func SkipPlacementRulesCheck(b *Builder) { } // NewBuilder creates a Builder. -func NewBuilder(desc string, ci ClusterInformer, region *core.RegionInfo, opts ...BuilderOption) *Builder { +func NewBuilder(desc string, ci sche.SharedCluster, region *core.RegionInfo, opts ...BuilderOption) *Builder { b := &Builder{ desc: desc, - ClusterInformer: ci, + SharedCluster: ci, regionID: region.GetID(), regionEpoch: region.GetRegionEpoch(), approximateSize: region.GetApproximateSize(), @@ -145,7 +136,7 @@ func NewBuilder(desc string, ci ClusterInformer, region *core.RegionInfo, opts . // placement rules var rules []*placement.Rule - if err == nil && !b.skipPlacementRulesCheck && b.GetOpts().IsPlacementRulesEnabled() { + if err == nil && !b.skipPlacementRulesCheck && b.GetSharedConfig().IsPlacementRulesEnabled() { fit := b.GetRuleManager().FitRegion(b.GetBasicCluster(), region) for _, rf := range fit.RuleFits { rules = append(rules, rf.Rule) @@ -161,14 +152,14 @@ func NewBuilder(desc string, ci ClusterInformer, region *core.RegionInfo, opts . } // build flags - supportConfChangeV2 := versioninfo.IsFeatureSupported(b.GetOpts().GetClusterVersion(), versioninfo.ConfChangeV2) + supportConfChangeV2 := versioninfo.IsFeatureSupported(b.GetSharedConfig().GetClusterVersion(), versioninfo.ConfChangeV2) b.rules = rules b.originPeers = originPeers b.unhealthyPeers = unhealthyPeers b.originLeaderStoreID = originLeaderStoreID b.targetPeers = originPeers.Copy() - b.useJointConsensus = supportConfChangeV2 && b.GetOpts().IsUseJointConsensus() + b.useJointConsensus = supportConfChangeV2 && b.GetSharedConfig().IsUseJointConsensus() b.err = err return b } @@ -380,9 +371,15 @@ func (b *Builder) SetExpectedRoles(roles map[uint64]placement.PeerRoleType) *Bui return b } -// EnableLightWeight marks the region as light weight. It is used for scatter regions. -func (b *Builder) EnableLightWeight() *Builder { - b.lightWeight = true +// SetAddLightPeer marks the add peer as light weight. It is used for scatter regions. +func (b *Builder) SetAddLightPeer() *Builder { + b.addLightPeer = true + return b +} + +// SetRemoveLightPeer marks the remove peer as light weight. It is used for scatter regions. +func (b *Builder) SetRemoveLightPeer() *Builder { + b.removeLightPeer = true return b } @@ -403,7 +400,6 @@ func (b *Builder) Build(kind OpKind) (*Operator, error) { if brief, b.err = b.prepareBuild(); b.err != nil { return nil, b.err } - if b.useJointConsensus { kind, b.err = b.buildStepsWithJointConsensus(kind) } else { @@ -493,7 +489,7 @@ func (b *Builder) prepareBuild() (string, error) { if o == nil || (!b.useJointConsensus && !core.IsLearner(o) && core.IsLearner(n)) { if n.GetId() == 0 { // Allocate peer ID if need. - id, err := b.GetAllocator().Alloc() + id, err := b.AllocID() if err != nil { return "", err } @@ -543,12 +539,16 @@ func (b *Builder) brief() string { switch { case len(b.toAdd) > 0 && len(b.toRemove) > 0: op := "mv peer" - if b.lightWeight { + if b.addLightPeer && b.removeLightPeer { op = "mv light peer" } return fmt.Sprintf("%s: store %s to %s", op, b.toRemove, b.toAdd) case len(b.toAdd) > 0: return fmt.Sprintf("add peer: store %s", b.toAdd) + case len(b.toRemove) > 0 && len(b.toPromote) > 0: + return fmt.Sprintf("promote peer: store %s, rm peer: store %s", b.toRemove, b.toPromote) + case len(b.toRemove) > 0 && len(b.toDemote) > 0: + return fmt.Sprintf("demote peer: store %s, rm peer: store %s", b.toDemote, b.toRemove) case len(b.toRemove) > 0: return fmt.Sprintf("rm peer: store %s", b.toRemove) case len(b.toPromote) > 0: @@ -782,11 +782,7 @@ func (b *Builder) execPromoteNonWitness(peer *metapb.Peer) { } func (b *Builder) execAddPeer(peer *metapb.Peer) { - if b.lightWeight { - b.steps = append(b.steps, AddLearner{ToStore: peer.GetStoreId(), PeerID: peer.GetId(), IsLightWeight: b.lightWeight, IsWitness: peer.GetIsWitness(), SendStore: b.originLeaderStoreID}) - } else { - b.steps = append(b.steps, AddLearner{ToStore: peer.GetStoreId(), PeerID: peer.GetId(), IsWitness: peer.GetIsWitness(), SendStore: b.originLeaderStoreID}) - } + b.steps = append(b.steps, AddLearner{ToStore: peer.GetStoreId(), PeerID: peer.GetId(), IsLightWeight: b.addLightPeer, IsWitness: peer.GetIsWitness(), SendStore: b.originLeaderStoreID}) if !core.IsLearner(peer) { b.steps = append(b.steps, PromoteLearner{ToStore: peer.GetStoreId(), PeerID: peer.GetId(), IsWitness: peer.GetIsWitness()}) } @@ -800,9 +796,9 @@ func (b *Builder) execRemovePeer(peer *metapb.Peer) { var isDownStore bool store := b.GetBasicCluster().GetStore(removeStoreID) if store != nil { - isDownStore = store.DownTime() > b.GetOpts().GetMaxStoreDownTime() + isDownStore = store.DownTime() > b.GetSharedConfig().GetMaxStoreDownTime() } - b.steps = append(b.steps, RemovePeer{FromStore: removeStoreID, PeerID: peer.GetId(), IsDownStore: isDownStore}) + b.steps = append(b.steps, RemovePeer{FromStore: removeStoreID, PeerID: peer.GetId(), IsDownStore: isDownStore, IsLightWeight: b.removeLightPeer}) delete(b.currentPeers, removeStoreID) delete(b.toRemove, removeStoreID) } @@ -916,7 +912,7 @@ func (b *Builder) allowLeader(peer *metapb.Peer, ignoreClusterLimit bool) bool { stateFilter := &filter.StoreStateFilter{ActionScope: "operator-builder", TransferLeader: true} // store state filter - if !stateFilter.Target(b.GetOpts(), store).IsOK() { + if !stateFilter.Target(b.GetSharedConfig(), store).IsOK() { return false } @@ -1187,7 +1183,7 @@ func (b *Builder) labelMatch(x, y uint64) int { if sx == nil || sy == nil { return 0 } - labels := b.GetOpts().GetLocationLabels() + labels := b.GetSharedConfig().GetLocationLabels() for i, l := range labels { if sx.GetLabelValue(l) != sy.GetLabelValue(l) { return i diff --git a/pkg/schedule/operator/builder_test.go b/pkg/schedule/operator/builder_test.go index f4ead4ffc97..864734eb5ff 100644 --- a/pkg/schedule/operator/builder_test.go +++ b/pkg/schedule/operator/builder_test.go @@ -108,13 +108,13 @@ func (suite *operatorBuilderTestSuite) TestRecord() { 3: {StoreId: 3, Role: metapb.PeerRole_Learner}, 4: {StoreId: 4}, } - builder := suite.newBuilder().SetPeers(m).EnableLightWeight() + builder := suite.newBuilder().SetPeers(m).SetAddLightPeer() suite.Len(builder.targetPeers, 3) suite.Equal(m[2], builder.targetPeers[2]) suite.Equal(m[3], builder.targetPeers[3]) suite.Equal(m[4], builder.targetPeers[4]) suite.Equal(uint64(0), builder.targetLeaderStoreID) - suite.True(builder.lightWeight) + suite.True(builder.addLightPeer) } func (suite *operatorBuilderTestSuite) TestPrepareBuild() { diff --git a/pkg/schedule/operator/create_operator.go b/pkg/schedule/operator/create_operator.go index 0caa656f045..1c96128ab32 100644 --- a/pkg/schedule/operator/create_operator.go +++ b/pkg/schedule/operator/create_operator.go @@ -24,41 +24,61 @@ import ( "github.com/pingcap/log" "github.com/tikv/pd/pkg/core" "github.com/tikv/pd/pkg/errs" + sche "github.com/tikv/pd/pkg/schedule/core" "github.com/tikv/pd/pkg/schedule/placement" "github.com/tikv/pd/pkg/utils/logutil" "go.uber.org/zap" ) // CreateAddPeerOperator creates an operator that adds a new peer. -func CreateAddPeerOperator(desc string, ci ClusterInformer, region *core.RegionInfo, peer *metapb.Peer, kind OpKind) (*Operator, error) { +func CreateAddPeerOperator(desc string, ci sche.SharedCluster, region *core.RegionInfo, peer *metapb.Peer, kind OpKind) (*Operator, error) { return NewBuilder(desc, ci, region). AddPeer(peer). Build(kind) } // CreateDemoteVoterOperator creates an operator that demotes a voter -func CreateDemoteVoterOperator(desc string, ci ClusterInformer, region *core.RegionInfo, peer *metapb.Peer) (*Operator, error) { +func CreateDemoteVoterOperator(desc string, ci sche.SharedCluster, region *core.RegionInfo, peer *metapb.Peer) (*Operator, error) { return NewBuilder(desc, ci, region). DemoteVoter(peer.GetStoreId()). Build(0) } // CreatePromoteLearnerOperator creates an operator that promotes a learner. -func CreatePromoteLearnerOperator(desc string, ci ClusterInformer, region *core.RegionInfo, peer *metapb.Peer) (*Operator, error) { +func CreatePromoteLearnerOperator(desc string, ci sche.SharedCluster, region *core.RegionInfo, peer *metapb.Peer) (*Operator, error) { return NewBuilder(desc, ci, region). PromoteLearner(peer.GetStoreId()). Build(0) } +// CreatePromoteLearnerOperatorAndRemovePeer creates an operator that promotes a learner and removes a peer. +func CreatePromoteLearnerOperatorAndRemovePeer(desc string, ci sche.SharedCluster, region *core.RegionInfo, toPromote *metapb.Peer, toRemove *metapb.Peer) (*Operator, error) { + return NewBuilder(desc, ci, region). + PromoteLearner(toPromote.GetStoreId()). + RemovePeer(toRemove.GetStoreId()). + Build(0) +} + +// CreateDemoteLearnerOperatorAndRemovePeer creates an operator that demotes a learner and removes a peer. +func CreateDemoteLearnerOperatorAndRemovePeer(desc string, ci sche.SharedCluster, region *core.RegionInfo, toDemote *metapb.Peer, toRemove *metapb.Peer) (*Operator, error) { + if !ci.GetSharedConfig().IsUseJointConsensus() { + return nil, errors.Errorf("cannot build demote learner operator due to disabling using joint state") + } + return NewBuilder(desc, ci, region). + DemoteVoter(toDemote.GetStoreId()). + RemovePeer(toRemove.GetStoreId()). + Build(0) +} + // CreateRemovePeerOperator creates an operator that removes a peer from region. -func CreateRemovePeerOperator(desc string, ci ClusterInformer, kind OpKind, region *core.RegionInfo, storeID uint64) (*Operator, error) { +func CreateRemovePeerOperator(desc string, ci sche.SharedCluster, kind OpKind, region *core.RegionInfo, storeID uint64) (*Operator, error) { return NewBuilder(desc, ci, region). RemovePeer(storeID). Build(kind) } // CreateTransferLeaderOperator creates an operator that transfers the leader from a source store to a target store. -func CreateTransferLeaderOperator(desc string, ci ClusterInformer, region *core.RegionInfo, sourceStoreID uint64, targetStoreID uint64, targetStoreIDs []uint64, kind OpKind) (*Operator, error) { +func CreateTransferLeaderOperator(desc string, ci sche.SharedCluster, region *core.RegionInfo, sourceStoreID uint64, targetStoreID uint64, targetStoreIDs []uint64, kind OpKind) (*Operator, error) { return NewBuilder(desc, ci, region, SkipOriginJointStateCheck). SetLeader(targetStoreID). SetLeaders(targetStoreIDs). @@ -66,7 +86,7 @@ func CreateTransferLeaderOperator(desc string, ci ClusterInformer, region *core. } // CreateForceTransferLeaderOperator creates an operator that transfers the leader from a source store to a target store forcible. -func CreateForceTransferLeaderOperator(desc string, ci ClusterInformer, region *core.RegionInfo, sourceStoreID uint64, targetStoreID uint64, kind OpKind) (*Operator, error) { +func CreateForceTransferLeaderOperator(desc string, ci sche.SharedCluster, region *core.RegionInfo, sourceStoreID uint64, targetStoreID uint64, kind OpKind) (*Operator, error) { return NewBuilder(desc, ci, region, SkipOriginJointStateCheck, SkipPlacementRulesCheck). SetLeader(targetStoreID). EnableForceTargetLeader(). @@ -74,7 +94,7 @@ func CreateForceTransferLeaderOperator(desc string, ci ClusterInformer, region * } // CreateMoveRegionOperator creates an operator that moves a region to specified stores. -func CreateMoveRegionOperator(desc string, ci ClusterInformer, region *core.RegionInfo, kind OpKind, roles map[uint64]placement.PeerRoleType) (*Operator, error) { +func CreateMoveRegionOperator(desc string, ci sche.SharedCluster, region *core.RegionInfo, kind OpKind, roles map[uint64]placement.PeerRoleType) (*Operator, error) { // construct the peers from roles oldPeers := region.GetPeers() peers := make(map[uint64]*metapb.Peer) @@ -96,7 +116,7 @@ func CreateMoveRegionOperator(desc string, ci ClusterInformer, region *core.Regi } // CreateMovePeerOperator creates an operator that replaces an old peer with a new peer. -func CreateMovePeerOperator(desc string, ci ClusterInformer, region *core.RegionInfo, kind OpKind, oldStore uint64, peer *metapb.Peer) (*Operator, error) { +func CreateMovePeerOperator(desc string, ci sche.SharedCluster, region *core.RegionInfo, kind OpKind, oldStore uint64, peer *metapb.Peer) (*Operator, error) { return NewBuilder(desc, ci, region). RemovePeer(oldStore). AddPeer(peer). @@ -104,7 +124,7 @@ func CreateMovePeerOperator(desc string, ci ClusterInformer, region *core.Region } // CreateMoveWitnessOperator creates an operator that replaces an old witness with a new witness. -func CreateMoveWitnessOperator(desc string, ci ClusterInformer, region *core.RegionInfo, sourceStoreID uint64, targetStoreID uint64) (*Operator, error) { +func CreateMoveWitnessOperator(desc string, ci sche.SharedCluster, region *core.RegionInfo, sourceStoreID uint64, targetStoreID uint64) (*Operator, error) { return NewBuilder(desc, ci, region). BecomeNonWitness(sourceStoreID). BecomeWitness(targetStoreID). @@ -112,7 +132,7 @@ func CreateMoveWitnessOperator(desc string, ci ClusterInformer, region *core.Reg } // CreateReplaceLeaderPeerOperator creates an operator that replaces an old peer with a new peer, and move leader from old store firstly. -func CreateReplaceLeaderPeerOperator(desc string, ci ClusterInformer, region *core.RegionInfo, kind OpKind, oldStore uint64, peer *metapb.Peer, leader *metapb.Peer) (*Operator, error) { +func CreateReplaceLeaderPeerOperator(desc string, ci sche.SharedCluster, region *core.RegionInfo, kind OpKind, oldStore uint64, peer *metapb.Peer, leader *metapb.Peer) (*Operator, error) { return NewBuilder(desc, ci, region). RemovePeer(oldStore). AddPeer(peer). @@ -121,7 +141,7 @@ func CreateReplaceLeaderPeerOperator(desc string, ci ClusterInformer, region *co } // CreateMoveLeaderOperator creates an operator that replaces an old leader with a new leader. -func CreateMoveLeaderOperator(desc string, ci ClusterInformer, region *core.RegionInfo, kind OpKind, oldStore uint64, peer *metapb.Peer) (*Operator, error) { +func CreateMoveLeaderOperator(desc string, ci sche.SharedCluster, region *core.RegionInfo, kind OpKind, oldStore uint64, peer *metapb.Peer) (*Operator, error) { return NewBuilder(desc, ci, region). RemovePeer(oldStore). AddPeer(peer). @@ -156,7 +176,7 @@ func CreateSplitRegionOperator(desc string, region *core.RegionInfo, kind OpKind } // CreateMergeRegionOperator creates an operator that merge two region into one. -func CreateMergeRegionOperator(desc string, ci ClusterInformer, source *core.RegionInfo, target *core.RegionInfo, kind OpKind) ([]*Operator, error) { +func CreateMergeRegionOperator(desc string, ci sche.SharedCluster, source *core.RegionInfo, target *core.RegionInfo, kind OpKind) ([]*Operator, error) { if core.IsInJointState(source.GetPeers()...) || core.IsInJointState(target.GetPeers()...) { return nil, errors.Errorf("cannot merge regions which are in joint state") } @@ -214,7 +234,7 @@ func isRegionMatch(a, b *core.RegionInfo) bool { } // CreateScatterRegionOperator creates an operator that scatters the specified region. -func CreateScatterRegionOperator(desc string, ci ClusterInformer, origin *core.RegionInfo, targetPeers map[uint64]*metapb.Peer, targetLeader uint64) (*Operator, error) { +func CreateScatterRegionOperator(desc string, ci sche.SharedCluster, origin *core.RegionInfo, targetPeers map[uint64]*metapb.Peer, targetLeader uint64, skipLimitCheck bool) (*Operator, error) { // randomly pick a leader. var ids []uint64 for id, peer := range targetPeers { @@ -229,10 +249,16 @@ func CreateScatterRegionOperator(desc string, ci ClusterInformer, origin *core.R if targetLeader != 0 { leader = targetLeader } - return NewBuilder(desc, ci, origin). + + builder := NewBuilder(desc, ci, origin) + if skipLimitCheck { + builder.SetRemoveLightPeer() + } + + return builder. SetPeers(targetPeers). SetLeader(leader). - EnableLightWeight(). + SetAddLightPeer(). // EnableForceTargetLeader in order to ignore the leader schedule limit EnableForceTargetLeader(). Build(OpAdmin) @@ -242,11 +268,11 @@ func CreateScatterRegionOperator(desc string, ci ClusterInformer, origin *core.R const OpDescLeaveJointState = "leave-joint-state" // CreateLeaveJointStateOperator creates an operator that let region leave joint state. -func CreateLeaveJointStateOperator(desc string, ci ClusterInformer, origin *core.RegionInfo) (*Operator, error) { +func CreateLeaveJointStateOperator(desc string, ci sche.SharedCluster, origin *core.RegionInfo) (*Operator, error) { b := NewBuilder(desc, ci, origin, SkipOriginJointStateCheck, SkipPlacementRulesCheck) if b.err == nil && !core.IsInJointState(origin.GetPeers()...) { - b.err = errors.Errorf("cannot build leave joint state operator for region which is not in joint state") + b.err = errors.Errorf("cannot build leave joint state operator due to disabling using joint state") } if b.err != nil { @@ -302,14 +328,14 @@ func CreateLeaveJointStateOperator(desc string, ci ClusterInformer, origin *core } // CreateWitnessPeerOperator creates an operator that set a follower or learner peer with witness -func CreateWitnessPeerOperator(desc string, ci ClusterInformer, region *core.RegionInfo, peer *metapb.Peer) (*Operator, error) { +func CreateWitnessPeerOperator(desc string, ci sche.SharedCluster, region *core.RegionInfo, peer *metapb.Peer) (*Operator, error) { return NewBuilder(desc, ci, region). BecomeWitness(peer.GetStoreId()). Build(OpWitness) } // CreateNonWitnessPeerOperator creates an operator that set a peer with non-witness -func CreateNonWitnessPeerOperator(desc string, ci ClusterInformer, region *core.RegionInfo, peer *metapb.Peer) (*Operator, error) { +func CreateNonWitnessPeerOperator(desc string, ci sche.SharedCluster, region *core.RegionInfo, peer *metapb.Peer) (*Operator, error) { return NewBuilder(desc, ci, region). BecomeNonWitness(peer.GetStoreId()). Build(OpWitness) diff --git a/pkg/schedule/operator/influence.go b/pkg/schedule/operator/influence.go index 5cce6ffdf61..2b42ffa3516 100644 --- a/pkg/schedule/operator/influence.go +++ b/pkg/schedule/operator/influence.go @@ -32,6 +32,13 @@ func NewOpInfluence() *OpInfluence { } } +// Add adds another influence. +func (m *OpInfluence) Add(other *OpInfluence) { + for id, v := range other.StoresInfluence { + m.GetStoreInfluence(id).add(v) + } +} + // GetStoreInfluence get storeInfluence of specific store. func (m OpInfluence) GetStoreInfluence(id uint64) *StoreInfluence { storeInfluence, ok := m.StoresInfluence[id] @@ -52,8 +59,19 @@ type StoreInfluence struct { StepCost map[storelimit.Type]int64 } +func (s *StoreInfluence) add(other *StoreInfluence) { + s.RegionCount += other.RegionCount + s.RegionSize += other.RegionSize + s.LeaderSize += other.LeaderSize + s.LeaderCount += other.LeaderCount + s.WitnessCount += other.WitnessCount + for _, v := range storelimit.TypeNameValue { + s.AddStepCost(v, other.GetStepCost(v)) + } +} + // ResourceProperty returns delta size of leader/region by influence. -func (s StoreInfluence) ResourceProperty(kind constant.ScheduleKind) int64 { +func (s *StoreInfluence) ResourceProperty(kind constant.ScheduleKind) int64 { switch kind.Resource { case constant.LeaderKind: switch kind.Policy { @@ -74,7 +92,7 @@ func (s StoreInfluence) ResourceProperty(kind constant.ScheduleKind) int64 { } // GetStepCost returns the specific type step cost -func (s StoreInfluence) GetStepCost(limitType storelimit.Type) int64 { +func (s *StoreInfluence) GetStepCost(limitType storelimit.Type) int64 { if s.StepCost == nil { return 0 } diff --git a/pkg/schedule/operator/metrics.go b/pkg/schedule/operator/metrics.go index 33c23e925cf..20bb4e6b7ca 100644 --- a/pkg/schedule/operator/metrics.go +++ b/pkg/schedule/operator/metrics.go @@ -35,6 +35,14 @@ var ( Help: "Counter of operator meeting limit", }, []string{"type", "name"}) + storeLimitCostCounter = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "pd", + Subsystem: "schedule", + Name: "store_limit_cost", + Help: "limit rate cost of store.", + }, []string{"store", "limit_type"}) + // OperatorExceededStoreLimitCounter exposes the counter when operator meet exceeded store limit. OperatorExceededStoreLimitCounter = prometheus.NewCounterVec( prometheus.CounterOpts{ @@ -43,10 +51,41 @@ var ( Name: "operator_exceeded_store_limit", Help: "Counter of operator meeting store limit", }, []string{"desc"}) + + // TODO: pre-allocate gauge metrics + operatorCounter = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "pd", + Subsystem: "schedule", + Name: "operators_count", + Help: "Counter of schedule operators.", + }, []string{"type", "event"}) + + operatorDuration = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Namespace: "pd", + Subsystem: "schedule", + Name: "finish_operators_duration_seconds", + Help: "Bucketed histogram of processing time (s) of finished operator.", + Buckets: []float64{0.5, 1, 2, 4, 8, 16, 20, 40, 60, 90, 120, 180, 240, 300, 480, 600, 720, 900, 1200, 1800, 3600}, + }, []string{"type"}) + + operatorSizeHist = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Namespace: "pd", + Subsystem: "schedule", + Name: "operator_region_size", + Help: "Bucketed histogram of the operator region size.", + Buckets: prometheus.ExponentialBuckets(1, 2, 20), // 1MB~1TB + }, []string{"type"}) ) func init() { prometheus.MustRegister(operatorStepDuration) prometheus.MustRegister(OperatorLimitCounter) prometheus.MustRegister(OperatorExceededStoreLimitCounter) + prometheus.MustRegister(operatorCounter) + prometheus.MustRegister(operatorDuration) + prometheus.MustRegister(operatorSizeHist) + prometheus.MustRegister(storeLimitCostCounter) } diff --git a/pkg/schedule/operator/operator.go b/pkg/schedule/operator/operator.go index f4b1d819cf4..755c69d00fd 100644 --- a/pkg/schedule/operator/operator.go +++ b/pkg/schedule/operator/operator.go @@ -18,6 +18,7 @@ import ( "encoding/json" "fmt" "reflect" + "strconv" "strings" "sync/atomic" "time" @@ -32,6 +33,41 @@ const ( // OperatorExpireTime is the duration that when an operator is not started // after it, the operator will be considered expired. OperatorExpireTime = 3 * time.Second + cancelReason = "cancel-reason" +) + +// CancelReasonType is the type of cancel reason. +type CancelReasonType string + +var ( + // RegionNotFound is the cancel reason when the region is not found. + RegionNotFound CancelReasonType = "region not found" + // EpochNotMatch is the cancel reason when the region epoch is not match. + EpochNotMatch CancelReasonType = "epoch not match" + // AlreadyExist is the cancel reason when the operator is running. + AlreadyExist CancelReasonType = "already exist" + // AdminStop is the cancel reason when the operator is stopped by adminer. + AdminStop CancelReasonType = "admin stop" + // NotInRunningState is the cancel reason when the operator is not in running state. + NotInRunningState CancelReasonType = "not in running state" + // Succeed is the cancel reason when the operator is finished successfully. + Succeed CancelReasonType = "succeed" + // Timeout is the cancel reason when the operator is timeout. + Timeout CancelReasonType = "timeout" + // Expired is the cancel reason when the operator is expired. + Expired CancelReasonType = "expired" + // NotInCreateStatus is the cancel reason when the operator is not in create status. + NotInCreateStatus CancelReasonType = "not in create status" + // StaleStatus is the cancel reason when the operator is in a stale status. + StaleStatus CancelReasonType = "stale status" + // ExceedStoreLimit is the cancel reason when the operator exceeds the store limit. + ExceedStoreLimit CancelReasonType = "exceed store limit" + // ExceedWaitLimit is the cancel reason when the operator exceeds the waiting queue limit. + ExceedWaitLimit CancelReasonType = "exceed wait limit" + // RelatedMergeRegion is the cancel reason when the operator is cancelled by related merge region. + RelatedMergeRegion CancelReasonType = "related merge region" + // Unknown is the cancel reason when the operator is cancelled by an unknown reason. + Unknown CancelReasonType = "unknown" ) // Operator contains execution steps generated by scheduler. @@ -52,6 +88,7 @@ type Operator struct { AdditionalInfos map[string]string ApproximateSize int64 timeout time.Duration + influence *OpInfluence } // NewOperator creates a new operator. @@ -83,14 +120,16 @@ func NewOperator(desc, brief string, regionID uint64, regionEpoch *metapb.Region // Sync some attribute with the given timeout. func (o *Operator) Sync(other *Operator) { o.timeout = other.timeout + o.AdditionalInfos[string(RelatedMergeRegion)] = strconv.FormatUint(other.RegionID(), 10) + other.AdditionalInfos[string(RelatedMergeRegion)] = strconv.FormatUint(o.RegionID(), 10) } func (o *Operator) String() string { stepStrs := make([]string, len(o.steps)) for i := range o.steps { - stepStrs[i] = o.steps[i].String() + stepStrs[i] = fmt.Sprintf("%d:{%s}", i, o.steps[i].String()) } - s := fmt.Sprintf("%s {%s} (kind:%s, region:%v(%v, %v), createAt:%s, startAt:%s, currentStep:%v, size:%d, steps:[%s]，timeout:[%s])", + s := fmt.Sprintf("%s {%s} (kind:%s, region:%v(%v, %v), createAt:%s, startAt:%s, currentStep:%v, size:%d, steps:[%s], timeout:[%s])", o.desc, o.brief, o.kind, o.regionID, o.regionEpoch.GetVersion(), o.regionEpoch.GetConfVer(), o.GetCreateTime(), o.GetStartTime(), atomic.LoadInt32(&o.currentStep), o.ApproximateSize, strings.Join(stepStrs, ", "), o.timeout.String()) if o.CheckSuccess() { @@ -102,6 +141,11 @@ func (o *Operator) String() string { return s } +// Brief returns the operator's short brief. +func (o *Operator) Brief() string { + return o.brief +} + // MarshalJSON serializes custom types to JSON. func (o *Operator) MarshalJSON() ([]byte, error) { return []byte(`"` + o.String() + `"`), nil @@ -221,7 +265,10 @@ func (o *Operator) CheckSuccess() bool { } // Cancel marks the operator canceled. -func (o *Operator) Cancel() bool { +func (o *Operator) Cancel(reason CancelReasonType) bool { + if _, ok := o.AdditionalInfos[cancelReason]; !ok { + o.AdditionalInfos[cancelReason] = string(reason) + } return o.status.To(CANCELED) } @@ -342,9 +389,17 @@ func (o *Operator) UnfinishedInfluence(opInfluence OpInfluence, region *core.Reg // TotalInfluence calculates the store difference which whole operator steps make. func (o *Operator) TotalInfluence(opInfluence OpInfluence, region *core.RegionInfo) { - for step := 0; step < len(o.steps); step++ { - o.steps[step].Influence(opInfluence, region) + // skip if region is nil and not cache influence. + if region == nil && o.influence == nil { + return + } + if o.influence == nil { + o.influence = NewOpInfluence() + for step := 0; step < len(o.steps); step++ { + o.steps[step].Influence(*o.influence, region) + } } + opInfluence.Add(o.influence) } // OpHistory is used to log and visualize completed operators. diff --git a/pkg/schedule/operator_controller.go b/pkg/schedule/operator/operator_controller.go similarity index 63% rename from pkg/schedule/operator_controller.go rename to pkg/schedule/operator/operator_controller.go index b64d88c1aa3..1385a120337 100644 --- a/pkg/schedule/operator_controller.go +++ b/pkg/schedule/operator/operator_controller.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package schedule +package operator import ( "container/heap" @@ -29,9 +29,8 @@ import ( "github.com/tikv/pd/pkg/core/constant" "github.com/tikv/pd/pkg/core/storelimit" "github.com/tikv/pd/pkg/errs" + "github.com/tikv/pd/pkg/schedule/config" "github.com/tikv/pd/pkg/schedule/hbstream" - "github.com/tikv/pd/pkg/schedule/labeler" - "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/utils/syncutil" "github.com/tikv/pd/pkg/versioninfo" "go.uber.org/zap" @@ -47,91 +46,95 @@ const ( var ( slowNotifyInterval = 5 * time.Second fastNotifyInterval = 2 * time.Second - // PushOperatorTickInterval is the interval try to push the operator. - PushOperatorTickInterval = 500 * time.Millisecond // StoreBalanceBaseTime represents the base time of balance rate. StoreBalanceBaseTime float64 = 60 - // FastOperatorFinishTime min finish time, if finish duration less than it,op will be pushed to fast operator queue + // FastOperatorFinishTime min finish time, if finish duration less than it, op will be pushed to fast operator queue FastOperatorFinishTime = 10 * time.Second ) -// OperatorController is used to limit the speed of scheduling. -type OperatorController struct { +// Controller is used to limit the speed of scheduling. +type Controller struct { syncutil.RWMutex ctx context.Context - cluster Cluster - operators map[uint64]*operator.Operator + config config.SharedConfigProvider + cluster *core.BasicCluster + operators map[uint64]*Operator hbStreams *hbstream.HeartbeatStreams fastOperators *cache.TTLUint64 - counts map[operator.OpKind]uint64 - opRecords *OperatorRecords + counts map[OpKind]uint64 + records *records wop WaitingOperator - wopStatus *WaitingOperatorStatus + wopStatus *waitingOperatorStatus opNotifierQueue operatorQueue } -// NewOperatorController creates a OperatorController. -func NewOperatorController(ctx context.Context, cluster Cluster, hbStreams *hbstream.HeartbeatStreams) *OperatorController { - return &OperatorController{ +// NewController creates a Controller. +func NewController(ctx context.Context, cluster *core.BasicCluster, config config.SharedConfigProvider, hbStreams *hbstream.HeartbeatStreams) *Controller { + return &Controller{ ctx: ctx, cluster: cluster, - operators: make(map[uint64]*operator.Operator), + config: config, + operators: make(map[uint64]*Operator), hbStreams: hbStreams, fastOperators: cache.NewIDTTL(ctx, time.Minute, FastOperatorFinishTime), - counts: make(map[operator.OpKind]uint64), - opRecords: NewOperatorRecords(ctx), - wop: NewRandBuckets(), - wopStatus: NewWaitingOperatorStatus(), + counts: make(map[OpKind]uint64), + records: newRecords(ctx), + wop: newRandBuckets(), + wopStatus: newWaitingOperatorStatus(), opNotifierQueue: make(operatorQueue, 0), } } // Ctx returns a context which will be canceled once RaftCluster is stopped. // For now, it is only used to control the lifetime of TTL cache in schedulers. -func (oc *OperatorController) Ctx() context.Context { +func (oc *Controller) Ctx() context.Context { return oc.ctx } -// GetCluster exports cluster to evict-scheduler for check store status. -func (oc *OperatorController) GetCluster() Cluster { +// GetCluster exports basic cluster to evict-scheduler for check store status. +func (oc *Controller) GetCluster() *core.BasicCluster { oc.RLock() defer oc.RUnlock() return oc.cluster } +// GetHBStreams returns the heartbeat steams. +func (oc *Controller) GetHBStreams() *hbstream.HeartbeatStreams { + return oc.hbStreams +} + // Dispatch is used to dispatch the operator of a region. -func (oc *OperatorController) Dispatch(region *core.RegionInfo, source string) { - // Check existed operator. +func (oc *Controller) Dispatch(region *core.RegionInfo, source string, recordOpStepWithTTL func(regionID uint64)) { + // Check existed if op := oc.GetOperator(region.GetID()); op != nil { failpoint.Inject("concurrentRemoveOperator", func() { time.Sleep(500 * time.Millisecond) }) - // Update operator status: // The operator status should be STARTED. // Check will call CheckSuccess and CheckTimeout. step := op.Check(region) switch op.Status() { - case operator.STARTED: + case STARTED: operatorCounter.WithLabelValues(op.Desc(), "check").Inc() if source == DispatchFromHeartBeat && oc.checkStaleOperator(op, step, region) { return } oc.SendScheduleCommand(region, step, source) - case operator.SUCCESS: + case SUCCESS: if op.ContainNonWitnessStep() { - oc.cluster.RecordOpStepWithTTL(op.RegionID()) + recordOpStepWithTTL(op.RegionID()) } - if oc.RemoveOperator(op) { - operatorWaitCounter.WithLabelValues(op.Desc(), "promote-success").Inc() + if oc.RemoveOperator(op, Succeed) { + operatorCounter.WithLabelValues(op.Desc(), "promote-success").Inc() oc.PromoteWaitingOperator() } if time.Since(op.GetStartTime()) < FastOperatorFinishTime { log.Debug("op finish duration less than 10s", zap.Uint64("region-id", op.RegionID())) oc.pushFastOperator(op) } - case operator.TIMEOUT: - if oc.RemoveOperator(op) { + case TIMEOUT: + if oc.RemoveOperator(op, Timeout) { operatorCounter.WithLabelValues(op.Desc(), "promote-timeout").Inc() oc.PromoteWaitingOperator() } @@ -141,27 +144,26 @@ func (oc *OperatorController) Dispatch(region *core.RegionInfo, source string) { // CANCELED, REPLACED must remove before transition. log.Error("dispatching operator with unexpected status", zap.Uint64("region-id", op.RegionID()), - zap.String("status", operator.OpStatusToString(op.Status())), + zap.String("status", OpStatusToString(op.Status())), zap.Reflect("operator", op), errs.ZapError(errs.ErrUnexpectedOperatorStatus)) - operatorWaitCounter.WithLabelValues(op.Desc(), "unexpected").Inc() failpoint.Inject("unexpectedOperator", func() { panic(op) }) - _ = op.Cancel() + _ = op.Cancel(NotInRunningState) oc.buryOperator(op) - operatorWaitCounter.WithLabelValues(op.Desc(), "promote-unexpected").Inc() + operatorCounter.WithLabelValues(op.Desc(), "promote-unexpected").Inc() oc.PromoteWaitingOperator() } } } } -func (oc *OperatorController) checkStaleOperator(op *operator.Operator, step operator.OpStep, region *core.RegionInfo) bool { - err := step.CheckInProgress(oc.cluster, region) +func (oc *Controller) checkStaleOperator(op *Operator, step OpStep, region *core.RegionInfo) bool { + err := step.CheckInProgress(oc.cluster, oc.config, region) if err != nil { - if oc.RemoveOperator(op, zap.String("reason", err.Error())) { - operatorCounter.WithLabelValues(op.Desc(), "stale").Inc() - operatorWaitCounter.WithLabelValues(op.Desc(), "promote-stale").Inc() + log.Info("operator is stale", zap.Uint64("region-id", op.RegionID()), errs.ZapError(err)) + if oc.RemoveOperator(op, StaleStatus) { + operatorCounter.WithLabelValues(op.Desc(), "promote-stale").Inc() oc.PromoteWaitingOperator() return true } @@ -174,14 +176,15 @@ func (oc *OperatorController) checkStaleOperator(op *operator.Operator, step ope latest := region.GetRegionEpoch() changes := latest.GetConfVer() - origin.GetConfVer() if changes > op.ConfVerChanged(region) { + log.Info("operator is stale", + zap.Uint64("region-id", op.RegionID()), + zap.Uint64("diff", changes), + zap.Reflect("latest-epoch", region.GetRegionEpoch())) if oc.RemoveOperator( op, - zap.String("reason", "stale operator, confver does not meet expectations"), - zap.Reflect("latest-epoch", region.GetRegionEpoch()), - zap.Uint64("diff", changes), + EpochNotMatch, ) { - operatorCounter.WithLabelValues(op.Desc(), "stale").Inc() - operatorWaitCounter.WithLabelValues(op.Desc(), "promote-stale").Inc() + operatorCounter.WithLabelValues(op.Desc(), "promote-stale").Inc() oc.PromoteWaitingOperator() return true } @@ -190,10 +193,10 @@ func (oc *OperatorController) checkStaleOperator(op *operator.Operator, step ope return false } -func (oc *OperatorController) getNextPushOperatorTime(step operator.OpStep, now time.Time) time.Time { +func (oc *Controller) getNextPushOperatorTime(step OpStep, now time.Time) time.Time { nextTime := slowNotifyInterval switch step.(type) { - case operator.TransferLeader, operator.PromoteLearner, operator.ChangePeerV2Enter, operator.ChangePeerV2Leave: + case TransferLeader, PromoteLearner, ChangePeerV2Enter, ChangePeerV2Leave: nextTime = fastNotifyInterval } return now.Add(nextTime) @@ -202,7 +205,7 @@ func (oc *OperatorController) getNextPushOperatorTime(step operator.OpStep, now // pollNeedDispatchRegion returns the region need to dispatch, // "next" is true to indicate that it may exist in next attempt, // and false is the end for the poll. -func (oc *OperatorController) pollNeedDispatchRegion() (r *core.RegionInfo, next bool) { +func (oc *Controller) pollNeedDispatchRegion() (r *core.RegionInfo, next bool) { oc.Lock() defer oc.Unlock() if oc.opNotifierQueue.Len() == 0 { @@ -217,7 +220,7 @@ func (oc *OperatorController) pollNeedDispatchRegion() (r *core.RegionInfo, next r = oc.cluster.GetRegion(regionID) if r == nil { _ = oc.removeOperatorLocked(op) - if op.Cancel() { + if op.Cancel(RegionNotFound) { log.Warn("remove operator because region disappeared", zap.Uint64("region-id", op.RegionID()), zap.Stringer("operator", op)) @@ -243,7 +246,7 @@ func (oc *OperatorController) pollNeedDispatchRegion() (r *core.RegionInfo, next } // PushOperators periodically pushes the unfinished operator to the executor(TiKV). -func (oc *OperatorController) PushOperators() { +func (oc *Controller) PushOperators(recordOpStepWithTTL func(regionID uint64)) { for { r, next := oc.pollNeedDispatchRegion() if !next { @@ -253,12 +256,12 @@ func (oc *OperatorController) PushOperators() { continue } - oc.Dispatch(r, DispatchFromNotifierQueue) + oc.Dispatch(r, DispatchFromNotifierQueue, recordOpStepWithTTL) } } // AddWaitingOperator adds operators to waiting operators. -func (oc *OperatorController) AddWaitingOperator(ops ...*operator.Operator) int { +func (oc *Controller) AddWaitingOperator(ops ...*Operator) int { oc.Lock() added := 0 needPromoted := 0 @@ -267,14 +270,14 @@ func (oc *OperatorController) AddWaitingOperator(ops ...*operator.Operator) int op := ops[i] desc := op.Desc() isMerge := false - if op.Kind()&operator.OpMerge != 0 { + if op.Kind()&OpMerge != 0 { if i+1 >= len(ops) { // should not be here forever log.Error("orphan merge operators found", zap.String("desc", desc), errs.ZapError(errs.ErrMergeOperator.FastGenByArgs("orphan operator found"))) oc.Unlock() return added } - if ops[i+1].Kind()&operator.OpMerge == 0 { + if ops[i+1].Kind()&OpMerge == 0 { log.Error("merge operator should be paired", zap.String("desc", ops[i+1].Desc()), errs.ZapError(errs.ErrMergeOperator.FastGenByArgs("operator should be paired"))) oc.Unlock() @@ -282,14 +285,14 @@ func (oc *OperatorController) AddWaitingOperator(ops ...*operator.Operator) int } isMerge = true } - if !oc.checkAddOperator(false, op) { - _ = op.Cancel() + if pass, reason := oc.checkAddOperator(false, op); !pass { + _ = op.Cancel(reason) oc.buryOperator(op) if isMerge { // Merge operation have two operators, cancel them all i++ next := ops[i] - _ = next.Cancel() + _ = next.Cancel(reason) oc.buryOperator(next) } continue @@ -302,14 +305,14 @@ func (oc *OperatorController) AddWaitingOperator(ops ...*operator.Operator) int added++ oc.wop.PutOperator(ops[i]) } - operatorWaitCounter.WithLabelValues(desc, "put").Inc() + operatorCounter.WithLabelValues(desc, "put").Inc() oc.wopStatus.ops[desc]++ added++ needPromoted++ } oc.Unlock() - operatorWaitCounter.WithLabelValues(ops[0].Desc(), "promote-add").Add(float64(needPromoted)) + operatorCounter.WithLabelValues(ops[0].Desc(), "promote-add").Add(float64(needPromoted)) for i := 0; i < needPromoted; i++ { oc.PromoteWaitingOperator() } @@ -317,16 +320,23 @@ func (oc *OperatorController) AddWaitingOperator(ops ...*operator.Operator) int } // AddOperator adds operators to the running operators. -func (oc *OperatorController) AddOperator(ops ...*operator.Operator) bool { +func (oc *Controller) AddOperator(ops ...*Operator) bool { oc.Lock() defer oc.Unlock() // note: checkAddOperator uses false param for `isPromoting`. // This is used to keep check logic before fixing issue #4946, // but maybe user want to add operator when waiting queue is busy - if oc.exceedStoreLimitLocked(ops...) || !oc.checkAddOperator(false, ops...) { + if oc.exceedStoreLimitLocked(ops...) { for _, op := range ops { - _ = op.Cancel() + _ = op.Cancel(ExceedStoreLimit) + oc.buryOperator(op) + } + return false + } + if pass, reason := oc.checkAddOperator(false, ops...); !pass { + for _, op := range ops { + _ = op.Cancel(reason) oc.buryOperator(op) } return false @@ -340,22 +350,31 @@ func (oc *OperatorController) AddOperator(ops ...*operator.Operator) bool { } // PromoteWaitingOperator promotes operators from waiting operators. -func (oc *OperatorController) PromoteWaitingOperator() { +func (oc *Controller) PromoteWaitingOperator() { oc.Lock() defer oc.Unlock() - var ops []*operator.Operator + var ops []*Operator for { // GetOperator returns one operator or two merge operators ops = oc.wop.GetOperator() if ops == nil { return } - operatorWaitCounter.WithLabelValues(ops[0].Desc(), "get").Inc() + operatorCounter.WithLabelValues(ops[0].Desc(), "get").Inc() + if oc.exceedStoreLimitLocked(ops...) { + for _, op := range ops { + operatorCounter.WithLabelValues(op.Desc(), "exceed-limit").Inc() + _ = op.Cancel(ExceedStoreLimit) + oc.buryOperator(op) + } + oc.wopStatus.ops[ops[0].Desc()]-- + continue + } - if oc.exceedStoreLimitLocked(ops...) || !oc.checkAddOperator(true, ops...) { + if pass, reason := oc.checkAddOperator(true, ops...); !pass { for _, op := range ops { - operatorWaitCounter.WithLabelValues(op.Desc(), "promote-canceled").Inc() - _ = op.Cancel() + operatorCounter.WithLabelValues(op.Desc(), "check-failed").Inc() + _ = op.Cancel(reason) oc.buryOperator(op) } oc.wopStatus.ops[ops[0].Desc()]-- @@ -376,17 +395,17 @@ func (oc *OperatorController) PromoteWaitingOperator() { // There are several situations that cannot be added: // - There is no such region in the cluster // - The epoch of the operator and the epoch of the corresponding region are no longer consistent. -// - The region already has a higher priority or same priority operator. +// - The region already has a higher priority or same priority // - Exceed the max number of waiting operators // - At least one operator is expired. -func (oc *OperatorController) checkAddOperator(isPromoting bool, ops ...*operator.Operator) bool { +func (oc *Controller) checkAddOperator(isPromoting bool, ops ...*Operator) (bool, CancelReasonType) { for _, op := range ops { region := oc.cluster.GetRegion(op.RegionID()) if region == nil { log.Debug("region not found, cancel add operator", zap.Uint64("region-id", op.RegionID())) - operatorWaitCounter.WithLabelValues(op.Desc(), "not-found").Inc() - return false + operatorCounter.WithLabelValues(op.Desc(), "not-found").Inc() + return false, RegionNotFound } if region.GetRegionEpoch().GetVersion() != op.RegionEpoch().GetVersion() || region.GetRegionEpoch().GetConfVer() != op.RegionEpoch().GetConfVer() { @@ -394,60 +413,52 @@ func (oc *OperatorController) checkAddOperator(isPromoting bool, ops ...*operato zap.Uint64("region-id", op.RegionID()), zap.Reflect("old", region.GetRegionEpoch()), zap.Reflect("new", op.RegionEpoch())) - operatorWaitCounter.WithLabelValues(op.Desc(), "epoch-not-match").Inc() - return false + operatorCounter.WithLabelValues(op.Desc(), "epoch-not-match").Inc() + return false, EpochNotMatch } if old := oc.operators[op.RegionID()]; old != nil && !isHigherPriorityOperator(op, old) { log.Debug("already have operator, cancel add operator", zap.Uint64("region-id", op.RegionID()), zap.Reflect("old", old)) - operatorWaitCounter.WithLabelValues(op.Desc(), "already-have").Inc() - return false + operatorCounter.WithLabelValues(op.Desc(), "already-have").Inc() + return false, AlreadyExist } - if op.Status() != operator.CREATED { + if op.Status() != CREATED { log.Error("trying to add operator with unexpected status", zap.Uint64("region-id", op.RegionID()), - zap.String("status", operator.OpStatusToString(op.Status())), + zap.String("status", OpStatusToString(op.Status())), zap.Reflect("operator", op), errs.ZapError(errs.ErrUnexpectedOperatorStatus)) failpoint.Inject("unexpectedOperator", func() { panic(op) }) - operatorWaitCounter.WithLabelValues(op.Desc(), "unexpected-status").Inc() - return false + operatorCounter.WithLabelValues(op.Desc(), "unexpected-status").Inc() + return false, NotInCreateStatus } - if !isPromoting && oc.wopStatus.ops[op.Desc()] >= oc.cluster.GetOpts().GetSchedulerMaxWaitingOperator() { - log.Debug("exceed max return false", zap.Uint64("waiting", oc.wopStatus.ops[op.Desc()]), zap.String("desc", op.Desc()), zap.Uint64("max", oc.cluster.GetOpts().GetSchedulerMaxWaitingOperator())) - operatorWaitCounter.WithLabelValues(op.Desc(), "exceed-max").Inc() - return false + if !isPromoting && oc.wopStatus.ops[op.Desc()] >= oc.config.GetSchedulerMaxWaitingOperator() { + log.Debug("exceed max return false", zap.Uint64("waiting", oc.wopStatus.ops[op.Desc()]), zap.String("desc", op.Desc()), zap.Uint64("max", oc.config.GetSchedulerMaxWaitingOperator())) + operatorCounter.WithLabelValues(op.Desc(), "exceed-max-waiting").Inc() + return false, ExceedWaitLimit } - if op.SchedulerKind() == operator.OpAdmin || op.IsLeaveJointStateOperator() { + if op.SchedulerKind() == OpAdmin || op.IsLeaveJointStateOperator() { continue } - if cl, ok := oc.cluster.(interface{ GetRegionLabeler() *labeler.RegionLabeler }); ok { - l := cl.GetRegionLabeler() - if l.ScheduleDisabled(region) { - log.Debug("schedule disabled", zap.Uint64("region-id", op.RegionID())) - operatorWaitCounter.WithLabelValues(op.Desc(), "schedule-disabled").Inc() - return false - } - } } - expired := false + var reason CancelReasonType for _, op := range ops { if op.CheckExpired() { - expired = true - operatorWaitCounter.WithLabelValues(op.Desc(), "expired").Inc() + reason = Expired + operatorCounter.WithLabelValues(op.Desc(), "expired").Inc() } } - return !expired + return reason != Expired, reason } -func isHigherPriorityOperator(new, old *operator.Operator) bool { +func isHigherPriorityOperator(new, old *Operator) bool { return new.GetPriorityLevel() > old.GetPriorityLevel() } -func (oc *OperatorController) addOperatorLocked(op *operator.Operator) bool { +func (oc *Controller) addOperatorLocked(op *Operator) bool { regionID := op.RegionID() log.Info("add operator", zap.Uint64("region-id", regionID), @@ -465,7 +476,7 @@ func (oc *OperatorController) addOperatorLocked(op *operator.Operator) bool { if !op.Start() { log.Error("adding operator with unexpected status", zap.Uint64("region-id", regionID), - zap.String("status", operator.OpStatusToString(op.Status())), + zap.String("status", OpStatusToString(op.Status())), zap.Reflect("operator", op), errs.ZapError(errs.ErrUnexpectedOperatorStatus)) failpoint.Inject("unexpectedOperator", func() { panic(op) @@ -476,8 +487,7 @@ func (oc *OperatorController) addOperatorLocked(op *operator.Operator) bool { oc.operators[regionID] = op operatorCounter.WithLabelValues(op.Desc(), "start").Inc() operatorSizeHist.WithLabelValues(op.Desc()).Observe(float64(op.ApproximateSize)) - operatorWaitDuration.WithLabelValues(op.Desc()).Observe(op.ElapsedTime().Seconds()) - opInfluence := NewTotalOpInfluence([]*operator.Operator{op}, oc.cluster) + opInfluence := NewTotalOpInfluence([]*Operator{op}, oc.cluster) for storeID := range opInfluence.StoresInfluence { store := oc.cluster.GetStore(storeID) if store == nil { @@ -496,7 +506,7 @@ func (oc *OperatorController) addOperatorLocked(op *operator.Operator) bool { } oc.updateCounts(oc.operators) - var step operator.OpStep + var step OpStep if region := oc.cluster.GetRegion(op.RegionID()); region != nil { if step = op.Check(region); step != nil { oc.SendScheduleCommand(region, step, DispatchFromCreate) @@ -511,57 +521,94 @@ func (oc *OperatorController) addOperatorLocked(op *operator.Operator) bool { return true } -// RemoveOperator removes a operator from the running operators. -func (oc *OperatorController) RemoveOperator(op *operator.Operator, extraFields ...zap.Field) bool { +func (oc *Controller) ack(op *Operator) { + opInfluence := NewTotalOpInfluence([]*Operator{op}, oc.cluster) + for storeID := range opInfluence.StoresInfluence { + for _, v := range storelimit.TypeNameValue { + limiter := oc.getOrCreateStoreLimit(storeID, v) + if limiter == nil { + return + } + cost := opInfluence.GetStoreInfluence(storeID).GetStepCost(v) + limiter.Ack(cost, v) + } + } +} + +// RemoveOperator removes an operator from the running operators. +func (oc *Controller) RemoveOperator(op *Operator, reasons ...CancelReasonType) bool { oc.Lock() removed := oc.removeOperatorLocked(op) oc.Unlock() + var cancelReason CancelReasonType + if len(reasons) > 0 { + cancelReason = reasons[0] + } else { + cancelReason = Unknown + } if removed { - if op.Cancel() { + if op.Cancel(cancelReason) { log.Info("operator removed", zap.Uint64("region-id", op.RegionID()), zap.Duration("takes", op.RunningTime()), zap.Reflect("operator", op)) } - oc.buryOperator(op, extraFields...) + oc.buryOperator(op) } return removed } -func (oc *OperatorController) removeOperatorWithoutBury(op *operator.Operator) bool { +func (oc *Controller) removeOperatorWithoutBury(op *Operator) bool { oc.Lock() defer oc.Unlock() return oc.removeOperatorLocked(op) } -func (oc *OperatorController) removeOperatorLocked(op *operator.Operator) bool { +func (oc *Controller) removeOperatorLocked(op *Operator) bool { regionID := op.RegionID() if cur := oc.operators[regionID]; cur == op { delete(oc.operators, regionID) oc.updateCounts(oc.operators) operatorCounter.WithLabelValues(op.Desc(), "remove").Inc() + oc.ack(op) + if op.Kind()&OpMerge != 0 { + oc.removeRelatedMergeOperator(op) + } return true } return false } -func (oc *OperatorController) buryOperator(op *operator.Operator, extraFields ...zap.Field) { +func (oc *Controller) removeRelatedMergeOperator(op *Operator) { + relatedID, _ := strconv.ParseUint(op.AdditionalInfos[string(RelatedMergeRegion)], 10, 64) + if relatedOp := oc.operators[relatedID]; relatedOp != nil && relatedOp.Status() != CANCELED { + log.Info("operator canceled related merge region", + zap.Uint64("region-id", relatedOp.RegionID()), + zap.String("additional-info", relatedOp.GetAdditionalInfo()), + zap.Duration("takes", relatedOp.RunningTime())) + oc.removeOperatorLocked(relatedOp) + relatedOp.Cancel(RelatedMergeRegion) + oc.buryOperator(relatedOp) + } +} + +func (oc *Controller) buryOperator(op *Operator) { st := op.Status() - if !operator.IsEndStatus(st) { + if !IsEndStatus(st) { log.Error("burying operator with non-end status", zap.Uint64("region-id", op.RegionID()), - zap.String("status", operator.OpStatusToString(op.Status())), + zap.String("status", OpStatusToString(op.Status())), zap.Reflect("operator", op), errs.ZapError(errs.ErrUnexpectedOperatorStatus)) failpoint.Inject("unexpectedOperator", func() { panic(op) }) operatorCounter.WithLabelValues(op.Desc(), "unexpected").Inc() - _ = op.Cancel() + _ = op.Cancel(Unknown) } switch st { - case operator.SUCCESS: + case SUCCESS: log.Info("operator finish", zap.Uint64("region-id", op.RegionID()), zap.Duration("takes", op.RunningTime()), @@ -572,66 +619,62 @@ func (oc *OperatorController) buryOperator(op *operator.Operator, extraFields .. for _, counter := range op.FinishedCounters { counter.Inc() } - case operator.REPLACED: + case REPLACED: log.Info("replace old operator", zap.Uint64("region-id", op.RegionID()), zap.Duration("takes", op.RunningTime()), zap.Reflect("operator", op), zap.String("additional-info", op.GetAdditionalInfo())) operatorCounter.WithLabelValues(op.Desc(), "replace").Inc() - case operator.EXPIRED: + case EXPIRED: log.Info("operator expired", zap.Uint64("region-id", op.RegionID()), zap.Duration("lives", op.ElapsedTime()), zap.Reflect("operator", op)) operatorCounter.WithLabelValues(op.Desc(), "expire").Inc() - case operator.TIMEOUT: + case TIMEOUT: log.Info("operator timeout", zap.Uint64("region-id", op.RegionID()), zap.Duration("takes", op.RunningTime()), zap.Reflect("operator", op), zap.String("additional-info", op.GetAdditionalInfo())) operatorCounter.WithLabelValues(op.Desc(), "timeout").Inc() - case operator.CANCELED: - fields := []zap.Field{ + case CANCELED: + log.Info("operator canceled", zap.Uint64("region-id", op.RegionID()), zap.Duration("takes", op.RunningTime()), zap.Reflect("operator", op), zap.String("additional-info", op.GetAdditionalInfo()), - } - fields = append(fields, extraFields...) - log.Info("operator canceled", - fields..., ) operatorCounter.WithLabelValues(op.Desc(), "cancel").Inc() } - oc.opRecords.Put(op) + oc.records.Put(op) } // GetOperatorStatus gets the operator and its status with the specify id. -func (oc *OperatorController) GetOperatorStatus(id uint64) *OperatorWithStatus { +func (oc *Controller) GetOperatorStatus(id uint64) *OpWithStatus { oc.Lock() defer oc.Unlock() if op, ok := oc.operators[id]; ok { - return NewOperatorWithStatus(op) + return NewOpWithStatus(op) } - return oc.opRecords.Get(id) + return oc.records.Get(id) } -// GetOperator gets a operator from the given region. -func (oc *OperatorController) GetOperator(regionID uint64) *operator.Operator { +// GetOperator gets an operator from the given region. +func (oc *Controller) GetOperator(regionID uint64) *Operator { oc.RLock() defer oc.RUnlock() return oc.operators[regionID] } // GetOperators gets operators from the running operators. -func (oc *OperatorController) GetOperators() []*operator.Operator { +func (oc *Controller) GetOperators() []*Operator { oc.RLock() defer oc.RUnlock() - operators := make([]*operator.Operator, 0, len(oc.operators)) + operators := make([]*Operator, 0, len(oc.operators)) for _, op := range oc.operators { operators = append(operators, op) } @@ -640,20 +683,20 @@ func (oc *OperatorController) GetOperators() []*operator.Operator { } // GetWaitingOperators gets operators from the waiting operators. -func (oc *OperatorController) GetWaitingOperators() []*operator.Operator { +func (oc *Controller) GetWaitingOperators() []*Operator { oc.RLock() defer oc.RUnlock() return oc.wop.ListOperator() } // SendScheduleCommand sends a command to the region. -func (oc *OperatorController) SendScheduleCommand(region *core.RegionInfo, step operator.OpStep, source string) { +func (oc *Controller) SendScheduleCommand(region *core.RegionInfo, step OpStep, source string) { log.Info("send schedule command", zap.Uint64("region-id", region.GetID()), zap.Stringer("step", step), zap.String("source", source)) - useConfChangeV2 := versioninfo.IsFeatureSupported(oc.cluster.GetOpts().GetClusterVersion(), versioninfo.ConfChangeV2) + useConfChangeV2 := versioninfo.IsFeatureSupported(oc.config.GetClusterVersion(), versioninfo.ConfChangeV2) cmd := step.GetCmd(region, useConfChangeV2) if cmd == nil { return @@ -661,15 +704,15 @@ func (oc *OperatorController) SendScheduleCommand(region *core.RegionInfo, step oc.hbStreams.SendMsg(region, cmd) } -func (oc *OperatorController) pushFastOperator(op *operator.Operator) { +func (oc *Controller) pushFastOperator(op *Operator) { oc.fastOperators.Put(op.RegionID(), op) } // GetRecords gets operators' records. -func (oc *OperatorController) GetRecords(from time.Time) []*operator.OpRecord { - records := make([]*operator.OpRecord, 0, oc.opRecords.ttl.Len()) - for _, id := range oc.opRecords.ttl.GetAllID() { - op := oc.opRecords.Get(id) +func (oc *Controller) GetRecords(from time.Time) []*OpRecord { + records := make([]*OpRecord, 0, oc.records.ttl.Len()) + for _, id := range oc.records.ttl.GetAllID() { + op := oc.records.Get(id) if op == nil || op.FinishTime.Before(from) { continue } @@ -679,10 +722,10 @@ func (oc *OperatorController) GetRecords(from time.Time) []*operator.OpRecord { } // GetHistory gets operators' history. -func (oc *OperatorController) GetHistory(start time.Time) []operator.OpHistory { - history := make([]operator.OpHistory, 0, oc.opRecords.ttl.Len()) - for _, id := range oc.opRecords.ttl.GetAllID() { - op := oc.opRecords.Get(id) +func (oc *Controller) GetHistory(start time.Time) []OpHistory { + history := make([]OpHistory, 0, oc.records.ttl.Len()) + for _, id := range oc.records.ttl.GetAllID() { + op := oc.records.Get(id) if op == nil || op.FinishTime.Before(start) { continue } @@ -692,7 +735,7 @@ func (oc *OperatorController) GetHistory(start time.Time) []operator.OpHistory { } // updateCounts updates resource counts using current pending operators. -func (oc *OperatorController) updateCounts(operators map[uint64]*operator.Operator) { +func (oc *Controller) updateCounts(operators map[uint64]*Operator) { for k := range oc.counts { delete(oc.counts, k) } @@ -703,16 +746,16 @@ func (oc *OperatorController) updateCounts(operators map[uint64]*operator.Operat // OperatorCount gets the count of operators filtered by kind. // kind only has one OpKind. -func (oc *OperatorController) OperatorCount(kind operator.OpKind) uint64 { +func (oc *Controller) OperatorCount(kind OpKind) uint64 { oc.RLock() defer oc.RUnlock() return oc.counts[kind] } // GetOpInfluence gets OpInfluence. -func (oc *OperatorController) GetOpInfluence(cluster Cluster) operator.OpInfluence { - influence := operator.OpInfluence{ - StoresInfluence: make(map[uint64]*operator.StoreInfluence), +func (oc *Controller) GetOpInfluence(cluster *core.BasicCluster) OpInfluence { + influence := OpInfluence{ + StoresInfluence: make(map[uint64]*StoreInfluence), } oc.RLock() defer oc.RUnlock() @@ -728,13 +771,13 @@ func (oc *OperatorController) GetOpInfluence(cluster Cluster) operator.OpInfluen } // GetFastOpInfluence get fast finish operator influence -func (oc *OperatorController) GetFastOpInfluence(cluster Cluster, influence operator.OpInfluence) { +func (oc *Controller) GetFastOpInfluence(cluster *core.BasicCluster, influence OpInfluence) { for _, id := range oc.fastOperators.GetAllID() { value, ok := oc.fastOperators.Get(id) if !ok { continue } - op, ok := value.(*operator.Operator) + op, ok := value.(*Operator) if !ok { continue } @@ -743,16 +786,14 @@ func (oc *OperatorController) GetFastOpInfluence(cluster Cluster, influence oper } // AddOpInfluence add operator influence for cluster -func AddOpInfluence(op *operator.Operator, influence operator.OpInfluence, cluster Cluster) { +func AddOpInfluence(op *Operator, influence OpInfluence, cluster *core.BasicCluster) { region := cluster.GetRegion(op.RegionID()) - if region != nil { - op.TotalInfluence(influence, region) - } + op.TotalInfluence(influence, region) } // NewTotalOpInfluence creates a OpInfluence. -func NewTotalOpInfluence(operators []*operator.Operator, cluster Cluster) operator.OpInfluence { - influence := *operator.NewOpInfluence() +func NewTotalOpInfluence(operators []*Operator, cluster *core.BasicCluster) OpInfluence { + influence := *NewOpInfluence() for _, op := range operators { AddOpInfluence(op, influence, cluster) @@ -762,73 +803,73 @@ func NewTotalOpInfluence(operators []*operator.Operator, cluster Cluster) operat } // SetOperator is only used for test. -func (oc *OperatorController) SetOperator(op *operator.Operator) { +func (oc *Controller) SetOperator(op *Operator) { oc.Lock() defer oc.Unlock() oc.operators[op.RegionID()] = op oc.updateCounts(oc.operators) } -// OperatorWithStatus records the operator and its status. -type OperatorWithStatus struct { - *operator.Operator +// OpWithStatus records the operator and its status. +type OpWithStatus struct { + *Operator Status pdpb.OperatorStatus FinishTime time.Time } -// NewOperatorWithStatus creates an OperatorStatus from an operator. -func NewOperatorWithStatus(op *operator.Operator) *OperatorWithStatus { - return &OperatorWithStatus{ +// NewOpWithStatus creates an OpWithStatus from an operator. +func NewOpWithStatus(op *Operator) *OpWithStatus { + return &OpWithStatus{ Operator: op, - Status: operator.OpStatusToPDPB(op.Status()), + Status: OpStatusToPDPB(op.Status()), FinishTime: time.Now(), } } // MarshalJSON returns the status of operator as a JSON string -func (o *OperatorWithStatus) MarshalJSON() ([]byte, error) { +func (o *OpWithStatus) MarshalJSON() ([]byte, error) { return []byte(`"` + fmt.Sprintf("status: %s, operator: %s", o.Status.String(), o.Operator.String()) + `"`), nil } -// OperatorRecords remains the operator and its status for a while. -type OperatorRecords struct { +// records remains the operator and its status for a while. +type records struct { ttl *cache.TTLUint64 } const operatorStatusRemainTime = 10 * time.Minute -// NewOperatorRecords returns a OperatorRecords. -func NewOperatorRecords(ctx context.Context) *OperatorRecords { - return &OperatorRecords{ +// newRecords returns a records. +func newRecords(ctx context.Context) *records { + return &records{ ttl: cache.NewIDTTL(ctx, time.Minute, operatorStatusRemainTime), } } // Get gets the operator and its status. -func (o *OperatorRecords) Get(id uint64) *OperatorWithStatus { +func (o *records) Get(id uint64) *OpWithStatus { v, exist := o.ttl.Get(id) if !exist { return nil } - return v.(*OperatorWithStatus) + return v.(*OpWithStatus) } // Put puts the operator and its status. -func (o *OperatorRecords) Put(op *operator.Operator) { +func (o *records) Put(op *Operator) { id := op.RegionID() - record := NewOperatorWithStatus(op) + record := NewOpWithStatus(op) o.ttl.Put(id, record) } -// ExceedStoreLimit returns true if the store exceeds the cost limit after adding the operator. Otherwise, returns false. -func (oc *OperatorController) ExceedStoreLimit(ops ...*operator.Operator) bool { +// ExceedStoreLimit returns true if the store exceeds the cost limit after adding the Otherwise, returns false. +func (oc *Controller) ExceedStoreLimit(ops ...*Operator) bool { oc.Lock() defer oc.Unlock() return oc.exceedStoreLimitLocked(ops...) } -// exceedStoreLimitLocked returns true if the store exceeds the cost limit after adding the operator. Otherwise, returns false. -func (oc *OperatorController) exceedStoreLimitLocked(ops ...*operator.Operator) bool { +// exceedStoreLimitLocked returns true if the store exceeds the cost limit after adding the Otherwise, returns false. +func (oc *Controller) exceedStoreLimitLocked(ops ...*Operator) bool { // The operator with Urgent priority, like admin operators, should ignore the store limit check. var desc string if len(ops) != 0 { @@ -849,7 +890,7 @@ func (oc *OperatorController) exceedStoreLimitLocked(ops ...*operator.Operator) return false } if !limiter.Available(stepCost, v, ops[0].GetPriorityLevel()) { - operator.OperatorExceededStoreLimitCounter.WithLabelValues(desc).Inc() + OperatorExceededStoreLimitCounter.WithLabelValues(desc).Inc() return true } } @@ -858,15 +899,16 @@ func (oc *OperatorController) exceedStoreLimitLocked(ops ...*operator.Operator) } // getOrCreateStoreLimit is used to get or create the limit of a store. -func (oc *OperatorController) getOrCreateStoreLimit(storeID uint64, limitType storelimit.Type) storelimit.StoreLimit { - ratePerSec := oc.cluster.GetOpts().GetStoreLimitByType(storeID, limitType) / StoreBalanceBaseTime +func (oc *Controller) getOrCreateStoreLimit(storeID uint64, limitType storelimit.Type) storelimit.StoreLimit { + ratePerSec := oc.config.GetStoreLimitByType(storeID, limitType) / StoreBalanceBaseTime s := oc.cluster.GetStore(storeID) if s == nil { log.Error("invalid store ID", zap.Uint64("store-id", storeID)) return nil } - - limit := s.GetStoreLimit() - limit.Reset(ratePerSec, limitType) - return limit + // The other limits do not need to update by config exclude StoreRateLimit. + if limit, ok := s.GetStoreLimit().(*storelimit.StoreRateLimit); ok && limit.Rate(limitType) != ratePerSec { + oc.cluster.ResetStoreLimit(storeID, limitType, ratePerSec) + } + return s.GetStoreLimit() } diff --git a/pkg/schedule/operator_controller_test.go b/pkg/schedule/operator/operator_controller_test.go similarity index 70% rename from pkg/schedule/operator_controller_test.go rename to pkg/schedule/operator/operator_controller_test.go index ca15de2cc68..6d5f835ca68 100644 --- a/pkg/schedule/operator_controller_test.go +++ b/pkg/schedule/operator/operator_controller_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package schedule +package operator import ( "container/heap" @@ -33,7 +33,6 @@ import ( "github.com/tikv/pd/pkg/mock/mockconfig" "github.com/tikv/pd/pkg/schedule/hbstream" "github.com/tikv/pd/pkg/schedule/labeler" - "github.com/tikv/pd/pkg/schedule/operator" ) type operatorControllerTestSuite struct { @@ -49,26 +48,58 @@ func TestOperatorControllerTestSuite(t *testing.T) { func (suite *operatorControllerTestSuite) SetupSuite() { suite.ctx, suite.cancel = context.WithCancel(context.Background()) - suite.NoError(failpoint.Enable("github.com/tikv/pd/pkg/schedule/unexpectedOperator", "return(true)")) + suite.NoError(failpoint.Enable("github.com/tikv/pd/pkg/schedule/operator/unexpectedOperator", "return(true)")) } func (suite *operatorControllerTestSuite) TearDownSuite() { suite.cancel() } +func (suite *operatorControllerTestSuite) TestCacheInfluence() { + opt := mockconfig.NewTestOptions() + tc := mockcluster.NewCluster(suite.ctx, opt) + bc := tc.GetBasicCluster() + oc := NewController(suite.ctx, bc, tc.GetSharedConfig(), nil) + tc.AddLeaderStore(2, 1) + region := tc.AddLeaderRegion(1, 1, 2) + + steps := []OpStep{ + RemovePeer{FromStore: 2}, + } + op := NewTestOperator(1, &metapb.RegionEpoch{}, OpRegion, steps...) + oc.SetOperator(op) + suite.True(op.Start()) + influence := NewOpInfluence() + AddOpInfluence(op, *influence, bc) + suite.Equal(int64(-96), influence.GetStoreInfluence(2).RegionSize) + + // case: influence is same even if the region size changed. + region = region.Clone(core.SetApproximateSize(100)) + tc.PutRegion(region) + influence1 := NewOpInfluence() + AddOpInfluence(op, *influence1, bc) + suite.Equal(int64(-96), influence1.GetStoreInfluence(2).RegionSize) + + // case: influence is valid even if the region is removed. + tc.RemoveRegion(region) + influence2 := NewOpInfluence() + AddOpInfluence(op, *influence2, bc) + suite.Equal(int64(-96), influence2.GetStoreInfluence(2).RegionSize) +} + // issue #1338 func (suite *operatorControllerTestSuite) TestGetOpInfluence() { opt := mockconfig.NewTestOptions() tc := mockcluster.NewCluster(suite.ctx, opt) - oc := NewOperatorController(suite.ctx, tc, nil) + oc := NewController(suite.ctx, tc.GetBasicCluster(), tc.GetSharedConfig(), nil) tc.AddLeaderStore(2, 1) tc.AddLeaderRegion(1, 1, 2) tc.AddLeaderRegion(2, 1, 2) - steps := []operator.OpStep{ - operator.RemovePeer{FromStore: 2}, + steps := []OpStep{ + RemovePeer{FromStore: 2}, } - op1 := operator.NewTestOperator(1, &metapb.RegionEpoch{}, operator.OpRegion, steps...) - op2 := operator.NewTestOperator(2, &metapb.RegionEpoch{}, operator.OpRegion, steps...) + op1 := NewTestOperator(1, &metapb.RegionEpoch{}, OpRegion, steps...) + op2 := NewTestOperator(2, &metapb.RegionEpoch{}, OpRegion, steps...) suite.True(op1.Start()) oc.SetOperator(op1) suite.True(op2.Start()) @@ -91,7 +122,7 @@ func (suite *operatorControllerTestSuite) TestGetOpInfluence() { case <-ctx.Done(): return default: - oc.GetOpInfluence(tc) + oc.GetOpInfluence(tc.GetBasicCluster()) } } }(suite.ctx) @@ -103,34 +134,34 @@ func (suite *operatorControllerTestSuite) TestOperatorStatus() { opt := mockconfig.NewTestOptions() tc := mockcluster.NewCluster(suite.ctx, opt) stream := hbstream.NewTestHeartbeatStreams(suite.ctx, tc.ID, tc, false /* no need to run */) - oc := NewOperatorController(suite.ctx, tc, stream) + oc := NewController(suite.ctx, tc.GetBasicCluster(), tc.GetSharedConfig(), stream) tc.AddLeaderStore(1, 2) tc.AddLeaderStore(2, 0) tc.AddLeaderRegion(1, 1, 2) tc.AddLeaderRegion(2, 1, 2) - steps := []operator.OpStep{ - operator.RemovePeer{FromStore: 2}, - operator.AddPeer{ToStore: 2, PeerID: 4}, + steps := []OpStep{ + RemovePeer{FromStore: 2}, + AddPeer{ToStore: 2, PeerID: 4}, } region1 := tc.GetRegion(1) region2 := tc.GetRegion(2) - op1 := operator.NewTestOperator(1, &metapb.RegionEpoch{}, operator.OpRegion, steps...) - op2 := operator.NewTestOperator(2, &metapb.RegionEpoch{}, operator.OpRegion, steps...) + op1 := NewTestOperator(1, &metapb.RegionEpoch{}, OpRegion, steps...) + op2 := NewTestOperator(2, &metapb.RegionEpoch{}, OpRegion, steps...) suite.True(op1.Start()) oc.SetOperator(op1) suite.True(op2.Start()) oc.SetOperator(op2) suite.Equal(pdpb.OperatorStatus_RUNNING, oc.GetOperatorStatus(1).Status) suite.Equal(pdpb.OperatorStatus_RUNNING, oc.GetOperatorStatus(2).Status) - op1.SetStatusReachTime(operator.STARTED, time.Now().Add(-operator.SlowStepWaitTime-operator.FastStepWaitTime)) + op1.SetStatusReachTime(STARTED, time.Now().Add(-SlowStepWaitTime-FastStepWaitTime)) region2 = ApplyOperatorStep(region2, op2) tc.PutRegion(region2) - oc.Dispatch(region1, "test") - oc.Dispatch(region2, "test") + oc.Dispatch(region1, "test", nil) + oc.Dispatch(region2, "test", nil) suite.Equal(pdpb.OperatorStatus_TIMEOUT, oc.GetOperatorStatus(1).Status) suite.Equal(pdpb.OperatorStatus_RUNNING, oc.GetOperatorStatus(2).Status) ApplyOperator(tc, op2) - oc.Dispatch(region2, "test") + oc.Dispatch(region2, "test", nil) suite.Equal(pdpb.OperatorStatus_SUCCESS, oc.GetOperatorStatus(2).Status) } @@ -138,32 +169,32 @@ func (suite *operatorControllerTestSuite) TestFastFailOperator() { opt := mockconfig.NewTestOptions() tc := mockcluster.NewCluster(suite.ctx, opt) stream := hbstream.NewTestHeartbeatStreams(suite.ctx, tc.ID, tc, false /* no need to run */) - oc := NewOperatorController(suite.ctx, tc, stream) + oc := NewController(suite.ctx, tc.GetBasicCluster(), tc.GetSharedConfig(), stream) tc.AddLeaderStore(1, 2) tc.AddLeaderStore(2, 0) tc.AddLeaderStore(3, 0) tc.AddLeaderRegion(1, 1, 2) - steps := []operator.OpStep{ - operator.RemovePeer{FromStore: 2}, - operator.AddPeer{ToStore: 3, PeerID: 4}, + steps := []OpStep{ + RemovePeer{FromStore: 2}, + AddPeer{ToStore: 3, PeerID: 4}, } region := tc.GetRegion(1) - op := operator.NewTestOperator(1, &metapb.RegionEpoch{}, operator.OpRegion, steps...) + op := NewTestOperator(1, &metapb.RegionEpoch{}, OpRegion, steps...) suite.True(op.Start()) oc.SetOperator(op) - oc.Dispatch(region, "test") + oc.Dispatch(region, "test", nil) suite.Equal(pdpb.OperatorStatus_RUNNING, oc.GetOperatorStatus(1).Status) // change the leader region = region.Clone(core.WithLeader(region.GetPeer(2))) - oc.Dispatch(region, DispatchFromHeartBeat) - suite.Equal(operator.CANCELED, op.Status()) + oc.Dispatch(region, DispatchFromHeartBeat, nil) + suite.Equal(CANCELED, op.Status()) suite.Nil(oc.GetOperator(region.GetID())) // transfer leader to an illegal store. - op = operator.NewTestOperator(1, &metapb.RegionEpoch{}, operator.OpRegion, operator.TransferLeader{ToStore: 5}) + op = NewTestOperator(1, &metapb.RegionEpoch{}, OpRegion, TransferLeader{ToStore: 5}) oc.SetOperator(op) - oc.Dispatch(region, DispatchFromHeartBeat) - suite.Equal(operator.CANCELED, op.Status()) + oc.Dispatch(region, DispatchFromHeartBeat, nil) + suite.Equal(CANCELED, op.Status()) suite.Nil(oc.GetOperator(region.GetID())) } @@ -172,14 +203,14 @@ func (suite *operatorControllerTestSuite) TestFastFailWithUnhealthyStore() { opt := mockconfig.NewTestOptions() tc := mockcluster.NewCluster(suite.ctx, opt) stream := hbstream.NewTestHeartbeatStreams(suite.ctx, tc.ID, tc, false /* no need to run */) - oc := NewOperatorController(suite.ctx, tc, stream) + oc := NewController(suite.ctx, tc.GetBasicCluster(), tc.GetSharedConfig(), stream) tc.AddLeaderStore(1, 2) tc.AddLeaderStore(2, 0) tc.AddLeaderStore(3, 0) tc.AddLeaderRegion(1, 1, 2) region := tc.GetRegion(1) - steps := []operator.OpStep{operator.TransferLeader{ToStore: 2}} - op := operator.NewTestOperator(1, region.GetRegionEpoch(), operator.OpLeader, steps...) + steps := []OpStep{TransferLeader{ToStore: 2}} + op := NewTestOperator(1, region.GetRegionEpoch(), OpLeader, steps...) oc.SetOperator(op) suite.False(oc.checkStaleOperator(op, steps[0], region)) tc.SetStoreDown(2) @@ -187,42 +218,42 @@ func (suite *operatorControllerTestSuite) TestFastFailWithUnhealthyStore() { } func (suite *operatorControllerTestSuite) TestCheckAddUnexpectedStatus() { - suite.NoError(failpoint.Disable("github.com/tikv/pd/pkg/schedule/unexpectedOperator")) + suite.NoError(failpoint.Disable("github.com/tikv/pd/pkg/schedule/operator/unexpectedOperator")) opt := mockconfig.NewTestOptions() tc := mockcluster.NewCluster(suite.ctx, opt) stream := hbstream.NewTestHeartbeatStreams(suite.ctx, tc.ID, tc, false /* no need to run */) - oc := NewOperatorController(suite.ctx, tc, stream) + oc := NewController(suite.ctx, tc.GetBasicCluster(), tc.GetSharedConfig(), stream) tc.AddLeaderStore(1, 0) tc.AddLeaderStore(2, 1) tc.AddLeaderRegion(1, 2, 1) tc.AddLeaderRegion(2, 2, 1) region1 := tc.GetRegion(1) - steps := []operator.OpStep{ - operator.RemovePeer{FromStore: 1}, - operator.AddPeer{ToStore: 1, PeerID: 4}, + steps := []OpStep{ + RemovePeer{FromStore: 1}, + AddPeer{ToStore: 1, PeerID: 4}, } { // finished op - op := operator.NewTestOperator(1, &metapb.RegionEpoch{}, operator.OpRegion, operator.TransferLeader{ToStore: 2}) + op := NewTestOperator(1, &metapb.RegionEpoch{}, OpRegion, TransferLeader{ToStore: 2}) suite.True(oc.checkAddOperator(false, op)) op.Start() suite.False(oc.checkAddOperator(false, op)) // started suite.Nil(op.Check(region1)) - suite.Equal(operator.SUCCESS, op.Status()) + suite.Equal(SUCCESS, op.Status()) suite.False(oc.checkAddOperator(false, op)) // success } { // finished op canceled - op := operator.NewTestOperator(1, &metapb.RegionEpoch{}, operator.OpRegion, operator.TransferLeader{ToStore: 2}) + op := NewTestOperator(1, &metapb.RegionEpoch{}, OpRegion, TransferLeader{ToStore: 2}) suite.True(oc.checkAddOperator(false, op)) - suite.True(op.Cancel()) + suite.True(op.Cancel(AdminStop)) suite.False(oc.checkAddOperator(false, op)) } { // finished op replaced - op := operator.NewTestOperator(1, &metapb.RegionEpoch{}, operator.OpRegion, operator.TransferLeader{ToStore: 2}) + op := NewTestOperator(1, &metapb.RegionEpoch{}, OpRegion, TransferLeader{ToStore: 2}) suite.True(oc.checkAddOperator(false, op)) suite.True(op.Start()) suite.True(op.Replace()) @@ -230,23 +261,23 @@ func (suite *operatorControllerTestSuite) TestCheckAddUnexpectedStatus() { } { // finished op expired - op1 := operator.NewTestOperator(1, &metapb.RegionEpoch{}, operator.OpRegion, operator.TransferLeader{ToStore: 2}) - op2 := operator.NewTestOperator(2, &metapb.RegionEpoch{}, operator.OpRegion, operator.TransferLeader{ToStore: 1}) + op1 := NewTestOperator(1, &metapb.RegionEpoch{}, OpRegion, TransferLeader{ToStore: 2}) + op2 := NewTestOperator(2, &metapb.RegionEpoch{}, OpRegion, TransferLeader{ToStore: 1}) suite.True(oc.checkAddOperator(false, op1, op2)) - op1.SetStatusReachTime(operator.CREATED, time.Now().Add(-operator.OperatorExpireTime)) - op2.SetStatusReachTime(operator.CREATED, time.Now().Add(-operator.OperatorExpireTime)) + op1.SetStatusReachTime(CREATED, time.Now().Add(-OperatorExpireTime)) + op2.SetStatusReachTime(CREATED, time.Now().Add(-OperatorExpireTime)) suite.False(oc.checkAddOperator(false, op1, op2)) - suite.Equal(operator.EXPIRED, op1.Status()) - suite.Equal(operator.EXPIRED, op2.Status()) + suite.Equal(EXPIRED, op1.Status()) + suite.Equal(EXPIRED, op2.Status()) } // finished op never timeout { // unfinished op timeout - op := operator.NewTestOperator(1, &metapb.RegionEpoch{}, operator.OpRegion, steps...) + op := NewTestOperator(1, &metapb.RegionEpoch{}, OpRegion, steps...) suite.True(oc.checkAddOperator(false, op)) op.Start() - op.SetStatusReachTime(operator.STARTED, time.Now().Add(-operator.SlowStepWaitTime-operator.FastStepWaitTime)) + op.SetStatusReachTime(STARTED, time.Now().Add(-SlowStepWaitTime-FastStepWaitTime)) suite.True(op.CheckTimeout()) suite.False(oc.checkAddOperator(false, op)) } @@ -257,29 +288,28 @@ func (suite *operatorControllerTestSuite) TestConcurrentRemoveOperator() { opt := mockconfig.NewTestOptions() tc := mockcluster.NewCluster(suite.ctx, opt) stream := hbstream.NewTestHeartbeatStreams(suite.ctx, tc.ID, tc, false /* no need to run */) - oc := NewOperatorController(suite.ctx, tc, stream) + oc := NewController(suite.ctx, tc.GetBasicCluster(), tc.GetSharedConfig(), stream) tc.AddLeaderStore(1, 0) tc.AddLeaderStore(2, 1) tc.AddLeaderRegion(1, 2, 1) region1 := tc.GetRegion(1) - steps := []operator.OpStep{ - operator.RemovePeer{FromStore: 1}, - operator.AddPeer{ToStore: 1, PeerID: 4}, + steps := []OpStep{ + RemovePeer{FromStore: 1}, + AddPeer{ToStore: 1, PeerID: 4}, } // finished op with normal priority - op1 := operator.NewTestOperator(1, &metapb.RegionEpoch{}, operator.OpRegion, operator.TransferLeader{ToStore: 2}) + op1 := NewTestOperator(1, &metapb.RegionEpoch{}, OpRegion, TransferLeader{ToStore: 2}) // unfinished op with high priority - op2 := operator.NewTestOperator(1, &metapb.RegionEpoch{}, operator.OpRegion|operator.OpAdmin, steps...) + op2 := NewTestOperator(1, &metapb.RegionEpoch{}, OpRegion|OpAdmin, steps...) suite.True(op1.Start()) oc.SetOperator(op1) - suite.NoError(failpoint.Enable("github.com/tikv/pd/pkg/schedule/concurrentRemoveOperator", "return(true)")) - + suite.NoError(failpoint.Enable("github.com/tikv/pd/pkg/schedule/operator/concurrentRemoveOperator", "return(true)")) var wg sync.WaitGroup wg.Add(2) go func() { - oc.Dispatch(region1, "test") + oc.Dispatch(region1, "test", nil) wg.Done() }() go func() { @@ -292,26 +322,27 @@ func (suite *operatorControllerTestSuite) TestConcurrentRemoveOperator() { wg.Wait() suite.Equal(op2, oc.GetOperator(1)) + suite.NoError(failpoint.Disable("github.com/tikv/pd/pkg/schedule/operator/concurrentRemoveOperator")) } func (suite *operatorControllerTestSuite) TestPollDispatchRegion() { opt := mockconfig.NewTestOptions() tc := mockcluster.NewCluster(suite.ctx, opt) stream := hbstream.NewTestHeartbeatStreams(suite.ctx, tc.ID, tc, false /* no need to run */) - oc := NewOperatorController(suite.ctx, tc, stream) + oc := NewController(suite.ctx, tc.GetBasicCluster(), tc.GetSharedConfig(), stream) tc.AddLeaderStore(1, 2) tc.AddLeaderStore(2, 1) tc.AddLeaderRegion(1, 1, 2) tc.AddLeaderRegion(2, 1, 2) tc.AddLeaderRegion(4, 2, 1) - steps := []operator.OpStep{ - operator.RemovePeer{FromStore: 2}, - operator.AddPeer{ToStore: 2, PeerID: 4}, + steps := []OpStep{ + RemovePeer{FromStore: 2}, + AddPeer{ToStore: 2, PeerID: 4}, } - op1 := operator.NewTestOperator(1, &metapb.RegionEpoch{}, operator.OpRegion, operator.TransferLeader{ToStore: 2}) - op2 := operator.NewTestOperator(2, &metapb.RegionEpoch{}, operator.OpRegion, steps...) - op3 := operator.NewTestOperator(3, &metapb.RegionEpoch{}, operator.OpRegion, steps...) - op4 := operator.NewTestOperator(4, &metapb.RegionEpoch{}, operator.OpRegion, operator.TransferLeader{ToStore: 2}) + op1 := NewTestOperator(1, &metapb.RegionEpoch{}, OpRegion, TransferLeader{ToStore: 2}) + op2 := NewTestOperator(2, &metapb.RegionEpoch{}, OpRegion, steps...) + op3 := NewTestOperator(3, &metapb.RegionEpoch{}, OpRegion, steps...) + op4 := NewTestOperator(4, &metapb.RegionEpoch{}, OpRegion, TransferLeader{ToStore: 2}) region1 := tc.GetRegion(1) region2 := tc.GetRegion(2) region4 := tc.GetRegion(4) @@ -371,7 +402,7 @@ func (suite *operatorControllerTestSuite) TestStoreLimit() { opt := mockconfig.NewTestOptions() tc := mockcluster.NewCluster(suite.ctx, opt) stream := hbstream.NewTestHeartbeatStreams(suite.ctx, tc.ID, tc, false /* no need to run */) - oc := NewOperatorController(suite.ctx, tc, stream) + oc := NewController(suite.ctx, tc.GetBasicCluster(), tc.GetSharedConfig(), stream) tc.AddLeaderStore(1, 0) tc.UpdateLeaderCount(1, 1000) tc.AddLeaderStore(2, 0) @@ -383,53 +414,53 @@ func (suite *operatorControllerTestSuite) TestStoreLimit() { tc.SetStoreLimit(2, storelimit.AddPeer, 60) for i := uint64(1); i <= 5; i++ { - op := operator.NewTestOperator(1, &metapb.RegionEpoch{}, operator.OpRegion, operator.AddPeer{ToStore: 2, PeerID: i}) + op := NewTestOperator(1, &metapb.RegionEpoch{}, OpRegion, AddPeer{ToStore: 2, PeerID: i}) suite.True(oc.AddOperator(op)) suite.checkRemoveOperatorSuccess(oc, op) } - op := operator.NewTestOperator(1, &metapb.RegionEpoch{}, operator.OpRegion, operator.AddPeer{ToStore: 2, PeerID: 1}) + op := NewTestOperator(1, &metapb.RegionEpoch{}, OpRegion, AddPeer{ToStore: 2, PeerID: 1}) suite.False(oc.AddOperator(op)) suite.False(oc.RemoveOperator(op)) tc.SetStoreLimit(2, storelimit.AddPeer, 120) for i := uint64(1); i <= 10; i++ { - op = operator.NewTestOperator(i, &metapb.RegionEpoch{}, operator.OpRegion, operator.AddPeer{ToStore: 2, PeerID: i}) + op = NewTestOperator(i, &metapb.RegionEpoch{}, OpRegion, AddPeer{ToStore: 2, PeerID: i}) suite.True(oc.AddOperator(op)) suite.checkRemoveOperatorSuccess(oc, op) } tc.SetAllStoresLimit(storelimit.AddPeer, 60) for i := uint64(1); i <= 5; i++ { - op = operator.NewTestOperator(i, &metapb.RegionEpoch{}, operator.OpRegion, operator.AddPeer{ToStore: 2, PeerID: i}) + op = NewTestOperator(i, &metapb.RegionEpoch{}, OpRegion, AddPeer{ToStore: 2, PeerID: i}) suite.True(oc.AddOperator(op)) suite.checkRemoveOperatorSuccess(oc, op) } - op = operator.NewTestOperator(1, &metapb.RegionEpoch{}, operator.OpRegion, operator.AddPeer{ToStore: 2, PeerID: 1}) + op = NewTestOperator(1, &metapb.RegionEpoch{}, OpRegion, AddPeer{ToStore: 2, PeerID: 1}) suite.False(oc.AddOperator(op)) suite.False(oc.RemoveOperator(op)) tc.SetStoreLimit(2, storelimit.RemovePeer, 60) for i := uint64(1); i <= 5; i++ { - op := operator.NewTestOperator(1, &metapb.RegionEpoch{}, operator.OpRegion, operator.RemovePeer{FromStore: 2}) + op := NewTestOperator(1, &metapb.RegionEpoch{}, OpRegion, RemovePeer{FromStore: 2}) suite.True(oc.AddOperator(op)) suite.checkRemoveOperatorSuccess(oc, op) } - op = operator.NewTestOperator(1, &metapb.RegionEpoch{}, operator.OpRegion, operator.RemovePeer{FromStore: 2}) + op = NewTestOperator(1, &metapb.RegionEpoch{}, OpRegion, RemovePeer{FromStore: 2}) suite.False(oc.AddOperator(op)) suite.False(oc.RemoveOperator(op)) tc.SetStoreLimit(2, storelimit.RemovePeer, 120) for i := uint64(1); i <= 10; i++ { - op = operator.NewTestOperator(i, &metapb.RegionEpoch{}, operator.OpRegion, operator.RemovePeer{FromStore: 2}) + op = NewTestOperator(i, &metapb.RegionEpoch{}, OpRegion, RemovePeer{FromStore: 2}) suite.True(oc.AddOperator(op)) suite.checkRemoveOperatorSuccess(oc, op) } tc.SetAllStoresLimit(storelimit.RemovePeer, 60) for i := uint64(1); i <= 5; i++ { - op = operator.NewTestOperator(i, &metapb.RegionEpoch{}, operator.OpRegion, operator.RemovePeer{FromStore: 2}) + op = NewTestOperator(i, &metapb.RegionEpoch{}, OpRegion, RemovePeer{FromStore: 2}) suite.True(oc.AddOperator(op)) suite.checkRemoveOperatorSuccess(oc, op) } - op = operator.NewTestOperator(1, &metapb.RegionEpoch{}, operator.OpRegion, operator.RemovePeer{FromStore: 2}) + op = NewTestOperator(1, &metapb.RegionEpoch{}, OpRegion, RemovePeer{FromStore: 2}) suite.False(oc.AddOperator(op)) suite.False(oc.RemoveOperator(op)) } @@ -438,18 +469,18 @@ func (suite *operatorControllerTestSuite) TestStoreLimit() { func (suite *operatorControllerTestSuite) TestDispatchOutdatedRegion() { cluster := mockcluster.NewCluster(suite.ctx, mockconfig.NewTestOptions()) stream := hbstream.NewTestHeartbeatStreams(suite.ctx, cluster.ID, cluster, false /* no need to run */) - controller := NewOperatorController(suite.ctx, cluster, stream) + controller := NewController(suite.ctx, cluster.GetBasicCluster(), cluster.GetSharedConfig(), stream) cluster.AddLeaderStore(1, 2) cluster.AddLeaderStore(2, 0) cluster.SetAllStoresLimit(storelimit.RemovePeer, 600) cluster.AddLeaderRegion(1, 1, 2) - steps := []operator.OpStep{ - operator.TransferLeader{FromStore: 1, ToStore: 2}, - operator.RemovePeer{FromStore: 1}, + steps := []OpStep{ + TransferLeader{FromStore: 1, ToStore: 2}, + RemovePeer{FromStore: 1}, } - op := operator.NewTestOperator(1, &metapb.RegionEpoch{ConfVer: 0, Version: 0}, operator.OpRegion, steps...) + op := NewTestOperator(1, &metapb.RegionEpoch{ConfVer: 0, Version: 0}, OpRegion, steps...) suite.True(controller.AddOperator(op)) suite.Equal(1, stream.MsgLength()) @@ -457,7 +488,7 @@ func (suite *operatorControllerTestSuite) TestDispatchOutdatedRegion() { region := cluster.MockRegionInfo(1, 2, []uint64{1, 2}, []uint64{}, &metapb.RegionEpoch{ConfVer: 0, Version: 0}) - controller.Dispatch(region, DispatchFromHeartBeat) + controller.Dispatch(region, DispatchFromHeartBeat, nil) suite.Equal(uint64(0), op.ConfVerChanged(region)) suite.Equal(2, stream.MsgLength()) @@ -465,13 +496,13 @@ func (suite *operatorControllerTestSuite) TestDispatchOutdatedRegion() { region = cluster.MockRegionInfo(1, 2, []uint64{2}, []uint64{}, &metapb.RegionEpoch{ConfVer: 0, Version: 0}) - controller.Dispatch(region, DispatchFromHeartBeat) + controller.Dispatch(region, DispatchFromHeartBeat, nil) suite.Equal(uint64(1), op.ConfVerChanged(region)) suite.Equal(2, stream.MsgLength()) // add and dispatch op again, the op should be stale - op = operator.NewTestOperator(1, &metapb.RegionEpoch{ConfVer: 0, Version: 0}, - operator.OpRegion, steps...) + op = NewTestOperator(1, &metapb.RegionEpoch{ConfVer: 0, Version: 0}, + OpRegion, steps...) suite.True(controller.AddOperator(op)) suite.Equal(uint64(0), op.ConfVerChanged(region)) suite.Equal(3, stream.MsgLength()) @@ -479,7 +510,7 @@ func (suite *operatorControllerTestSuite) TestDispatchOutdatedRegion() { // report region with an abnormal confver region = cluster.MockRegionInfo(1, 1, []uint64{1, 2}, []uint64{}, &metapb.RegionEpoch{ConfVer: 1, Version: 0}) - controller.Dispatch(region, DispatchFromHeartBeat) + controller.Dispatch(region, DispatchFromHeartBeat, nil) suite.Equal(uint64(0), op.ConfVerChanged(region)) // no new step suite.Equal(3, stream.MsgLength()) @@ -488,7 +519,7 @@ func (suite *operatorControllerTestSuite) TestDispatchOutdatedRegion() { func (suite *operatorControllerTestSuite) TestCalcInfluence() { cluster := mockcluster.NewCluster(suite.ctx, mockconfig.NewTestOptions()) stream := hbstream.NewTestHeartbeatStreams(suite.ctx, cluster.ID, cluster, false /* no need to run */) - controller := NewOperatorController(suite.ctx, cluster, stream) + controller := NewController(suite.ctx, cluster.GetBasicCluster(), cluster.GetSharedConfig(), stream) epoch := &metapb.RegionEpoch{ConfVer: 0, Version: 0} region := cluster.MockRegionInfo(1, 1, []uint64{2}, []uint64{}, epoch) @@ -497,16 +528,16 @@ func (suite *operatorControllerTestSuite) TestCalcInfluence() { cluster.AddRegionStore(1, 1) cluster.AddRegionStore(3, 1) - steps := []operator.OpStep{ - operator.AddLearner{ToStore: 3, PeerID: 3}, - operator.PromoteLearner{ToStore: 3, PeerID: 3}, - operator.TransferLeader{FromStore: 1, ToStore: 3}, - operator.RemovePeer{FromStore: 1}, + steps := []OpStep{ + AddLearner{ToStore: 3, PeerID: 3}, + PromoteLearner{ToStore: 3, PeerID: 3}, + TransferLeader{FromStore: 1, ToStore: 3}, + RemovePeer{FromStore: 1}, } - op := operator.NewTestOperator(1, epoch, operator.OpRegion, steps...) + op := NewTestOperator(1, epoch, OpRegion, steps...) suite.True(controller.AddOperator(op)) - check := func(influence operator.OpInfluence, id uint64, expect *operator.StoreInfluence) { + check := func(influence OpInfluence, id uint64, expect *StoreInfluence) { si := influence.GetStoreInfluence(id) suite.Equal(si.LeaderCount, expect.LeaderCount) suite.Equal(si.LeaderSize, expect.LeaderSize) @@ -516,8 +547,8 @@ func (suite *operatorControllerTestSuite) TestCalcInfluence() { suite.Equal(si.StepCost[storelimit.RemovePeer], expect.StepCost[storelimit.RemovePeer]) } - influence := controller.GetOpInfluence(cluster) - check(influence, 1, &operator.StoreInfluence{ + influence := controller.GetOpInfluence(cluster.GetBasicCluster()) + check(influence, 1, &StoreInfluence{ LeaderSize: -20, LeaderCount: -1, RegionSize: -20, @@ -526,7 +557,7 @@ func (suite *operatorControllerTestSuite) TestCalcInfluence() { storelimit.RemovePeer: 200, }, }) - check(influence, 3, &operator.StoreInfluence{ + check(influence, 3, &StoreInfluence{ LeaderSize: 20, LeaderCount: 1, RegionSize: 20, @@ -543,8 +574,8 @@ func (suite *operatorControllerTestSuite) TestCalcInfluence() { suite.True(steps[0].IsFinish(region2)) op.Check(region2) - influence = controller.GetOpInfluence(cluster) - check(influence, 1, &operator.StoreInfluence{ + influence = controller.GetOpInfluence(cluster.GetBasicCluster()) + check(influence, 1, &StoreInfluence{ LeaderSize: -20, LeaderCount: -1, RegionSize: -20, @@ -553,7 +584,7 @@ func (suite *operatorControllerTestSuite) TestCalcInfluence() { storelimit.RemovePeer: 200, }, }) - check(influence, 3, &operator.StoreInfluence{ + check(influence, 3, &StoreInfluence{ LeaderSize: 20, LeaderCount: 1, RegionSize: 0, @@ -565,7 +596,7 @@ func (suite *operatorControllerTestSuite) TestCalcInfluence() { func (suite *operatorControllerTestSuite) TestDispatchUnfinishedStep() { cluster := mockcluster.NewCluster(suite.ctx, mockconfig.NewTestOptions()) stream := hbstream.NewTestHeartbeatStreams(suite.ctx, cluster.ID, cluster, false /* no need to run */) - controller := NewOperatorController(suite.ctx, cluster, stream) + controller := NewController(suite.ctx, cluster.GetBasicCluster(), cluster.GetSharedConfig(), stream) // Create a new region with epoch(0, 0) // the region has two peers with its peer id allocated incrementally. @@ -580,24 +611,24 @@ func (suite *operatorControllerTestSuite) TestDispatchUnfinishedStep() { cluster.AddRegionStore(3, 1) // The next allocated peer should have peerid 3, so we add this peer // to store 3 - testSteps := [][]operator.OpStep{ + testSteps := [][]OpStep{ { - operator.AddLearner{ToStore: 3, PeerID: 3}, - operator.PromoteLearner{ToStore: 3, PeerID: 3}, - operator.TransferLeader{FromStore: 1, ToStore: 3}, - operator.RemovePeer{FromStore: 1}, + AddLearner{ToStore: 3, PeerID: 3}, + PromoteLearner{ToStore: 3, PeerID: 3}, + TransferLeader{FromStore: 1, ToStore: 3}, + RemovePeer{FromStore: 1}, }, { - operator.AddLearner{ToStore: 3, PeerID: 3, IsLightWeight: true}, - operator.PromoteLearner{ToStore: 3, PeerID: 3}, - operator.TransferLeader{FromStore: 1, ToStore: 3}, - operator.RemovePeer{FromStore: 1}, + AddLearner{ToStore: 3, PeerID: 3, IsLightWeight: true}, + PromoteLearner{ToStore: 3, PeerID: 3}, + TransferLeader{FromStore: 1, ToStore: 3}, + RemovePeer{FromStore: 1}, }, } for _, steps := range testSteps { // Create an operator - op := operator.NewTestOperator(1, epoch, operator.OpRegion, steps...) + op := NewTestOperator(1, epoch, OpRegion, steps...) suite.True(controller.AddOperator(op)) suite.Equal(1, stream.MsgLength()) @@ -614,7 +645,7 @@ func (suite *operatorControllerTestSuite) TestDispatchUnfinishedStep() { suite.NotNil(region2.GetPendingPeers()) suite.False(steps[0].IsFinish(region2)) - controller.Dispatch(region2, DispatchFromHeartBeat) + controller.Dispatch(region2, DispatchFromHeartBeat, nil) // In this case, the conf version has been changed, but the // peer added is in pending state, the operator should not be @@ -633,7 +664,7 @@ func (suite *operatorControllerTestSuite) TestDispatchUnfinishedStep() { core.WithIncConfVer(), ) suite.True(steps[0].IsFinish(region3)) - controller.Dispatch(region3, DispatchFromHeartBeat) + controller.Dispatch(region3, DispatchFromHeartBeat, nil) suite.Equal(uint64(1), op.ConfVerChanged(region3)) suite.Equal(2, stream.MsgLength()) @@ -642,7 +673,7 @@ func (suite *operatorControllerTestSuite) TestDispatchUnfinishedStep() { core.WithIncConfVer(), ) suite.True(steps[1].IsFinish(region4)) - controller.Dispatch(region4, DispatchFromHeartBeat) + controller.Dispatch(region4, DispatchFromHeartBeat, nil) suite.Equal(uint64(2), op.ConfVerChanged(region4)) suite.Equal(3, stream.MsgLength()) @@ -651,7 +682,7 @@ func (suite *operatorControllerTestSuite) TestDispatchUnfinishedStep() { core.WithLeader(region4.GetStorePeer(3)), ) suite.True(steps[2].IsFinish(region5)) - controller.Dispatch(region5, DispatchFromHeartBeat) + controller.Dispatch(region5, DispatchFromHeartBeat, nil) suite.Equal(uint64(2), op.ConfVerChanged(region5)) suite.Equal(4, stream.MsgLength()) @@ -661,7 +692,7 @@ func (suite *operatorControllerTestSuite) TestDispatchUnfinishedStep() { core.WithIncConfVer(), ) suite.True(steps[3].IsFinish(region6)) - controller.Dispatch(region6, DispatchFromHeartBeat) + controller.Dispatch(region6, DispatchFromHeartBeat, nil) suite.Equal(uint64(3), op.ConfVerChanged(region6)) // The Operator has finished, so no message should be sent @@ -692,7 +723,7 @@ func newRegionInfo(id uint64, startKey, endKey string, size, keys int64, leader ) } -func (suite *operatorControllerTestSuite) checkRemoveOperatorSuccess(oc *OperatorController, op *operator.Operator) { +func (suite *operatorControllerTestSuite) checkRemoveOperatorSuccess(oc *Controller, op *Operator) { suite.True(oc.RemoveOperator(op)) suite.True(op.IsEnd()) suite.Equal(op, oc.GetOperatorStatus(op.RegionID()).Operator) @@ -702,11 +733,11 @@ func (suite *operatorControllerTestSuite) TestAddWaitingOperator() { opts := mockconfig.NewTestOptions() cluster := mockcluster.NewCluster(suite.ctx, opts) stream := hbstream.NewTestHeartbeatStreams(suite.ctx, cluster.ID, cluster, false /* no need to run */) - controller := NewOperatorController(suite.ctx, cluster, stream) + controller := NewController(suite.ctx, cluster.GetBasicCluster(), cluster.GetSharedConfig(), stream) cluster.AddLabelsStore(1, 1, map[string]string{"host": "host1"}) cluster.AddLabelsStore(2, 1, map[string]string{"host": "host2"}) cluster.AddLabelsStore(3, 1, map[string]string{"host": "host3"}) - addPeerOp := func(i uint64) *operator.Operator { + addPeerOp := func(i uint64) *Operator { start := fmt.Sprintf("%da", i) end := fmt.Sprintf("%db", i) region := newRegionInfo(i, start, end, 1, 1, []uint64{101, 1}, []uint64{101, 1}) @@ -714,7 +745,7 @@ func (suite *operatorControllerTestSuite) TestAddWaitingOperator() { peer := &metapb.Peer{ StoreId: 2, } - op, err := operator.CreateAddPeerOperator("add-peer", cluster, region, peer, operator.OpKind(0)) + op, err := CreateAddPeerOperator("add-peer", cluster, region, peer, OpKind(0)) suite.NoError(err) suite.NotNil(op) @@ -722,7 +753,7 @@ func (suite *operatorControllerTestSuite) TestAddWaitingOperator() { } // a batch of operators should be added atomically - var batch []*operator.Operator + var batch []*Operator for i := uint64(0); i < cluster.GetSchedulerMaxWaitingOperator(); i++ { batch = append(batch, addPeerOp(i)) } @@ -748,7 +779,7 @@ func (suite *operatorControllerTestSuite) TestAddWaitingOperator() { target := newRegionInfo(102, "0a", "0b", 1, 1, []uint64{101, 1}, []uint64{101, 1}) cluster.PutRegion(target) - ops, err := operator.CreateMergeRegionOperator("merge-region", cluster, source, target, operator.OpMerge) + ops, err := CreateMergeRegionOperator("merge-region", cluster, source, target, OpMerge) suite.NoError(err) suite.Len(ops, 2) @@ -762,23 +793,8 @@ func (suite *operatorControllerTestSuite) TestAddWaitingOperator() { }) suite.True(labelerManager.ScheduleDisabled(source)) - // add operator should be failed since it is labeled with `schedule=deny`. - suite.Equal(0, controller.AddWaitingOperator(ops...)) - - // add operator should be success without `schedule=deny` - labelerManager.DeleteLabelRule("schedulelabel") - labelerManager.ScheduleDisabled(source) - suite.False(labelerManager.ScheduleDisabled(source)) - // now there is one operator being allowed to add, if it is a merge operator - // both of the pair are allowed - ops, err = operator.CreateMergeRegionOperator("merge-region", cluster, source, target, operator.OpMerge) - suite.NoError(err) - suite.Len(ops, 2) + // add operator should be success since it is not check in addWaitingOperator suite.Equal(2, controller.AddWaitingOperator(ops...)) - suite.Equal(0, controller.AddWaitingOperator(ops...)) - - // no space left, new operator can not be added. - suite.Equal(0, controller.AddWaitingOperator(addPeerOp(0))) } // issue #5279 @@ -786,16 +802,16 @@ func (suite *operatorControllerTestSuite) TestInvalidStoreId() { opt := mockconfig.NewTestOptions() tc := mockcluster.NewCluster(suite.ctx, opt) stream := hbstream.NewTestHeartbeatStreams(suite.ctx, tc.ID, tc, false /* no need to run */) - oc := NewOperatorController(suite.ctx, tc, stream) + oc := NewController(suite.ctx, tc.GetBasicCluster(), tc.GetSharedConfig(), stream) // If PD and store 3 are gone, PD will not have info of store 3 after recreating it. tc.AddRegionStore(1, 1) tc.AddRegionStore(2, 1) tc.AddRegionStore(4, 1) tc.AddLeaderRegionWithRange(1, "", "", 1, 2, 3, 4) - steps := []operator.OpStep{ - operator.RemovePeer{FromStore: 3, PeerID: 3, IsDownStore: false}, + steps := []OpStep{ + RemovePeer{FromStore: 3, PeerID: 3, IsDownStore: false}, } - op := operator.NewTestOperator(1, &metapb.RegionEpoch{}, operator.OpRegion, steps...) + op := NewTestOperator(1, &metapb.RegionEpoch{}, OpRegion, steps...) suite.True(oc.addOperatorLocked(op)) // Although store 3 does not exist in PD, PD can also send op to TiKV. suite.Equal(pdpb.OperatorStatus_RUNNING, oc.GetOperatorStatus(1).Status) diff --git a/pkg/schedule/operator_queue.go b/pkg/schedule/operator/operator_queue.go similarity index 93% rename from pkg/schedule/operator_queue.go rename to pkg/schedule/operator/operator_queue.go index 62746a8530f..7765427793f 100644 --- a/pkg/schedule/operator_queue.go +++ b/pkg/schedule/operator/operator_queue.go @@ -12,16 +12,14 @@ // See the License for the specific language governing permissions and // limitations under the License. -package schedule +package operator import ( "time" - - "github.com/tikv/pd/pkg/schedule/operator" ) type operatorWithTime struct { - op *operator.Operator + op *Operator time time.Time } diff --git a/pkg/schedule/operator/status.go b/pkg/schedule/operator/status.go index 45a267f96c4..6cbef1b9711 100644 --- a/pkg/schedule/operator/status.go +++ b/pkg/schedule/operator/status.go @@ -31,7 +31,7 @@ const ( // Followings are end status, i.e. no next status. SUCCESS // Finished successfully CANCELED // Canceled due to some reason - REPLACED // Replaced by an higher priority operator + REPLACED // Replaced by a higher priority operator EXPIRED // Didn't start to run for too long TIMEOUT // Running for too long // Status list end diff --git a/pkg/schedule/operator/step.go b/pkg/schedule/operator/step.go index d8d86451543..1a2107cb265 100644 --- a/pkg/schedule/operator/step.go +++ b/pkg/schedule/operator/step.go @@ -27,15 +27,16 @@ import ( "github.com/pingcap/log" "github.com/tikv/pd/pkg/core" "github.com/tikv/pd/pkg/core/storelimit" + "github.com/tikv/pd/pkg/schedule/config" "github.com/tikv/pd/pkg/utils/typeutil" "go.uber.org/zap" ) const ( - // DefaultSlowExecutorRate is the fast rate of the step executor. + // DefaultSlowExecutorRate is the slow rate of the step executor. // default: 6 s/Mb DefaultSlowExecutorRate = 6 - // DefaultFastExecutorRate is the slow rate of the step executor. + // DefaultFastExecutorRate is the fast rate of the step executor. // default: 0.6 s/Mb DefaultFastExecutorRate = 0.6 // FastStepWaitTime is the duration that the OpStep may take. @@ -53,7 +54,7 @@ type OpStep interface { fmt.Stringer ConfVerChanged(region *core.RegionInfo) uint64 IsFinish(region *core.RegionInfo) bool - CheckInProgress(ci ClusterInformer, region *core.RegionInfo) error + CheckInProgress(ci *core.BasicCluster, config config.SharedConfigProvider, region *core.RegionInfo) error Influence(opInfluence OpInfluence, region *core.RegionInfo) Timeout(regionSize int64) time.Duration GetCmd(region *core.RegionInfo, useConfChangeV2 bool) *pdpb.RegionHeartbeatResponse @@ -87,7 +88,7 @@ func (tl TransferLeader) IsFinish(region *core.RegionInfo) bool { } // CheckInProgress checks if the step is in the progress of advancing. -func (tl TransferLeader) CheckInProgress(ci ClusterInformer, region *core.RegionInfo) error { +func (tl TransferLeader) CheckInProgress(ci *core.BasicCluster, config config.SharedConfigProvider, region *core.RegionInfo) error { errList := make([]error, 0, len(tl.ToStores)+1) for _, storeID := range append(tl.ToStores, tl.ToStore) { peer := region.GetStorePeer(tl.ToStore) @@ -99,7 +100,7 @@ func (tl TransferLeader) CheckInProgress(ci ClusterInformer, region *core.Region errList = append(errList, errors.New("peer already is a learner")) continue } - if err := validateStore(ci, storeID); err != nil { + if err := validateStore(ci, config, storeID); err != nil { errList = append(errList, err) continue } @@ -192,8 +193,8 @@ func (ap AddPeer) Influence(opInfluence OpInfluence, region *core.RegionInfo) { } // CheckInProgress checks if the step is in the progress of advancing. -func (ap AddPeer) CheckInProgress(ci ClusterInformer, region *core.RegionInfo) error { - if err := validateStore(ci, ap.ToStore); err != nil { +func (ap AddPeer) CheckInProgress(ci *core.BasicCluster, config config.SharedConfigProvider, region *core.RegionInfo) error { + if err := validateStore(ci, config, ap.ToStore); err != nil { return err } peer := region.GetStorePeer(ap.ToStore) @@ -246,8 +247,8 @@ func (bw BecomeWitness) IsFinish(region *core.RegionInfo) bool { } // CheckInProgress checks if the step is in the progress of advancing. -func (bw BecomeWitness) CheckInProgress(ci ClusterInformer, region *core.RegionInfo) error { - if err := validateStore(ci, bw.StoreID); err != nil { +func (bw BecomeWitness) CheckInProgress(ci *core.BasicCluster, config config.SharedConfigProvider, region *core.RegionInfo) error { + if err := validateStore(ci, config, bw.StoreID); err != nil { return err } peer := region.GetStorePeer(bw.StoreID) @@ -308,8 +309,8 @@ func (bn BecomeNonWitness) IsFinish(region *core.RegionInfo) bool { } // CheckInProgress checks if the step is in the progress of advancing. -func (bn BecomeNonWitness) CheckInProgress(ci ClusterInformer, region *core.RegionInfo) error { - if err := validateStore(ci, bn.StoreID); err != nil { +func (bn BecomeNonWitness) CheckInProgress(ci *core.BasicCluster, config config.SharedConfigProvider, region *core.RegionInfo) error { + if err := validateStore(ci, config, bn.StoreID); err != nil { return err } peer := region.GetStorePeer(bn.StoreID) @@ -394,14 +395,14 @@ func (bsw BatchSwitchWitness) IsFinish(region *core.RegionInfo) bool { } // CheckInProgress checks if the step is in the progress of advancing. -func (bsw BatchSwitchWitness) CheckInProgress(ci ClusterInformer, region *core.RegionInfo) error { +func (bsw BatchSwitchWitness) CheckInProgress(ci *core.BasicCluster, config config.SharedConfigProvider, region *core.RegionInfo) error { for _, w := range bsw.ToWitnesses { - if err := w.CheckInProgress(ci, region); err != nil { + if err := w.CheckInProgress(ci, config, region); err != nil { return err } } for _, nw := range bsw.ToNonWitnesses { - if err := nw.CheckInProgress(ci, region); err != nil { + if err := nw.CheckInProgress(ci, config, region); err != nil { return err } } @@ -477,8 +478,8 @@ func (al AddLearner) IsFinish(region *core.RegionInfo) bool { } // CheckInProgress checks if the step is in the progress of advancing. -func (al AddLearner) CheckInProgress(ci ClusterInformer, region *core.RegionInfo) error { - if err := validateStore(ci, al.ToStore); err != nil { +func (al AddLearner) CheckInProgress(ci *core.BasicCluster, config config.SharedConfigProvider, region *core.RegionInfo) error { + if err := validateStore(ci, config, al.ToStore); err != nil { return err } peer := region.GetStorePeer(al.ToStore) @@ -563,7 +564,7 @@ func (pl PromoteLearner) IsFinish(region *core.RegionInfo) bool { } // CheckInProgress checks if the step is in the progress of advancing. -func (pl PromoteLearner) CheckInProgress(_ ClusterInformer, region *core.RegionInfo) error { +func (pl PromoteLearner) CheckInProgress(_ *core.BasicCluster, config config.SharedConfigProvider, region *core.RegionInfo) error { peer := region.GetStorePeer(pl.ToStore) if peer.GetId() != pl.PeerID { return errors.New("peer does not exist") @@ -587,6 +588,7 @@ func (pl PromoteLearner) GetCmd(_ *core.RegionInfo, useConfChangeV2 bool) *pdpb. // RemovePeer is an OpStep that removes a region peer. type RemovePeer struct { FromStore, PeerID uint64 + IsLightWeight bool IsDownStore bool } @@ -614,7 +616,7 @@ func (rp RemovePeer) IsFinish(region *core.RegionInfo) bool { } // CheckInProgress checks if the step is in the progress of advancing. -func (rp RemovePeer) CheckInProgress(_ ClusterInformer, region *core.RegionInfo) error { +func (rp RemovePeer) CheckInProgress(_ *core.BasicCluster, config config.SharedConfigProvider, region *core.RegionInfo) error { if rp.FromStore == region.GetLeader().GetStoreId() { return errors.New("cannot remove leader peer") } @@ -634,6 +636,10 @@ func (rp RemovePeer) Influence(opInfluence OpInfluence, region *core.RegionInfo) return } + if rp.IsLightWeight { + return + } + if rp.IsDownStore && regionSize > storelimit.SmallRegionThreshold { regionSize = storelimit.SmallRegionThreshold } @@ -684,7 +690,7 @@ func (mr MergeRegion) IsFinish(region *core.RegionInfo) bool { } // CheckInProgress checks if the step is in the progress of advancing. -func (mr MergeRegion) CheckInProgress(_ ClusterInformer, _ *core.RegionInfo) error { +func (mr MergeRegion) CheckInProgress(_ *core.BasicCluster, config config.SharedConfigProvider, _ *core.RegionInfo) error { return nil } @@ -752,7 +758,7 @@ func (sr SplitRegion) Influence(opInfluence OpInfluence, region *core.RegionInfo } // CheckInProgress checks if the step is in the progress of advancing. -func (sr SplitRegion) CheckInProgress(_ ClusterInformer, _ *core.RegionInfo) error { +func (sr SplitRegion) CheckInProgress(_ *core.BasicCluster, config config.SharedConfigProvider, _ *core.RegionInfo) error { return nil } @@ -877,7 +883,7 @@ func (cpe ChangePeerV2Enter) IsFinish(region *core.RegionInfo) bool { } // CheckInProgress checks if the step is in the progress of advancing. -func (cpe ChangePeerV2Enter) CheckInProgress(_ ClusterInformer, region *core.RegionInfo) error { +func (cpe ChangePeerV2Enter) CheckInProgress(_ *core.BasicCluster, config config.SharedConfigProvider, region *core.RegionInfo) error { inJointState, notInJointState := false, false for _, pl := range cpe.PromoteLearners { peer := region.GetStorePeer(pl.ToStore) @@ -1006,7 +1012,7 @@ func (cpl ChangePeerV2Leave) IsFinish(region *core.RegionInfo) bool { } // CheckInProgress checks if the step is in the progress of advancing. -func (cpl ChangePeerV2Leave) CheckInProgress(_ ClusterInformer, region *core.RegionInfo) error { +func (cpl ChangePeerV2Leave) CheckInProgress(_ *core.BasicCluster, config config.SharedConfigProvider, region *core.RegionInfo) error { inJointState, notInJointState, demoteLeader := false, false, false leaderStoreID := region.GetLeader().GetStoreId() @@ -1084,12 +1090,12 @@ func (cpl ChangePeerV2Leave) GetCmd(region *core.RegionInfo, useConfChangeV2 boo } } -func validateStore(ci ClusterInformer, id uint64) error { - store := ci.GetBasicCluster().GetStore(id) +func validateStore(ci *core.BasicCluster, config config.SharedConfigProvider, id uint64) error { + store := ci.GetStore(id) if store == nil { return errors.New("target store does not exist") } - if store.DownTime() > ci.GetOpts().GetMaxStoreDownTime() { + if store.DownTime() > config.GetMaxStoreDownTime() { return errors.New("target store is down") } return nil diff --git a/pkg/schedule/operator/step_test.go b/pkg/schedule/operator/step_test.go index 3b797575149..4865180a8bb 100644 --- a/pkg/schedule/operator/step_test.go +++ b/pkg/schedule/operator/step_test.go @@ -566,7 +566,7 @@ func (suite *operatorStepTestSuite) check(step OpStep, desc string, testCases [] region := core.NewRegionInfo(&metapb.Region{Id: 1, Peers: testCase.Peers}, testCase.Peers[0]) suite.Equal(testCase.ConfVerChanged, step.ConfVerChanged(region)) suite.Equal(testCase.IsFinish, step.IsFinish(region)) - err := step.CheckInProgress(suite.cluster, region) + err := step.CheckInProgress(suite.cluster.GetBasicCluster(), suite.cluster.GetSharedConfig(), region) testCase.CheckInProgress(err) _ = step.GetCmd(region, true) diff --git a/pkg/schedule/test_util.go b/pkg/schedule/operator/test_util.go similarity index 84% rename from pkg/schedule/test_util.go rename to pkg/schedule/operator/test_util.go index 7b2e959b71c..8206189daa6 100644 --- a/pkg/schedule/test_util.go +++ b/pkg/schedule/operator/test_util.go @@ -12,23 +12,22 @@ // See the License for the specific language governing permissions and // limitations under the License. -package schedule +package operator import ( "github.com/pingcap/kvproto/pkg/metapb" "github.com/tikv/pd/pkg/core" "github.com/tikv/pd/pkg/mock/mockcluster" - "github.com/tikv/pd/pkg/schedule/operator" ) // ApplyOperatorStep applies operator step. Only for test purpose. -func ApplyOperatorStep(region *core.RegionInfo, op *operator.Operator) *core.RegionInfo { +func ApplyOperatorStep(region *core.RegionInfo, op *Operator) *core.RegionInfo { _ = op.Start() if step := op.Check(region); step != nil { switch s := step.(type) { - case operator.TransferLeader: + case TransferLeader: region = region.Clone(core.WithLeader(region.GetStorePeer(s.ToStore))) - case operator.AddPeer: + case AddPeer: if region.GetStorePeer(s.ToStore) != nil { panic("Add peer that exists") } @@ -37,7 +36,7 @@ func ApplyOperatorStep(region *core.RegionInfo, op *operator.Operator) *core.Reg StoreId: s.ToStore, } region = region.Clone(core.WithAddPeer(peer)) - case operator.RemovePeer: + case RemovePeer: if region.GetStorePeer(s.FromStore) == nil { panic("Remove peer that doesn't exist") } @@ -45,7 +44,7 @@ func ApplyOperatorStep(region *core.RegionInfo, op *operator.Operator) *core.Reg panic("Cannot remove the leader peer") } region = region.Clone(core.WithRemoveStorePeer(s.FromStore)) - case operator.AddLearner: + case AddLearner: if region.GetStorePeer(s.ToStore) != nil { panic("Add learner that exists") } @@ -55,7 +54,7 @@ func ApplyOperatorStep(region *core.RegionInfo, op *operator.Operator) *core.Reg Role: metapb.PeerRole_Learner, } region = region.Clone(core.WithAddPeer(peer)) - case operator.PromoteLearner: + case PromoteLearner: if region.GetStoreLearner(s.ToStore) == nil { panic("Promote peer that doesn't exist") } @@ -71,8 +70,8 @@ func ApplyOperatorStep(region *core.RegionInfo, op *operator.Operator) *core.Reg return region } -// ApplyOperator applies operator. Only for test purpose. -func ApplyOperator(mc *mockcluster.Cluster, op *operator.Operator) { +// ApplyOperator applies Only for test purpose. +func ApplyOperator(mc *mockcluster.Cluster, op *Operator) { origin := mc.GetRegion(op.RegionID()) region := origin for !op.IsEnd() { diff --git a/pkg/schedule/waiting_operator.go b/pkg/schedule/operator/waiting_operator.go similarity index 58% rename from pkg/schedule/waiting_operator.go rename to pkg/schedule/operator/waiting_operator.go index e413cd97aba..8f5c72b053b 100644 --- a/pkg/schedule/waiting_operator.go +++ b/pkg/schedule/operator/waiting_operator.go @@ -12,50 +12,47 @@ // See the License for the specific language governing permissions and // limitations under the License. -package schedule +package operator import ( "math/rand" - "time" - - "github.com/tikv/pd/pkg/schedule/operator" ) -// PriorityWeight is used to represent the weight of different priorities of operators. -var PriorityWeight = []float64{1.0, 4.0, 9.0, 16.0} +// priorityWeight is used to represent the weight of different priorities of operators. +var priorityWeight = []float64{1.0, 4.0, 9.0, 16.0} // WaitingOperator is an interface of waiting operators. type WaitingOperator interface { - PutOperator(op *operator.Operator) - GetOperator() []*operator.Operator - ListOperator() []*operator.Operator + PutOperator(op *Operator) + GetOperator() []*Operator + ListOperator() []*Operator } -// Bucket is used to maintain the operators created by a specific scheduler. -type Bucket struct { +// bucket is used to maintain the operators created by a specific scheduler. +type bucket struct { weight float64 - ops []*operator.Operator + ops []*Operator } -// RandBuckets is an implementation of waiting operators -type RandBuckets struct { +// randBuckets is an implementation of waiting operators +type randBuckets struct { totalWeight float64 - buckets []*Bucket + buckets []*bucket } -// NewRandBuckets creates a random buckets. -func NewRandBuckets() *RandBuckets { - var buckets []*Bucket - for i := 0; i < len(PriorityWeight); i++ { - buckets = append(buckets, &Bucket{ - weight: PriorityWeight[i], +// newRandBuckets creates a random buckets. +func newRandBuckets() *randBuckets { + var buckets []*bucket + for i := 0; i < len(priorityWeight); i++ { + buckets = append(buckets, &bucket{ + weight: priorityWeight[i], }) } - return &RandBuckets{buckets: buckets} + return &randBuckets{buckets: buckets} } // PutOperator puts an operator into the random buckets. -func (b *RandBuckets) PutOperator(op *operator.Operator) { +func (b *randBuckets) PutOperator(op *Operator) { priority := op.GetPriorityLevel() bucket := b.buckets[priority] if len(bucket.ops) == 0 { @@ -65,8 +62,8 @@ func (b *RandBuckets) PutOperator(op *operator.Operator) { } // ListOperator lists all operator in the random buckets. -func (b *RandBuckets) ListOperator() []*operator.Operator { - var ops []*operator.Operator +func (b *randBuckets) ListOperator() []*Operator { + var ops []*Operator for i := range b.buckets { bucket := b.buckets[i] ops = append(ops, bucket.ops...) @@ -75,7 +72,7 @@ func (b *RandBuckets) ListOperator() []*operator.Operator { } // GetOperator gets an operator from the random buckets. -func (b *RandBuckets) GetOperator() []*operator.Operator { +func (b *randBuckets) GetOperator() []*Operator { if b.totalWeight == 0 { return nil } @@ -88,10 +85,10 @@ func (b *RandBuckets) GetOperator() []*operator.Operator { } proportion := bucket.weight / b.totalWeight if r >= sum && r < sum+proportion { - var res []*operator.Operator + var res []*Operator res = append(res, bucket.ops[0]) // Merge operation has two operators, and thus it should be handled specifically. - if bucket.ops[0].Kind()&operator.OpMerge != 0 { + if bucket.ops[0].Kind()&OpMerge != 0 { res = append(res, bucket.ops[1]) bucket.ops = bucket.ops[2:] } else { @@ -107,18 +104,14 @@ func (b *RandBuckets) GetOperator() []*operator.Operator { return nil } -// WaitingOperatorStatus is used to limit the count of each kind of operators. -type WaitingOperatorStatus struct { +// waitingOperatorStatus is used to limit the count of each kind of operators. +type waitingOperatorStatus struct { ops map[string]uint64 } -// NewWaitingOperatorStatus creates a new WaitingOperatorStatus. -func NewWaitingOperatorStatus() *WaitingOperatorStatus { - return &WaitingOperatorStatus{ +// newWaitingOperatorStatus creates a new waitingOperatorStatus. +func newWaitingOperatorStatus() *waitingOperatorStatus { + return &waitingOperatorStatus{ make(map[string]uint64), } } - -func init() { - rand.New(rand.NewSource(time.Now().UnixNano())) -} diff --git a/pkg/schedule/waiting_operator_test.go b/pkg/schedule/operator/waiting_operator_test.go similarity index 65% rename from pkg/schedule/waiting_operator_test.go rename to pkg/schedule/operator/waiting_operator_test.go index 4dec91efe41..897f416cf38 100644 --- a/pkg/schedule/waiting_operator_test.go +++ b/pkg/schedule/operator/waiting_operator_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package schedule +package operator import ( "testing" @@ -20,14 +20,13 @@ import ( "github.com/pingcap/kvproto/pkg/metapb" "github.com/stretchr/testify/require" "github.com/tikv/pd/pkg/core/constant" - "github.com/tikv/pd/pkg/schedule/operator" ) func TestRandBuckets(t *testing.T) { re := require.New(t) - rb := NewRandBuckets() + rb := newRandBuckets() addOperators(rb) - for i := 0; i < len(PriorityWeight); i++ { + for i := 0; i < len(priorityWeight); i++ { op := rb.GetOperator() re.NotNil(op) } @@ -35,23 +34,23 @@ func TestRandBuckets(t *testing.T) { } func addOperators(wop WaitingOperator) { - op := operator.NewTestOperator(uint64(1), &metapb.RegionEpoch{}, operator.OpRegion, []operator.OpStep{ - operator.RemovePeer{FromStore: uint64(1)}, + op := NewTestOperator(uint64(1), &metapb.RegionEpoch{}, OpRegion, []OpStep{ + RemovePeer{FromStore: uint64(1)}, }...) op.SetPriorityLevel(constant.Medium) wop.PutOperator(op) - op = operator.NewTestOperator(uint64(2), &metapb.RegionEpoch{}, operator.OpRegion, []operator.OpStep{ - operator.RemovePeer{FromStore: uint64(2)}, + op = NewTestOperator(uint64(2), &metapb.RegionEpoch{}, OpRegion, []OpStep{ + RemovePeer{FromStore: uint64(2)}, }...) op.SetPriorityLevel(constant.High) wop.PutOperator(op) - op = operator.NewTestOperator(uint64(3), &metapb.RegionEpoch{}, operator.OpRegion, []operator.OpStep{ - operator.RemovePeer{FromStore: uint64(3)}, + op = NewTestOperator(uint64(3), &metapb.RegionEpoch{}, OpRegion, []OpStep{ + RemovePeer{FromStore: uint64(3)}, }...) op.SetPriorityLevel(constant.Low) wop.PutOperator(op) - op = operator.NewTestOperator(uint64(4), &metapb.RegionEpoch{}, operator.OpRegion, []operator.OpStep{ - operator.RemovePeer{FromStore: uint64(4)}, + op = NewTestOperator(uint64(4), &metapb.RegionEpoch{}, OpRegion, []OpStep{ + RemovePeer{FromStore: uint64(4)}, }...) op.SetPriorityLevel(constant.Urgent) wop.PutOperator(op) @@ -59,20 +58,20 @@ func addOperators(wop WaitingOperator) { func TestListOperator(t *testing.T) { re := require.New(t) - rb := NewRandBuckets() + rb := newRandBuckets() addOperators(rb) - re.Len(rb.ListOperator(), len(PriorityWeight)) + re.Len(rb.ListOperator(), len(priorityWeight)) } func TestRandomBucketsWithMergeRegion(t *testing.T) { re := require.New(t) - rb := NewRandBuckets() + rb := newRandBuckets() descs := []string{"merge-region", "admin-merge-region", "random-merge"} for j := 0; j < 100; j++ { // adds operators desc := descs[j%3] - op := operator.NewTestOperator(uint64(1), &metapb.RegionEpoch{}, operator.OpRegion|operator.OpMerge, []operator.OpStep{ - operator.MergeRegion{ + op := NewTestOperator(uint64(1), &metapb.RegionEpoch{}, OpRegion|OpMerge, []OpStep{ + MergeRegion{ FromRegion: &metapb.Region{ Id: 1, StartKey: []byte{}, @@ -87,8 +86,8 @@ func TestRandomBucketsWithMergeRegion(t *testing.T) { }...) op.SetDesc(desc) rb.PutOperator(op) - op = operator.NewTestOperator(uint64(2), &metapb.RegionEpoch{}, operator.OpRegion|operator.OpMerge, []operator.OpStep{ - operator.MergeRegion{ + op = NewTestOperator(uint64(2), &metapb.RegionEpoch{}, OpRegion|OpMerge, []OpStep{ + MergeRegion{ FromRegion: &metapb.Region{ Id: 1, StartKey: []byte{}, @@ -103,8 +102,8 @@ func TestRandomBucketsWithMergeRegion(t *testing.T) { }...) op.SetDesc(desc) rb.PutOperator(op) - op = operator.NewTestOperator(uint64(3), &metapb.RegionEpoch{}, operator.OpRegion, []operator.OpStep{ - operator.RemovePeer{FromStore: uint64(3)}, + op = NewTestOperator(uint64(3), &metapb.RegionEpoch{}, OpRegion, []OpStep{ + RemovePeer{FromStore: uint64(3)}, }...) op.SetDesc("testOperatorHigh") op.SetPriorityLevel(constant.High) diff --git a/pkg/schedule/placement/fit.go b/pkg/schedule/placement/fit.go index e3f6edb7e90..45afc5bcfa3 100644 --- a/pkg/schedule/placement/fit.go +++ b/pkg/schedule/placement/fit.go @@ -37,17 +37,32 @@ type RegionFit struct { // Replace return true if the replacement store is fit all constraints and isolation score is not less than the origin. func (f *RegionFit) Replace(srcStoreID uint64, dstStore *core.StoreInfo) bool { + if dstStore == nil { + return false + } fit := f.getRuleFitByStoreID(srcStoreID) // check the target store is fit all constraints. - if fit == nil || !MatchLabelConstraints(dstStore, fit.Rule.LabelConstraints) { + if fit == nil { return false } - // the members of the rule are same, it shouldn't check the score. + // it is used to transfer role like transfer leader. if fit.contain(dstStore.GetID()) { return true } + // it should be allowed to exchange role if both the source and target's rule role is same. + // e.g. transfer leader from store 1 to store 2, and store-1 and store-2 are both voter. + targetFit := f.getRuleFitByStoreID(dstStore.GetID()) + if targetFit != nil && targetFit.Rule.Role == fit.Rule.Role { + return true + } + + // the target store should be fit all constraints. + if !MatchLabelConstraints(dstStore, fit.Rule.LabelConstraints) { + return false + } + score := isolationStoreScore(srcStoreID, dstStore, fit.stores, fit.Rule.LocationLabels) // restore the source store. return fit.IsolationScore <= score @@ -107,6 +122,7 @@ type RuleFit struct { // IsolationScore indicates at which level of labeling these Peers are // isolated. A larger value is better. IsolationScore float64 `json:"isolation-score"` + WitnessScore int `json:"witness-score"` // stores is the stores that the peers are placed in. stores []*core.StoreInfo } @@ -139,6 +155,10 @@ func compareRuleFit(a, b *RuleFit) int { return -1 case a.IsolationScore > b.IsolationScore: return 1 + case a.WitnessScore > b.WitnessScore: + return -1 + case a.WitnessScore < b.WitnessScore: + return 1 default: return 0 } @@ -333,7 +353,7 @@ func (w *fitWorker) updateOrphanPeers(index int) { } func newRuleFit(rule *Rule, peers []*fitPeer, supportWitness bool) *RuleFit { - rf := &RuleFit{Rule: rule, IsolationScore: isolationScore(peers, rule.LocationLabels)} + rf := &RuleFit{Rule: rule, IsolationScore: isolationScore(peers, rule.LocationLabels), WitnessScore: witnessScore(peers, supportWitness && rule.IsWitness)} for _, p := range peers { rf.Peers = append(rf.Peers, p.Peer) rf.stores = append(rf.stores, p.store) @@ -429,3 +449,14 @@ func stateScore(region *core.RegionInfo, peerID uint64) int { return 2 } } + +func witnessScore(peers []*fitPeer, fitWitness bool) int { + var score int + if !fitWitness || len(peers) == 0 { + return 0 + } + for _, p := range peers { + score += p.store.GetWitnessCount() + } + return score +} diff --git a/pkg/schedule/placement/fit_region_test.go b/pkg/schedule/placement/fit_region_test.go index 55b0e76a2d2..0ec67b2a2aa 100644 --- a/pkg/schedule/placement/fit_region_test.go +++ b/pkg/schedule/placement/fit_region_test.go @@ -326,7 +326,7 @@ func BenchmarkFitRegionWithMoreRulesAndStoreLabels(b *testing.B) { label := &metapb.StoreLabel{Key: "exclusive", Value: "exclusive"} labels = append(labels, label) // 5 peers in 5 different stores, - // split the stores(peers) to three zones,make the number of peers in each zone: 2:2:1 + // split the stores(peers) to three zones, make the number of peers in each zone: 2:2:1 for _, peer := range region.GetPeers() { storeID := peer.StoreId store := core.NewStoreInfo(&metapb.Store{Id: storeID}, core.SetLastHeartbeatTS(time.Now()), core.SetStoreLabels(labels)) @@ -380,7 +380,7 @@ func BenchmarkFitRegionWithLocationLabels(b *testing.B) { // create stores lists := make([]*core.StoreInfo, 0) // 10 peers in 10 different stores, - // split the stores(peers) to three zones,make the number of peers in each zone: 4:3:3 + // split the stores(peers) to three zones, make the number of peers in each zone: 4:3:3 for idx, peer := range region.GetPeers() { storeID := peer.StoreId zoneInfo := &metapb.StoreLabel{Key: "zone", Value: fmt.Sprintf("z_%02d", idx%3)} diff --git a/pkg/schedule/placement/fit_test.go b/pkg/schedule/placement/fit_test.go index 65743451a2b..d7b3c91163e 100644 --- a/pkg/schedule/placement/fit_test.go +++ b/pkg/schedule/placement/fit_test.go @@ -133,13 +133,18 @@ func TestReplace(t *testing.T) { {"1111,2111,3111,1115_learner", []string{"3/voter//zone", "1/learner/engine=tiflash/host"}, 1115, 2115, true}, // replace failed when the target store is not tiflash {"1111,2111,3111,1115_learner", []string{"3/voter//zone", "1/learner/engine=tiflash/host"}, 1115, 1112, false}, - {"1111_lead,2111,3111", []string{"1/leader/zone=zone1/zone", "2/voter//zone"}, 1111, 1112, true}, + {"1111_leader,2111,3111", []string{"1/leader/zone=zone1/zone", "2/voter//zone"}, 1111, 1112, true}, // replace failed when the leader is not match the leader constraint. {"1111_leader,2111,3111", []string{"1/leader/zone=zone1/zone", "2/voter//zone"}, 1111, 2112, false}, // transfer leader - {"1111_leader,1121,1131", []string{"1/leader/host=host1+host2/host", "3/voter//host"}, 1111, 1121, true}, + {"1111_leader,1121,1131", []string{"1/leader/host=host1+host2/host", "2/voter//host"}, 1111, 1121, true}, // replace failed when the leader is not match the leader constraint. {"1111_leader,1121,1131", []string{"1/leader/host=host1+host2/host", "2/voter//host"}, 1111, 1131, false}, + + // transfer leader success with different rule when the role is the same. + {"1111_leader,1121,1131", []string{"1/voter/host=host1/host", "1/voter/host=host2/host", "1/voter/host=host3/host"}, 1111, 1121, true}, + //// transfer leader failed with different rule, but the role isn't same. + {"1111_leader,1121,1131", []string{"1/leader/host=host1/host", "1/voter/host=host2/host", "1/voter/host=host3/host"}, 1111, 1121, false}, } for _, tc := range testCases { region := makeRegion(tc.region) @@ -148,6 +153,7 @@ func TestReplace(t *testing.T) { rules = append(rules, makeRule(r)) } rf := fitRegion(stores.GetStores(), region, rules, false) + re.True(rf.IsSatisfied()) rf.regionStores = stores.GetStores() re.Equal(rf.Replace(tc.srcStoreID, stores.GetStore(tc.dstStoreID)), tc.ok) } diff --git a/pkg/schedule/placement/label_constraint.go b/pkg/schedule/placement/label_constraint.go index 2fcef35d05d..6cc2d4ed457 100644 --- a/pkg/schedule/placement/label_constraint.go +++ b/pkg/schedule/placement/label_constraint.go @@ -72,6 +72,7 @@ var legacyExclusiveLabels = []string{core.EngineKey, "exclusive"} // If a store has exclusiveLabels, it can only be selected when the label is // explicitly specified in constraints. func isExclusiveLabel(key string) bool { + // *Do not* change this behavior, we rely on it to physically isolate stores. return strings.HasPrefix(key, "$") || slice.AnyOf(legacyExclusiveLabels, func(i int) bool { return key == legacyExclusiveLabels[i] }) diff --git a/pkg/schedule/placement/rule_manager.go b/pkg/schedule/placement/rule_manager.go index d3f6bda066b..0237e0d1fed 100644 --- a/pkg/schedule/placement/rule_manager.go +++ b/pkg/schedule/placement/rule_manager.go @@ -50,11 +50,11 @@ type RuleManager struct { keyType string storeSetInformer core.StoreSetInformer cache *RegionRuleFitCacheManager - conf config.Config + conf config.SharedConfigProvider } // NewRuleManager creates a RuleManager instance. -func NewRuleManager(storage endpoint.RuleStorage, storeSetInformer core.StoreSetInformer, conf config.Config) *RuleManager { +func NewRuleManager(storage endpoint.RuleStorage, storeSetInformer core.StoreSetInformer, conf config.SharedConfigProvider) *RuleManager { return &RuleManager{ storage: storage, storeSetInformer: storeSetInformer, diff --git a/pkg/schedule/placement/rule_manager_test.go b/pkg/schedule/placement/rule_manager_test.go index 894f78f1fef..e5be8d74cd2 100644 --- a/pkg/schedule/placement/rule_manager_test.go +++ b/pkg/schedule/placement/rule_manager_test.go @@ -33,7 +33,7 @@ func newTestManager(t *testing.T, enableWitness bool) (endpoint.RuleStorage, *Ru store := endpoint.NewStorageEndpoint(kv.NewMemoryKV(), nil) var err error manager := NewRuleManager(store, nil, mockconfig.NewTestOptions()) - manager.conf.SetWitnessEnabled(enableWitness) + manager.conf.SetEnableWitness(enableWitness) err = manager.Initialize(3, []string{"zone", "rack", "host"}) re.NoError(err) return store, manager diff --git a/pkg/schedule/schedulers/balance_plan.go b/pkg/schedule/plan/balance_plan.go similarity index 59% rename from pkg/schedule/schedulers/balance_plan.go rename to pkg/schedule/plan/balance_plan.go index fc38de7a39b..819a00f94e7 100644 --- a/pkg/schedule/schedulers/balance_plan.go +++ b/pkg/schedule/plan/balance_plan.go @@ -12,12 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -package schedulers +package plan import ( "github.com/tikv/pd/pkg/core" "github.com/tikv/pd/pkg/errs" - "github.com/tikv/pd/pkg/schedule/plan" ) const ( @@ -29,82 +28,90 @@ const ( // createOperator ) -type balanceSchedulerPlan struct { - source *core.StoreInfo - target *core.StoreInfo - region *core.RegionInfo - status *plan.Status - step int +// BalanceSchedulerPlan is a plan for balance scheduler +type BalanceSchedulerPlan struct { + Source *core.StoreInfo + Target *core.StoreInfo + Region *core.RegionInfo + Status *Status + Step int } // NewBalanceSchedulerPlan returns a new balanceSchedulerBasePlan -func NewBalanceSchedulerPlan() *balanceSchedulerPlan { - basePlan := &balanceSchedulerPlan{ - status: plan.NewStatus(plan.StatusOK), +func NewBalanceSchedulerPlan() *BalanceSchedulerPlan { + basePlan := &BalanceSchedulerPlan{ + Status: NewStatus(StatusOK), } return basePlan } -func (p *balanceSchedulerPlan) GetStep() int { - return p.step +// GetStep is used to get current step of plan. +func (p *BalanceSchedulerPlan) GetStep() int { + return p.Step } -func (p *balanceSchedulerPlan) SetResource(resource interface{}) { - switch p.step { +// SetResource is used to set resource for current step. +func (p *BalanceSchedulerPlan) SetResource(resource interface{}) { + switch p.Step { // for balance-region/leader scheduler, the first step is selecting stores as source candidates. case pickSource: - p.source = resource.(*core.StoreInfo) + p.Source = resource.(*core.StoreInfo) // the second step is selecting region from source store. case pickRegion: - p.region = resource.(*core.RegionInfo) + p.Region = resource.(*core.RegionInfo) // the third step is selecting stores as target candidates. case pickTarget: - p.target = resource.(*core.StoreInfo) + p.Target = resource.(*core.StoreInfo) } } -func (p *balanceSchedulerPlan) SetResourceWithStep(resource interface{}, step int) { - p.step = step +// SetResourceWithStep is used to set resource for specific step. +func (p *BalanceSchedulerPlan) SetResourceWithStep(resource interface{}, step int) { + p.Step = step p.SetResource(resource) } -func (p *balanceSchedulerPlan) GetResource(step int) uint64 { - if p.step < step { +// GetResource is used to get resource for specific step. +func (p *BalanceSchedulerPlan) GetResource(step int) uint64 { + if p.Step < step { return 0 } // Please use with care. Add a nil check if need in the future switch step { case pickSource: - return p.source.GetID() + return p.Source.GetID() case pickRegion: - return p.region.GetID() + return p.Region.GetID() case pickTarget: - return p.target.GetID() + return p.Target.GetID() } return 0 } -func (p *balanceSchedulerPlan) GetStatus() *plan.Status { - return p.status +// GetStatus is used to get status of plan. +func (p *BalanceSchedulerPlan) GetStatus() *Status { + return p.Status } -func (p *balanceSchedulerPlan) SetStatus(status *plan.Status) { - p.status = status +// SetStatus is used to set status of plan. +func (p *BalanceSchedulerPlan) SetStatus(status *Status) { + p.Status = status } -func (p *balanceSchedulerPlan) Clone(opts ...plan.Option) plan.Plan { - plan := &balanceSchedulerPlan{ - status: p.status, +// Clone is used to clone a new plan. +func (p *BalanceSchedulerPlan) Clone(opts ...Option) Plan { + plan := &BalanceSchedulerPlan{ + Status: p.Status, } - plan.step = p.step - if p.step > pickSource { - plan.source = p.source + plan.Step = p.Step + if p.Step > pickSource { + plan.Source = p.Source } - if p.step > pickRegion { - plan.region = p.region + if p.Step > pickRegion { + plan.Region = p.Region } - if p.step > pickTarget { - plan.target = p.target + if p.Step > pickTarget { + plan.Target = p.Target } for _, opt := range opts { opt(plan) @@ -113,15 +120,15 @@ func (p *balanceSchedulerPlan) Clone(opts ...plan.Option) plan.Plan { } // BalancePlanSummary is used to summarize for BalancePlan -func BalancePlanSummary(plans []plan.Plan) (map[uint64]plan.Status, bool, error) { +func BalancePlanSummary(plans []Plan) (map[uint64]Status, bool, error) { // storeStatusCounter is used to count the number of various statuses of each store - storeStatusCounter := make(map[uint64]map[plan.Status]int) + storeStatusCounter := make(map[uint64]map[Status]int) // statusCounter is used to count the number of status which is regarded as best status of each store - statusCounter := make(map[uint64]plan.Status) + statusCounter := make(map[uint64]Status) storeMaxStep := make(map[uint64]int) normal := true for _, pi := range plans { - p, ok := pi.(*balanceSchedulerPlan) + p, ok := pi.(*BalanceSchedulerPlan) if !ok { return nil, false, errs.ErrDiagnosticLoadPlan } @@ -134,7 +141,7 @@ func BalancePlanSummary(plans []plan.Plan) (map[uint64]plan.Status, bool, error) // `step == pickRegion` is a special processing in summary, because we want to exclude the factor of region // and consider the failure as the status of source store. if step == pickRegion { - store = p.source.GetID() + store = p.Source.GetID() } else { store = p.GetResource(step) } @@ -143,20 +150,20 @@ func BalancePlanSummary(plans []plan.Plan) (map[uint64]plan.Status, bool, error) maxStep = -1 } if step > maxStep { - storeStatusCounter[store] = make(map[plan.Status]int) + storeStatusCounter[store] = make(map[Status]int) storeMaxStep[store] = step } else if step < maxStep { continue } - if !p.status.IsNormal() { + if !p.Status.IsNormal() { normal = false } - storeStatusCounter[store][*p.status]++ + storeStatusCounter[store][*p.Status]++ } for id, store := range storeStatusCounter { max := 0 - curStat := *plan.NewStatus(plan.StatusOK) + curStat := *NewStatus(StatusOK) for stat, c := range store { if balancePlanStatusComparer(max, curStat, c, stat) { max = c @@ -169,7 +176,7 @@ func BalancePlanSummary(plans []plan.Plan) (map[uint64]plan.Status, bool, error) } // balancePlanStatusComparer returns true if new status is better than old one. -func balancePlanStatusComparer(oldStatusCount int, oldStatus plan.Status, newStatusCount int, newStatus plan.Status) bool { +func balancePlanStatusComparer(oldStatusCount int, oldStatus Status, newStatusCount int, newStatus Status) bool { if newStatus.Priority() != oldStatus.Priority() { return newStatus.Priority() > oldStatus.Priority() } diff --git a/pkg/schedule/plan/balance_plan_test.go b/pkg/schedule/plan/balance_plan_test.go new file mode 100644 index 00000000000..59ad637d5c8 --- /dev/null +++ b/pkg/schedule/plan/balance_plan_test.go @@ -0,0 +1,271 @@ +// Copyright 2022 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package plan + +import ( + "context" + "testing" + + "github.com/pingcap/kvproto/pkg/metapb" + "github.com/stretchr/testify/suite" + "github.com/tikv/pd/pkg/core" +) + +type balanceSchedulerPlanAnalyzeTestSuite struct { + suite.Suite + + stores []*core.StoreInfo + regions []*core.RegionInfo + check func(map[uint64]Status, map[uint64]*Status) bool + ctx context.Context + cancel context.CancelFunc +} + +func TestBalanceSchedulerPlanAnalyzerTestSuite(t *testing.T) { + suite.Run(t, new(balanceSchedulerPlanAnalyzeTestSuite)) +} + +func (suite *balanceSchedulerPlanAnalyzeTestSuite) SetupSuite() { + suite.ctx, suite.cancel = context.WithCancel(context.Background()) + suite.check = func(output map[uint64]Status, expects map[uint64]*Status) bool { + for id, Status := range expects { + outputStatus, ok := output[id] + if !ok { + return false + } + if outputStatus != *Status { + return false + } + } + return true + } + suite.stores = []*core.StoreInfo{ + core.NewStoreInfo( + &metapb.Store{ + Id: 1, + }, + ), + core.NewStoreInfo( + &metapb.Store{ + Id: 2, + }, + ), + core.NewStoreInfo( + &metapb.Store{ + Id: 3, + }, + ), + core.NewStoreInfo( + &metapb.Store{ + Id: 4, + }, + ), + core.NewStoreInfo( + &metapb.Store{ + Id: 5, + }, + ), + } + suite.regions = []*core.RegionInfo{ + core.NewRegionInfo( + &metapb.Region{ + Id: 1, + }, + &metapb.Peer{ + Id: 1, + StoreId: 1, + }, + ), + core.NewRegionInfo( + &metapb.Region{ + Id: 2, + }, + &metapb.Peer{ + Id: 2, + StoreId: 2, + }, + ), + core.NewRegionInfo( + &metapb.Region{ + Id: 3, + }, + &metapb.Peer{ + Id: 3, + StoreId: 3, + }, + ), + } +} + +func (suite *balanceSchedulerPlanAnalyzeTestSuite) TearDownSuite() { + suite.cancel() +} + +func (suite *balanceSchedulerPlanAnalyzeTestSuite) TestAnalyzerResult1() { + plans := make([]Plan, 0) + plans = append(plans, &BalanceSchedulerPlan{Source: suite.stores[4], Step: 2, Target: suite.stores[0], Status: NewStatus(StatusStoreScoreDisallowed)}) + plans = append(plans, &BalanceSchedulerPlan{Source: suite.stores[4], Step: 2, Target: suite.stores[1], Status: NewStatus(StatusStoreScoreDisallowed)}) + plans = append(plans, &BalanceSchedulerPlan{Source: suite.stores[4], Step: 2, Target: suite.stores[2], Status: NewStatus(StatusStoreScoreDisallowed)}) + plans = append(plans, &BalanceSchedulerPlan{Source: suite.stores[4], Step: 2, Target: suite.stores[3], Status: NewStatus(StatusStoreNotMatchRule)}) + plans = append(plans, &BalanceSchedulerPlan{Source: suite.stores[4], Step: 2, Target: suite.stores[4], Status: NewStatus(StatusStoreAlreadyHasPeer)}) + plans = append(plans, &BalanceSchedulerPlan{Source: suite.stores[3], Step: 2, Target: suite.stores[0], Status: NewStatus(StatusStoreNotMatchRule)}) + plans = append(plans, &BalanceSchedulerPlan{Source: suite.stores[3], Step: 2, Target: suite.stores[1], Status: NewStatus(StatusStoreNotMatchRule)}) + plans = append(plans, &BalanceSchedulerPlan{Source: suite.stores[3], Step: 2, Target: suite.stores[2], Status: NewStatus(StatusStoreNotMatchRule)}) + plans = append(plans, &BalanceSchedulerPlan{Source: suite.stores[3], Step: 2, Target: suite.stores[3], Status: NewStatus(StatusStoreAlreadyHasPeer)}) + plans = append(plans, &BalanceSchedulerPlan{Source: suite.stores[3], Step: 2, Target: suite.stores[4], Status: NewStatus(StatusStoreNotMatchRule)}) + plans = append(plans, &BalanceSchedulerPlan{Source: suite.stores[2], Step: 2, Target: suite.stores[0], Status: NewStatus(StatusStoreScoreDisallowed)}) + plans = append(plans, &BalanceSchedulerPlan{Source: suite.stores[2], Step: 2, Target: suite.stores[1], Status: NewStatus(StatusStoreScoreDisallowed)}) + plans = append(plans, &BalanceSchedulerPlan{Source: suite.stores[2], Step: 2, Target: suite.stores[2], Status: NewStatus(StatusStoreAlreadyHasPeer)}) + plans = append(plans, &BalanceSchedulerPlan{Source: suite.stores[2], Step: 2, Target: suite.stores[3], Status: NewStatus(StatusStoreNotMatchRule)}) + plans = append(plans, &BalanceSchedulerPlan{Source: suite.stores[2], Step: 2, Target: suite.stores[4], Status: NewStatus(StatusStoreScoreDisallowed)}) + plans = append(plans, &BalanceSchedulerPlan{Source: suite.stores[1], Step: 2, Target: suite.stores[0], Status: NewStatus(StatusStoreScoreDisallowed)}) + plans = append(plans, &BalanceSchedulerPlan{Source: suite.stores[1], Step: 2, Target: suite.stores[1], Status: NewStatus(StatusStoreAlreadyHasPeer)}) + plans = append(plans, &BalanceSchedulerPlan{Source: suite.stores[1], Step: 2, Target: suite.stores[2], Status: NewStatus(StatusStoreScoreDisallowed)}) + plans = append(plans, &BalanceSchedulerPlan{Source: suite.stores[1], Step: 2, Target: suite.stores[3], Status: NewStatus(StatusStoreNotMatchRule)}) + plans = append(plans, &BalanceSchedulerPlan{Source: suite.stores[1], Step: 2, Target: suite.stores[4], Status: NewStatus(StatusStoreScoreDisallowed)}) + plans = append(plans, &BalanceSchedulerPlan{Source: suite.stores[0], Step: 2, Target: suite.stores[0], Status: NewStatus(StatusStoreAlreadyHasPeer)}) + plans = append(plans, &BalanceSchedulerPlan{Source: suite.stores[0], Step: 2, Target: suite.stores[1], Status: NewStatus(StatusStoreScoreDisallowed)}) + plans = append(plans, &BalanceSchedulerPlan{Source: suite.stores[0], Step: 2, Target: suite.stores[2], Status: NewStatus(StatusStoreScoreDisallowed)}) + plans = append(plans, &BalanceSchedulerPlan{Source: suite.stores[0], Step: 2, Target: suite.stores[3], Status: NewStatus(StatusStoreNotMatchRule)}) + plans = append(plans, &BalanceSchedulerPlan{Source: suite.stores[0], Step: 2, Target: suite.stores[4], Status: NewStatus(StatusStoreScoreDisallowed)}) + statuses, isNormal, err := BalancePlanSummary(plans) + suite.NoError(err) + suite.True(isNormal) + suite.True(suite.check(statuses, + map[uint64]*Status{ + 1: NewStatus(StatusStoreNotMatchRule), + 2: NewStatus(StatusStoreNotMatchRule), + 3: NewStatus(StatusStoreNotMatchRule), + 4: NewStatus(StatusStoreNotMatchRule), + 5: NewStatus(StatusStoreNotMatchRule), + })) +} + +func (suite *balanceSchedulerPlanAnalyzeTestSuite) TestAnalyzerResult2() { + plans := make([]Plan, 0) + plans = append(plans, &BalanceSchedulerPlan{Source: suite.stores[4], Step: 0, Status: NewStatus(StatusStoreDown)}) + plans = append(plans, &BalanceSchedulerPlan{Source: suite.stores[3], Step: 0, Status: NewStatus(StatusStoreDown)}) + plans = append(plans, &BalanceSchedulerPlan{Source: suite.stores[2], Step: 0, Status: NewStatus(StatusStoreDown)}) + plans = append(plans, &BalanceSchedulerPlan{Source: suite.stores[1], Step: 0, Status: NewStatus(StatusStoreDown)}) + plans = append(plans, &BalanceSchedulerPlan{Source: suite.stores[0], Step: 0, Status: NewStatus(StatusStoreDown)}) + statuses, isNormal, err := BalancePlanSummary(plans) + suite.NoError(err) + suite.False(isNormal) + suite.True(suite.check(statuses, + map[uint64]*Status{ + 1: NewStatus(StatusStoreDown), + 2: NewStatus(StatusStoreDown), + 3: NewStatus(StatusStoreDown), + 4: NewStatus(StatusStoreDown), + 5: NewStatus(StatusStoreDown), + })) +} + +func (suite *balanceSchedulerPlanAnalyzeTestSuite) TestAnalyzerResult3() { + plans := make([]Plan, 0) + plans = append(plans, &BalanceSchedulerPlan{Source: suite.stores[4], Step: 0, Status: NewStatus(StatusStoreDown)}) + plans = append(plans, &BalanceSchedulerPlan{Source: suite.stores[3], Region: suite.regions[0], Step: 1, Status: NewStatus(StatusRegionNotMatchRule)}) + plans = append(plans, &BalanceSchedulerPlan{Source: suite.stores[2], Region: suite.regions[0], Step: 1, Status: NewStatus(StatusRegionNotMatchRule)}) + plans = append(plans, &BalanceSchedulerPlan{Source: suite.stores[1], Region: suite.regions[1], Step: 1, Status: NewStatus(StatusRegionNotMatchRule)}) + plans = append(plans, &BalanceSchedulerPlan{Source: suite.stores[0], Region: suite.regions[1], Step: 1, Status: NewStatus(StatusRegionNotMatchRule)}) + statuses, isNormal, err := BalancePlanSummary(plans) + suite.NoError(err) + suite.False(isNormal) + suite.True(suite.check(statuses, + map[uint64]*Status{ + 1: NewStatus(StatusRegionNotMatchRule), + 2: NewStatus(StatusRegionNotMatchRule), + 3: NewStatus(StatusRegionNotMatchRule), + 4: NewStatus(StatusRegionNotMatchRule), + })) +} + +func (suite *balanceSchedulerPlanAnalyzeTestSuite) TestAnalyzerResult4() { + plans := make([]Plan, 0) + plans = append(plans, &BalanceSchedulerPlan{Source: suite.stores[4], Step: 0, Status: NewStatus(StatusStoreDown)}) + plans = append(plans, &BalanceSchedulerPlan{Source: suite.stores[3], Region: suite.regions[0], Step: 1, Status: NewStatus(StatusRegionNotMatchRule)}) + plans = append(plans, &BalanceSchedulerPlan{Source: suite.stores[2], Region: suite.regions[0], Step: 1, Status: NewStatus(StatusRegionNotMatchRule)}) + plans = append(plans, &BalanceSchedulerPlan{Source: suite.stores[1], Target: suite.stores[0], Step: 2, Status: NewStatus(StatusStoreScoreDisallowed)}) + plans = append(plans, &BalanceSchedulerPlan{Source: suite.stores[1], Target: suite.stores[1], Step: 2, Status: NewStatus(StatusStoreAlreadyHasPeer)}) + plans = append(plans, &BalanceSchedulerPlan{Source: suite.stores[1], Target: suite.stores[2], Step: 2, Status: NewStatus(StatusStoreNotMatchRule)}) + plans = append(plans, &BalanceSchedulerPlan{Source: suite.stores[1], Target: suite.stores[3], Step: 2, Status: NewStatus(StatusStoreNotMatchRule)}) + plans = append(plans, &BalanceSchedulerPlan{Source: suite.stores[1], Target: suite.stores[4], Step: 2, Status: NewStatus(StatusStoreDown)}) + plans = append(plans, &BalanceSchedulerPlan{Source: suite.stores[0], Target: suite.stores[0], Step: 2, Status: NewStatus(StatusStoreAlreadyHasPeer)}) + plans = append(plans, &BalanceSchedulerPlan{Source: suite.stores[0], Target: suite.stores[1], Step: 2, Status: NewStatus(StatusStoreScoreDisallowed)}) + plans = append(plans, &BalanceSchedulerPlan{Source: suite.stores[0], Target: suite.stores[2], Step: 2, Status: NewStatus(StatusStoreNotMatchRule)}) + plans = append(plans, &BalanceSchedulerPlan{Source: suite.stores[0], Target: suite.stores[3], Step: 2, Status: NewStatus(StatusStoreNotMatchRule)}) + plans = append(plans, &BalanceSchedulerPlan{Source: suite.stores[0], Target: suite.stores[4], Step: 2, Status: NewStatus(StatusStoreDown)}) + statuses, isNormal, err := BalancePlanSummary(plans) + suite.NoError(err) + suite.False(isNormal) + suite.True(suite.check(statuses, + map[uint64]*Status{ + 1: NewStatus(StatusStoreAlreadyHasPeer), + 2: NewStatus(StatusStoreAlreadyHasPeer), + 3: NewStatus(StatusStoreNotMatchRule), + 4: NewStatus(StatusStoreNotMatchRule), + 5: NewStatus(StatusStoreDown), + })) +} + +func (suite *balanceSchedulerPlanAnalyzeTestSuite) TestAnalyzerResult5() { + plans := make([]Plan, 0) + plans = append(plans, &BalanceSchedulerPlan{Source: suite.stores[4], Step: 0, Status: NewStatus(StatusStoreRemoveLimitThrottled)}) + plans = append(plans, &BalanceSchedulerPlan{Source: suite.stores[3], Region: suite.regions[0], Step: 1, Status: NewStatus(StatusRegionNotMatchRule)}) + plans = append(plans, &BalanceSchedulerPlan{Source: suite.stores[2], Region: suite.regions[0], Step: 1, Status: NewStatus(StatusRegionNotMatchRule)}) + plans = append(plans, &BalanceSchedulerPlan{Source: suite.stores[1], Target: suite.stores[0], Step: 2, Status: NewStatus(StatusStoreScoreDisallowed)}) + plans = append(plans, &BalanceSchedulerPlan{Source: suite.stores[1], Target: suite.stores[1], Step: 2, Status: NewStatus(StatusStoreAlreadyHasPeer)}) + plans = append(plans, &BalanceSchedulerPlan{Source: suite.stores[1], Target: suite.stores[2], Step: 2, Status: NewStatus(StatusStoreNotMatchRule)}) + plans = append(plans, &BalanceSchedulerPlan{Source: suite.stores[1], Target: suite.stores[3], Step: 2, Status: NewStatus(StatusStoreNotMatchRule)}) + plans = append(plans, &BalanceSchedulerPlan{Source: suite.stores[0], Target: suite.stores[0], Step: 2, Status: NewStatus(StatusStoreAlreadyHasPeer)}) + plans = append(plans, &BalanceSchedulerPlan{Source: suite.stores[0], Target: suite.stores[1], Step: 3, Status: NewStatus(StatusStoreScoreDisallowed)}) + plans = append(plans, &BalanceSchedulerPlan{Source: suite.stores[0], Target: suite.stores[2], Step: 2, Status: NewStatus(StatusStoreNotMatchRule)}) + plans = append(plans, &BalanceSchedulerPlan{Source: suite.stores[0], Target: suite.stores[3], Step: 2, Status: NewStatus(StatusStoreNotMatchRule)}) + statuses, isNormal, err := BalancePlanSummary(plans) + suite.NoError(err) + suite.False(isNormal) + suite.True(suite.check(statuses, + map[uint64]*Status{ + 1: NewStatus(StatusStoreAlreadyHasPeer), + 2: NewStatus(StatusStoreAlreadyHasPeer), + 3: NewStatus(StatusStoreNotMatchRule), + 4: NewStatus(StatusStoreNotMatchRule), + 5: NewStatus(StatusStoreRemoveLimitThrottled), + })) +} + +func (suite *balanceSchedulerPlanAnalyzeTestSuite) TestAnalyzerResult6() { + basePlan := NewBalanceSchedulerPlan() + collector := NewCollector(basePlan) + collector.Collect(SetResourceWithStep(suite.stores[0], 2), SetStatus(NewStatus(StatusStoreDown))) + collector.Collect(SetResourceWithStep(suite.stores[1], 2), SetStatus(NewStatus(StatusStoreDown))) + collector.Collect(SetResourceWithStep(suite.stores[2], 2), SetStatus(NewStatus(StatusStoreDown))) + collector.Collect(SetResourceWithStep(suite.stores[3], 2), SetStatus(NewStatus(StatusStoreDown))) + collector.Collect(SetResourceWithStep(suite.stores[4], 2), SetStatus(NewStatus(StatusStoreDown))) + basePlan.Source = suite.stores[0] + basePlan.Step++ + collector.Collect(SetResource(suite.regions[0]), SetStatus(NewStatus(StatusRegionNoLeader))) + statuses, isNormal, err := BalancePlanSummary(collector.GetPlans()) + suite.NoError(err) + suite.False(isNormal) + suite.True(suite.check(statuses, + map[uint64]*Status{ + 1: NewStatus(StatusStoreDown), + 2: NewStatus(StatusStoreDown), + 3: NewStatus(StatusStoreDown), + 4: NewStatus(StatusStoreDown), + 5: NewStatus(StatusStoreDown), + })) +} diff --git a/pkg/schedule/plan/status.go b/pkg/schedule/plan/status.go index abd8c1256c5..4242b631493 100644 --- a/pkg/schedule/plan/status.go +++ b/pkg/schedule/plan/status.go @@ -29,7 +29,7 @@ const ( StatusStoreScoreDisallowed = iota + 100 // StatusStoreAlreadyHasPeer represents the store is excluded due to the existed region peer. StatusStoreAlreadyHasPeer - // StatusNotMatchRule represents the placement rule cannot satisfy the requirement. + // StatusStoreNotMatchRule represents the placement rule cannot satisfy the requirement. StatusStoreNotMatchRule ) @@ -92,6 +92,8 @@ const ( StatusNoTargetRegion // StatusRegionLabelReject represents the plan conflicts with region label. StatusRegionLabelReject + // StatusRegionSendSnapshotThrottled represents the plan conflicts with send snapshot. + StatusRegionSendSnapshotThrottled ) const ( diff --git a/server/cluster/prepare_checker.go b/pkg/schedule/prepare_checker.go similarity index 81% rename from server/cluster/prepare_checker.go rename to pkg/schedule/prepare_checker.go index 39d4cab1f91..cd4b076b2d6 100644 --- a/server/cluster/prepare_checker.go +++ b/pkg/schedule/prepare_checker.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package cluster +package schedule import ( "time" @@ -48,7 +48,7 @@ func (checker *prepareChecker) check(c *core.BasicCluster) bool { return true } // The number of active regions should be more than total region of all stores * collectFactor - if float64(c.GetRegionCount())*collectFactor > float64(checker.sum) { + if float64(c.GetTotalRegionCount())*collectFactor > float64(checker.sum) { return false } for _, store := range c.GetStores() { @@ -65,7 +65,7 @@ func (checker *prepareChecker) check(c *core.BasicCluster) bool { return true } -func (checker *prepareChecker) collect(region *core.RegionInfo) { +func (checker *prepareChecker) Collect(region *core.RegionInfo) { checker.Lock() defer checker.Unlock() for _, p := range region.GetPeers() { @@ -74,8 +74,22 @@ func (checker *prepareChecker) collect(region *core.RegionInfo) { checker.sum++ } -func (checker *prepareChecker) isPrepared() bool { +func (checker *prepareChecker) IsPrepared() bool { checker.RLock() defer checker.RUnlock() return checker.prepared } + +// for test purpose +func (checker *prepareChecker) SetPrepared() { + checker.Lock() + defer checker.Unlock() + checker.prepared = true +} + +// for test purpose +func (checker *prepareChecker) GetSum() int { + checker.RLock() + defer checker.RUnlock() + return checker.sum +} diff --git a/pkg/schedule/scatter/metrics.go b/pkg/schedule/scatter/metrics.go new file mode 100644 index 00000000000..5fe41550764 --- /dev/null +++ b/pkg/schedule/scatter/metrics.go @@ -0,0 +1,40 @@ +// Copyright 2023 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scatter + +import "github.com/prometheus/client_golang/prometheus" + +var ( + scatterCounter = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "pd", + Subsystem: "schedule", + Name: "scatter_operators_count", + Help: "Counter of region scatter operators.", + }, []string{"type", "event"}) + + scatterDistributionCounter = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "pd", + Subsystem: "schedule", + Name: "scatter_distribution", + Help: "Counter of the distribution in scatter.", + }, []string{"store", "is_leader", "engine"}) +) + +func init() { + prometheus.MustRegister(scatterCounter) + prometheus.MustRegister(scatterDistributionCounter) +} diff --git a/pkg/schedule/region_scatterer.go b/pkg/schedule/scatter/region_scatterer.go similarity index 93% rename from pkg/schedule/region_scatterer.go rename to pkg/schedule/scatter/region_scatterer.go index b5c671a3752..75b06316294 100644 --- a/pkg/schedule/region_scatterer.go +++ b/pkg/schedule/scatter/region_scatterer.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package schedule +package scatter import ( "context" @@ -29,6 +29,7 @@ import ( "github.com/tikv/pd/pkg/core" "github.com/tikv/pd/pkg/core/constant" "github.com/tikv/pd/pkg/errs" + sche "github.com/tikv/pd/pkg/schedule/core" "github.com/tikv/pd/pkg/schedule/filter" "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/placement" @@ -53,6 +54,12 @@ var ( scatterSuccessCounter = scatterCounter.WithLabelValues("success", "") ) +const ( + maxSleepDuration = time.Minute + initialSleepDuration = 100 * time.Millisecond + maxRetryLimit = 30 +) + type selectedStores struct { mu syncutil.RWMutex groupDistribution *cache.TTLString // value type: map[uint64]uint64, group -> StoreID -> count @@ -131,15 +138,15 @@ func (s *selectedStores) getDistributionByGroupLocked(group string) (map[uint64] type RegionScatterer struct { ctx context.Context name string - cluster Cluster + cluster sche.ScatterCluster ordinaryEngine engineContext specialEngines sync.Map - opController *OperatorController + opController *operator.Controller } // NewRegionScatterer creates a region scatterer. // RegionScatter is used for the `Lightning`, it will scatter the specified regions before import data. -func NewRegionScatterer(ctx context.Context, cluster Cluster, opController *OperatorController) *RegionScatterer { +func NewRegionScatterer(ctx context.Context, cluster sche.ScatterCluster, opController *operator.Controller) *RegionScatterer { return &RegionScatterer{ ctx: ctx, name: regionScatterName, @@ -161,7 +168,7 @@ type engineContext struct { func newEngineContext(ctx context.Context, filterFuncs ...filterFunc) engineContext { filterFuncs = append(filterFuncs, func() filter.Filter { - return &filter.StoreStateFilter{ActionScope: regionScatterName, MoveRegion: true, ScatterRegion: true} + return &filter.StoreStateFilter{ActionScope: regionScatterName, MoveRegion: true, ScatterRegion: true, OperatorLevel: constant.High} }) return engineContext{ filterFuncs: filterFuncs, @@ -170,10 +177,6 @@ func newEngineContext(ctx context.Context, filterFuncs ...filterFunc) engineCont } } -const maxSleepDuration = time.Minute -const initialSleepDuration = 100 * time.Millisecond -const maxRetryLimit = 30 - // ScatterRegionsByRange directly scatter regions by ScatterRegions func (r *RegionScatterer) ScatterRegionsByRange(startKey, endKey []byte, group string, retryLimit int) (int, map[uint64]error, error) { regions := r.cluster.ScanRegions(startKey, endKey, -1) @@ -187,7 +190,7 @@ func (r *RegionScatterer) ScatterRegionsByRange(startKey, endKey []byte, group s regionMap[region.GetID()] = region } // If there existed any region failed to relocated after retry, add it into unProcessedRegions - opsCount, err := r.scatterRegions(regionMap, failures, group, retryLimit) + opsCount, err := r.scatterRegions(regionMap, failures, group, retryLimit, false) if err != nil { return 0, nil, err } @@ -195,7 +198,7 @@ func (r *RegionScatterer) ScatterRegionsByRange(startKey, endKey []byte, group s } // ScatterRegionsByID directly scatter regions by ScatterRegions -func (r *RegionScatterer) ScatterRegionsByID(regionsID []uint64, group string, retryLimit int) (int, map[uint64]error, error) { +func (r *RegionScatterer) ScatterRegionsByID(regionsID []uint64, group string, retryLimit int, skipStoreLimit bool) (int, map[uint64]error, error) { if len(regionsID) < 1 { scatterSkipEmptyRegionCounter.Inc() return 0, nil, errors.New("empty region") @@ -217,7 +220,7 @@ func (r *RegionScatterer) ScatterRegionsByID(regionsID []uint64, group string, r regionMap[region.GetID()] = region } // If there existed any region failed to relocated after retry, add it into unProcessedRegions - opsCount, err := r.scatterRegions(regionMap, failures, group, retryLimit) + opsCount, err := r.scatterRegions(regionMap, failures, group, retryLimit, skipStoreLimit) if err != nil { return 0, nil, err } @@ -230,7 +233,7 @@ func (r *RegionScatterer) ScatterRegionsByID(regionsID []uint64, group string, r // time.Sleep between each retry. // Failures indicates the regions which are failed to be relocated, the key of the failures indicates the regionID // and the value of the failures indicates the failure error. -func (r *RegionScatterer) scatterRegions(regions map[uint64]*core.RegionInfo, failures map[uint64]error, group string, retryLimit int) (int, error) { +func (r *RegionScatterer) scatterRegions(regions map[uint64]*core.RegionInfo, failures map[uint64]error, group string, retryLimit int, skipStoreLimit bool) (int, error) { if len(regions) < 1 { scatterSkipEmptyRegionCounter.Inc() return 0, errors.New("empty region") @@ -241,7 +244,7 @@ func (r *RegionScatterer) scatterRegions(regions map[uint64]*core.RegionInfo, fa opsCount := 0 for currentRetry := 0; currentRetry <= retryLimit; currentRetry++ { for _, region := range regions { - op, err := r.Scatter(region, group) + op, err := r.Scatter(region, group, skipStoreLimit) failpoint.Inject("scatterFail", func() { if region.GetID() == 1 { err = errors.New("mock error") @@ -260,8 +263,8 @@ func (r *RegionScatterer) scatterRegions(regions map[uint64]*core.RegionInfo, fa continue } failpoint.Inject("scatterHbStreamsDrain", func() { - r.opController.hbStreams.Drain(1) - r.opController.RemoveOperator(op) + r.opController.GetHBStreams().Drain(1) + r.opController.RemoveOperator(op, operator.AdminStop) }) } delete(failures, region.GetID()) @@ -278,7 +281,7 @@ func (r *RegionScatterer) scatterRegions(regions map[uint64]*core.RegionInfo, fa // Scatter relocates the region. If the group is defined, the regions' leader with the same group would be scattered // in a group level instead of cluster level. -func (r *RegionScatterer) Scatter(region *core.RegionInfo, group string) (*operator.Operator, error) { +func (r *RegionScatterer) Scatter(region *core.RegionInfo, group string, skipStoreLimit bool) (*operator.Operator, error) { if !filter.IsRegionReplicated(r.cluster, region) { r.cluster.AddSuspectRegions(region.GetID()) scatterSkipNotReplicatedCounter.Inc() @@ -298,10 +301,10 @@ func (r *RegionScatterer) Scatter(region *core.RegionInfo, group string) (*opera return nil, errors.Errorf("region %d is hot", region.GetID()) } - return r.scatterRegion(region, group), nil + return r.scatterRegion(region, group, skipStoreLimit), nil } -func (r *RegionScatterer) scatterRegion(region *core.RegionInfo, group string) *operator.Operator { +func (r *RegionScatterer) scatterRegion(region *core.RegionInfo, group string, skipStoreLimit bool) *operator.Operator { engineFilter := filter.NewEngineFilter(r.name, filter.NotSpecialEngines) ordinaryPeers := make(map[uint64]*metapb.Peer, len(region.GetPeers())) specialPeers := make(map[string]map[uint64]*metapb.Peer) @@ -312,7 +315,7 @@ func (r *RegionScatterer) scatterRegion(region *core.RegionInfo, group string) * if store == nil { return nil } - if engineFilter.Target(r.cluster.GetOpts(), store).IsOK() { + if engineFilter.Target(r.cluster.GetSharedConfig(), store).IsOK() { ordinaryPeers[peer.GetStoreId()] = peer } else { engine := store.GetLabelValue(core.EngineKey) @@ -380,7 +383,7 @@ func (r *RegionScatterer) scatterRegion(region *core.RegionInfo, group string) * r.Put(targetPeers, targetLeader, group) return nil } - op, err := operator.CreateScatterRegionOperator("scatter-region", r.cluster, region, targetPeers, targetLeader) + op, err := operator.CreateScatterRegionOperator("scatter-region", r.cluster, region, targetPeers, targetLeader, skipStoreLimit) if err != nil { scatterFailCounter.Inc() for _, peer := range region.GetPeers() { @@ -407,10 +410,7 @@ func allowLeader(fit *placement.RegionFit, peer *metapb.Peer) bool { return false } peerFit := fit.GetRuleFit(peer.GetId()) - if peerFit == nil || peerFit.Rule == nil { - return false - } - if peerFit.Rule.IsWitness { + if peerFit == nil || peerFit.Rule == nil || peerFit.Rule.IsWitness { return false } switch peerFit.Rule.Role { @@ -439,7 +439,7 @@ func (r *RegionScatterer) selectCandidates(region *core.RegionInfo, oldFit *plac filters := []filter.Filter{ filter.NewExcludedFilter(r.name, nil, selectedStores), } - scoreGuard := filter.NewPlacementSafeguard(r.name, r.cluster.GetOpts(), r.cluster.GetBasicCluster(), r.cluster.GetRuleManager(), region, sourceStore, oldFit) + scoreGuard := filter.NewPlacementSafeguard(r.name, r.cluster.GetSharedConfig(), r.cluster.GetBasicCluster(), r.cluster.GetRuleManager(), region, sourceStore, oldFit) for _, filterFunc := range context.filterFuncs { filters = append(filters, filterFunc()) } @@ -463,7 +463,7 @@ func (r *RegionScatterer) selectCandidates(region *core.RegionInfo, oldFit *plac // If the storeCount are all the same for the whole cluster(maxStoreTotalCount == minStoreTotalCount), any store // could be selected as candidate. if storeCount < maxStoreTotalCount || maxStoreTotalCount == minStoreTotalCount { - if filter.Target(r.cluster.GetOpts(), store, filters) { + if filter.Target(r.cluster.GetSharedConfig(), store, filters) { candidates = append(candidates, store.GetID()) } } @@ -533,7 +533,7 @@ func (r *RegionScatterer) Put(peers map[uint64]*metapb.Peer, leaderStoreID uint6 if store == nil { continue } - if engineFilter.Target(r.cluster.GetOpts(), store).IsOK() { + if engineFilter.Target(r.cluster.GetSharedConfig(), store).IsOK() { r.ordinaryEngine.selectedPeer.Put(storeID, group) scatterDistributionCounter.WithLabelValues( fmt.Sprintf("%v", storeID), diff --git a/pkg/schedule/region_scatterer_test.go b/pkg/schedule/scatter/region_scatterer_test.go similarity index 87% rename from pkg/schedule/region_scatterer_test.go rename to pkg/schedule/scatter/region_scatterer_test.go index a602598b4b9..f4ee5953e6f 100644 --- a/pkg/schedule/region_scatterer_test.go +++ b/pkg/schedule/scatter/region_scatterer_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package schedule +package scatter import ( "context" @@ -28,6 +28,7 @@ import ( "github.com/pingcap/kvproto/pkg/metapb" "github.com/stretchr/testify/require" "github.com/tikv/pd/pkg/core" + "github.com/tikv/pd/pkg/core/storelimit" "github.com/tikv/pd/pkg/mock/mockcluster" "github.com/tikv/pd/pkg/mock/mockconfig" "github.com/tikv/pd/pkg/schedule/hbstream" @@ -91,7 +92,7 @@ func scatter(re *require.Assertions, numStores, numRegions uint64, useRules bool opt := mockconfig.NewTestOptions() tc := mockcluster.NewCluster(ctx, opt) stream := hbstream.NewTestHeartbeatStreams(ctx, tc.ID, tc, false) - oc := NewOperatorController(ctx, tc, stream) + oc := operator.NewController(ctx, tc.GetBasicCluster(), tc.GetSharedConfig(), stream) tc.SetClusterVersion(versioninfo.MinSupportedVersion(versioninfo.Version4_0)) // Add ordinary stores. @@ -108,9 +109,9 @@ func scatter(re *require.Assertions, numStores, numRegions uint64, useRules bool for i := uint64(1); i <= numRegions; i++ { region := tc.GetRegion(i) - if op, _ := scatterer.Scatter(region, ""); op != nil { + if op, _ := scatterer.Scatter(region, "", false); op != nil { checkOperator(re, op) - ApplyOperator(tc, op) + operator.ApplyOperator(tc, op) } } @@ -148,7 +149,7 @@ func scatterSpecial(re *require.Assertions, numOrdinaryStores, numSpecialStores, opt := mockconfig.NewTestOptions() tc := mockcluster.NewCluster(ctx, opt) stream := hbstream.NewTestHeartbeatStreams(ctx, tc.ID, tc, false) - oc := NewOperatorController(ctx, tc, stream) + oc := operator.NewController(ctx, tc.GetBasicCluster(), tc.GetSharedConfig(), stream) tc.SetClusterVersion(versioninfo.MinSupportedVersion(versioninfo.Version4_0)) // Add ordinary stores. @@ -178,9 +179,9 @@ func scatterSpecial(re *require.Assertions, numOrdinaryStores, numSpecialStores, for i := uint64(1); i <= numRegions; i++ { region := tc.GetRegion(i) - if op, _ := scatterer.Scatter(region, ""); op != nil { + if op, _ := scatterer.Scatter(region, "", false); op != nil { checkOperator(re, op) - ApplyOperator(tc, op) + operator.ApplyOperator(tc, op) } } @@ -226,7 +227,7 @@ func TestStoreLimit(t *testing.T) { opt := mockconfig.NewTestOptions() tc := mockcluster.NewCluster(ctx, opt) stream := hbstream.NewTestHeartbeatStreams(ctx, tc.ID, tc, false) - oc := NewOperatorController(ctx, tc, stream) + oc := operator.NewController(ctx, tc.GetBasicCluster(), tc.GetSharedConfig(), stream) // Add stores 1~6. for i := uint64(1); i <= 5; i++ { @@ -245,7 +246,7 @@ func TestStoreLimit(t *testing.T) { for i := uint64(1); i <= 5; i++ { region := tc.GetRegion(i) - if op, _ := scatterer.Scatter(region, ""); op != nil { + if op, _ := scatterer.Scatter(region, "", false); op != nil { re.Equal(1, oc.AddWaitingOperator(op)) } } @@ -258,7 +259,7 @@ func TestScatterCheck(t *testing.T) { opt := mockconfig.NewTestOptions() tc := mockcluster.NewCluster(ctx, opt) stream := hbstream.NewTestHeartbeatStreams(ctx, tc.ID, tc, false) - oc := NewOperatorController(ctx, tc, stream) + oc := operator.NewController(ctx, tc.GetBasicCluster(), tc.GetSharedConfig(), stream) // Add 5 stores. for i := uint64(1); i <= 5; i++ { tc.AddRegionStore(i, 0) @@ -287,7 +288,7 @@ func TestScatterCheck(t *testing.T) { for _, testCase := range testCases { t.Log(testCase.name) scatterer := NewRegionScatterer(ctx, tc, oc) - _, err := scatterer.Scatter(testCase.checkRegion, "") + _, err := scatterer.Scatter(testCase.checkRegion, "", false) if testCase.needFix { re.Error(err) re.True(tc.CheckRegionUnderSuspect(1)) @@ -307,7 +308,7 @@ func TestSomeStoresFilteredScatterGroupInConcurrency(t *testing.T) { opt := mockconfig.NewTestOptions() tc := mockcluster.NewCluster(ctx, opt) stream := hbstream.NewTestHeartbeatStreams(ctx, tc.ID, tc, false) - oc := NewOperatorController(ctx, tc, stream) + oc := operator.NewController(ctx, tc.GetBasicCluster(), tc.GetSharedConfig(), stream) // Add 5 connected stores. for i := uint64(1); i <= 5; i++ { tc.AddRegionStore(i, 0) @@ -339,7 +340,7 @@ func TestSomeStoresFilteredScatterGroupInConcurrency(t *testing.T) { func scatterOnce(tc *mockcluster.Cluster, scatter *RegionScatterer, group string, wg *sync.WaitGroup) { regionID := 1 for i := 0; i < 100; i++ { - scatter.scatterRegion(tc.AddLeaderRegion(uint64(regionID), 1, 2, 3), group) + scatter.scatterRegion(tc.AddLeaderRegion(uint64(regionID), 1, 2, 3), group, false) regionID++ } wg.Done() @@ -352,7 +353,7 @@ func TestScatterGroupInConcurrency(t *testing.T) { opt := mockconfig.NewTestOptions() tc := mockcluster.NewCluster(ctx, opt) stream := hbstream.NewTestHeartbeatStreams(ctx, tc.ID, tc, false) - oc := NewOperatorController(ctx, tc, stream) + oc := operator.NewController(ctx, tc.GetBasicCluster(), tc.GetSharedConfig(), stream) // Add 5 stores. for i := uint64(1); i <= 5; i++ { tc.AddRegionStore(i, 0) @@ -386,7 +387,7 @@ func TestScatterGroupInConcurrency(t *testing.T) { for i := 0; i < 100; i++ { for j := 0; j < testCase.groupCount; j++ { scatterer.scatterRegion(tc.AddLeaderRegion(uint64(regionID), 1, 2, 3), - fmt.Sprintf("group-%v", j)) + fmt.Sprintf("group-%v", j), false) regionID++ } } @@ -424,7 +425,7 @@ func TestScatterForManyRegion(t *testing.T) { opt := mockconfig.NewTestOptions() tc := mockcluster.NewCluster(ctx, opt) stream := hbstream.NewTestHeartbeatStreams(ctx, tc.ID, tc, false) - oc := NewOperatorController(ctx, tc, stream) + oc := operator.NewController(ctx, tc.GetBasicCluster(), tc.GetSharedConfig(), stream) // Add 60 stores. for i := uint64(1); i <= 60; i++ { tc.AddRegionStore(i, 0) @@ -439,9 +440,9 @@ func TestScatterForManyRegion(t *testing.T) { } failures := map[uint64]error{} group := "group" - re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/schedule/scatterHbStreamsDrain", `return(true)`)) - scatterer.scatterRegions(regions, failures, group, 3) - re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/schedule/scatterHbStreamsDrain")) + re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/schedule/scatter/scatterHbStreamsDrain", `return(true)`)) + scatterer.scatterRegions(regions, failures, group, 3, false) + re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/schedule/scatter/scatterHbStreamsDrain")) re.Len(failures, 0) } @@ -452,7 +453,7 @@ func TestScattersGroup(t *testing.T) { opt := mockconfig.NewTestOptions() tc := mockcluster.NewCluster(ctx, opt) stream := hbstream.NewTestHeartbeatStreams(ctx, tc.ID, tc, false) - oc := NewOperatorController(ctx, tc, stream) + oc := operator.NewController(ctx, tc.GetBasicCluster(), tc.GetSharedConfig(), stream) // Add 5 stores. for i := uint64(1); i <= 5; i++ { tc.AddRegionStore(i, 0) @@ -470,7 +471,7 @@ func TestScattersGroup(t *testing.T) { failure: false, }, } - re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/schedule/scatterHbStreamsDrain", `return(true)`)) + re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/schedule/scatter/scatterHbStreamsDrain", `return(true)`)) for id, testCase := range testCases { group := fmt.Sprintf("gourp-%d", id) t.Log(testCase.name) @@ -481,10 +482,10 @@ func TestScattersGroup(t *testing.T) { } failures := map[uint64]error{} if testCase.failure { - re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/schedule/scatterFail", `return(true)`)) + re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/schedule/scatter/scatterFail", `return(true)`)) } - scatterer.scatterRegions(regions, failures, group, 3) + scatterer.scatterRegions(regions, failures, group, 3, false) max := uint64(0) min := uint64(math.MaxUint64) groupDistribution, exist := scatterer.ordinaryEngine.selectedLeader.GetGroupDistribution(group) @@ -505,12 +506,12 @@ func TestScattersGroup(t *testing.T) { re.Len(failures, 1) _, ok := failures[1] re.True(ok) - re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/schedule/scatterFail")) + re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/schedule/scatter/scatterFail")) } else { re.Empty(failures) } } - re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/schedule/scatterHbStreamsDrain")) + re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/schedule/scatter/scatterHbStreamsDrain")) } func TestSelectedStoreGC(t *testing.T) { @@ -541,7 +542,7 @@ func TestRegionFromDifferentGroups(t *testing.T) { opt := mockconfig.NewTestOptions() tc := mockcluster.NewCluster(ctx, opt) stream := hbstream.NewTestHeartbeatStreams(ctx, tc.ID, tc, false) - oc := NewOperatorController(ctx, tc, stream) + oc := operator.NewController(ctx, tc.GetBasicCluster(), tc.GetSharedConfig(), stream) // Add 6 stores. storeCount := 6 for i := uint64(1); i <= uint64(storeCount); i++ { @@ -551,7 +552,7 @@ func TestRegionFromDifferentGroups(t *testing.T) { regionCount := 50 for i := 1; i <= regionCount; i++ { p := rand.Perm(storeCount) - scatterer.scatterRegion(tc.AddLeaderRegion(uint64(i), uint64(p[0])+1, uint64(p[1])+1, uint64(p[2])+1), fmt.Sprintf("t%d", i)) + scatterer.scatterRegion(tc.AddLeaderRegion(uint64(i), uint64(p[0])+1, uint64(p[1])+1, uint64(p[2])+1), fmt.Sprintf("t%d", i), false) } check := func(ss *selectedStores) { max := uint64(0) @@ -577,7 +578,7 @@ func TestRegionHasLearner(t *testing.T) { opt := mockconfig.NewTestOptions() tc := mockcluster.NewCluster(ctx, opt) stream := hbstream.NewTestHeartbeatStreams(ctx, tc.ID, tc, false) - oc := NewOperatorController(ctx, tc, stream) + oc := operator.NewController(ctx, tc.GetBasicCluster(), tc.GetSharedConfig(), stream) // Add 8 stores. voterCount := uint64(6) storeCount := uint64(8) @@ -616,7 +617,7 @@ func TestRegionHasLearner(t *testing.T) { scatterer := NewRegionScatterer(ctx, tc, oc) regionCount := 50 for i := 1; i <= regionCount; i++ { - _, err := scatterer.Scatter(tc.AddRegionWithLearner(uint64(i), uint64(1), []uint64{uint64(2), uint64(3)}, []uint64{7}), "group") + _, err := scatterer.Scatter(tc.AddRegionWithLearner(uint64(i), uint64(1), []uint64{uint64(2), uint64(3)}, []uint64{7}), "group", false) re.NoError(err) } check := func(ss *selectedStores) { @@ -665,7 +666,7 @@ func TestSelectedStoresTooFewPeers(t *testing.T) { opt := mockconfig.NewTestOptions() tc := mockcluster.NewCluster(ctx, opt) stream := hbstream.NewTestHeartbeatStreams(ctx, tc.ID, tc, false) - oc := NewOperatorController(ctx, tc, stream) + oc := operator.NewController(ctx, tc.GetBasicCluster(), tc.GetSharedConfig(), stream) // Add 4 stores. for i := uint64(1); i <= 4; i++ { tc.AddRegionStore(i, 0) @@ -688,7 +689,7 @@ func TestSelectedStoresTooFewPeers(t *testing.T) { // Try to scatter a region with peer store id 2/3/4 for i := uint64(1); i < 20; i++ { region := tc.AddLeaderRegion(i+200, i%3+2, (i+1)%3+2, (i+2)%3+2) - op := scatterer.scatterRegion(region, group) + op := scatterer.scatterRegion(region, group, false) re.False(isPeerCountChanged(op)) } } @@ -702,7 +703,7 @@ func TestSelectedStoresTooManyPeers(t *testing.T) { opt := mockconfig.NewTestOptions() tc := mockcluster.NewCluster(ctx, opt) stream := hbstream.NewTestHeartbeatStreams(ctx, tc.ID, tc, false) - oc := NewOperatorController(ctx, tc, stream) + oc := operator.NewController(ctx, tc.GetBasicCluster(), tc.GetSharedConfig(), stream) // Add 4 stores. for i := uint64(1); i <= 5; i++ { tc.AddRegionStore(i, 0) @@ -725,7 +726,7 @@ func TestSelectedStoresTooManyPeers(t *testing.T) { // test region with peer 1 2 3 for i := uint64(1); i < 20; i++ { region := tc.AddLeaderRegion(i+200, i%3+1, (i+1)%3+1, (i+2)%3+1) - op := scatterer.scatterRegion(region, group) + op := scatterer.scatterRegion(region, group, false) re.False(isPeerCountChanged(op)) } } @@ -740,7 +741,7 @@ func TestBalanceRegion(t *testing.T) { opt.SetLocationLabels([]string{"host"}) tc := mockcluster.NewCluster(ctx, opt) stream := hbstream.NewTestHeartbeatStreams(ctx, tc.ID, tc, false) - oc := NewOperatorController(ctx, tc, stream) + oc := operator.NewController(ctx, tc.GetBasicCluster(), tc.GetSharedConfig(), stream) // Add 6 stores in 3 hosts. for i := uint64(2); i <= 7; i++ { tc.AddLabelsStore(i, 0, map[string]string{"host": strconv.FormatUint(i/2, 10)}) @@ -751,7 +752,7 @@ func TestBalanceRegion(t *testing.T) { scatterer := NewRegionScatterer(ctx, tc, oc) for i := uint64(1001); i <= 1300; i++ { region := tc.AddLeaderRegion(i, 2, 4, 6) - op := scatterer.scatterRegion(region, group) + op := scatterer.scatterRegion(region, group, false) re.False(isPeerCountChanged(op)) } for i := uint64(2); i <= 7; i++ { @@ -761,7 +762,7 @@ func TestBalanceRegion(t *testing.T) { // Test for unhealthy region // ref https://github.com/tikv/pd/issues/6099 region := tc.AddLeaderRegion(1500, 2, 3, 4, 6) - op := scatterer.scatterRegion(region, group) + op := scatterer.scatterRegion(region, group, false) re.False(isPeerCountChanged(op)) } @@ -781,3 +782,37 @@ func isPeerCountChanged(op *operator.Operator) bool { } return add != remove } + +func TestRemoveStoreLimit(t *testing.T) { + re := require.New(t) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + opt := mockconfig.NewTestOptions() + tc := mockcluster.NewCluster(ctx, opt) + stream := hbstream.NewTestHeartbeatStreams(ctx, tc.ID, tc, false) + oc := operator.NewController(ctx, tc.GetBasicCluster(), tc.GetSharedConfig(), stream) + + // Add stores 1~6. + for i := uint64(1); i <= 5; i++ { + tc.AddRegionStore(i, 0) + tc.SetStoreLimit(i, storelimit.AddPeer, 1) + tc.SetStoreLimit(i, storelimit.RemovePeer, 1) + } + + // Add regions 1~4. + seq := newSequencer(3) + // Region 1 has the same distribution with the Region 2, which is used to test selectPeerToReplace. + tc.AddLeaderRegion(1, 1, 2, 3) + for i := uint64(2); i <= 5; i++ { + tc.AddLeaderRegion(i, seq.next(), seq.next(), seq.next()) + } + + scatterer := NewRegionScatterer(ctx, tc, oc) + + for i := uint64(1); i <= 5; i++ { + region := tc.GetRegion(i) + if op, _ := scatterer.Scatter(region, "", true); op != nil { + re.True(oc.AddOperator(op)) + } + } +} diff --git a/pkg/schedule/schedulers/balance_benchmark_test.go b/pkg/schedule/schedulers/balance_benchmark_test.go index 30691b0188b..694d5edb658 100644 --- a/pkg/schedule/schedulers/balance_benchmark_test.go +++ b/pkg/schedule/schedulers/balance_benchmark_test.go @@ -22,7 +22,6 @@ import ( "github.com/stretchr/testify/assert" "github.com/tikv/pd/pkg/mock/mockcluster" "github.com/tikv/pd/pkg/mock/mockconfig" - "github.com/tikv/pd/pkg/schedule" "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/placement" "github.com/tikv/pd/pkg/schedule/plan" @@ -42,12 +41,12 @@ var ( // newBenchCluster store region count is same with storeID and // the tolerate define storeCount that store can elect candidate but not should balance // so the case bench the worst scene -func newBenchCluster(ruleEnable, labelEnable bool, tombstoneEnable bool) (context.CancelFunc, *mockcluster.Cluster, *schedule.OperatorController) { +func newBenchCluster(ruleEnable, labelEnable bool, tombstoneEnable bool) (context.CancelFunc, *mockcluster.Cluster, *operator.Controller) { Register() ctx, cancel := context.WithCancel(context.Background()) opt := mockconfig.NewTestOptions() tc := mockcluster.NewCluster(ctx, opt) - oc := schedule.NewOperatorController(ctx, tc, nil) + oc := operator.NewController(ctx, tc.GetBasicCluster(), tc.GetSharedConfig(), nil) opt.GetScheduleConfig().TolerantSizeRatio = float64(storeCount) opt.SetPlacementRuleEnabled(ruleEnable) @@ -91,12 +90,12 @@ func newBenchCluster(ruleEnable, labelEnable bool, tombstoneEnable bool) (contex return cancel, tc, oc } -func newBenchBigCluster(storeNumInOneRack, regionNum int) (context.CancelFunc, *mockcluster.Cluster, *schedule.OperatorController) { +func newBenchBigCluster(storeNumInOneRack, regionNum int) (context.CancelFunc, *mockcluster.Cluster, *operator.Controller) { Register() ctx, cancel := context.WithCancel(context.Background()) opt := mockconfig.NewTestOptions() tc := mockcluster.NewCluster(ctx, opt) - oc := schedule.NewOperatorController(ctx, tc, nil) + oc := operator.NewController(ctx, tc.GetBasicCluster(), tc.GetSharedConfig(), nil) opt.GetScheduleConfig().TolerantSizeRatio = float64(storeCount) opt.SetPlacementRuleEnabled(true) diff --git a/pkg/schedule/schedulers/balance_leader.go b/pkg/schedule/schedulers/balance_leader.go index 71b923dcbaf..c53f147c65d 100644 --- a/pkg/schedule/schedulers/balance_leader.go +++ b/pkg/schedule/schedulers/balance_leader.go @@ -28,7 +28,7 @@ import ( "github.com/tikv/pd/pkg/core" "github.com/tikv/pd/pkg/core/constant" "github.com/tikv/pd/pkg/errs" - "github.com/tikv/pd/pkg/schedule" + sche "github.com/tikv/pd/pkg/schedule/core" "github.com/tikv/pd/pkg/schedule/filter" "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/plan" @@ -120,7 +120,7 @@ func (conf *balanceLeaderSchedulerConfig) Clone() *balanceLeaderSchedulerConfig } func (conf *balanceLeaderSchedulerConfig) persistLocked() error { - data, err := schedule.EncodeConfig(conf) + data, err := EncodeConfig(conf) if err != nil { return err } @@ -161,7 +161,6 @@ type balanceLeaderScheduler struct { name string conf *balanceLeaderSchedulerConfig handler http.Handler - opController *schedule.OperatorController filters []filter.Filter counter *prometheus.CounterVec filterCounter *filter.Counter @@ -169,7 +168,7 @@ type balanceLeaderScheduler struct { // newBalanceLeaderScheduler creates a scheduler that tends to keep leaders on // each store balanced. -func newBalanceLeaderScheduler(opController *schedule.OperatorController, conf *balanceLeaderSchedulerConfig, options ...BalanceLeaderCreateOption) schedule.Scheduler { +func newBalanceLeaderScheduler(opController *operator.Controller, conf *balanceLeaderSchedulerConfig, options ...BalanceLeaderCreateOption) Scheduler { base := NewBaseScheduler(opController) s := &balanceLeaderScheduler{ BaseScheduler: base, @@ -177,7 +176,6 @@ func newBalanceLeaderScheduler(opController *schedule.OperatorController, conf * name: BalanceLeaderName, conf: conf, handler: newBalanceLeaderHandler(conf), - opController: opController, counter: balanceLeaderCounter, filterCounter: filter.NewCounter(filter.BalanceLeader.String()), } @@ -185,7 +183,7 @@ func newBalanceLeaderScheduler(opController *schedule.OperatorController, conf * option(s) } s.filters = []filter.Filter{ - &filter.StoreStateFilter{ActionScope: s.GetName(), TransferLeader: true}, + &filter.StoreStateFilter{ActionScope: s.GetName(), TransferLeader: true, OperatorLevel: constant.High}, filter.NewSpecialUseFilter(s.GetName()), } return s @@ -223,11 +221,11 @@ func (l *balanceLeaderScheduler) GetType() string { func (l *balanceLeaderScheduler) EncodeConfig() ([]byte, error) { l.conf.mu.RLock() defer l.conf.mu.RUnlock() - return schedule.EncodeConfig(l.conf) + return EncodeConfig(l.conf) } -func (l *balanceLeaderScheduler) IsScheduleAllowed(cluster schedule.Cluster) bool { - allowed := l.opController.OperatorCount(operator.OpLeader) < cluster.GetOpts().GetLeaderScheduleLimit() +func (l *balanceLeaderScheduler) IsScheduleAllowed(cluster sche.SchedulerCluster) bool { + allowed := l.OpController.OperatorCount(operator.OpLeader) < cluster.GetSchedulerConfig().GetLeaderScheduleLimit() if !allowed { operator.OperatorLimitCounter.WithLabelValues(l.GetType(), operator.OpLeader.String()).Inc() } @@ -326,10 +324,10 @@ func (cs *candidateStores) resortStoreWithPos(pos int) { } } -func (l *balanceLeaderScheduler) Schedule(cluster schedule.Cluster, dryRun bool) ([]*operator.Operator, []plan.Plan) { +func (l *balanceLeaderScheduler) Schedule(cluster sche.SchedulerCluster, dryRun bool) ([]*operator.Operator, []plan.Plan) { l.conf.mu.RLock() defer l.conf.mu.RUnlock() - basePlan := NewBalanceSchedulerPlan() + basePlan := plan.NewBalanceSchedulerPlan() var collector *plan.Collector if dryRun { collector = plan.NewCollector(basePlan) @@ -337,8 +335,8 @@ func (l *balanceLeaderScheduler) Schedule(cluster schedule.Cluster, dryRun bool) batch := l.conf.Batch balanceLeaderScheduleCounter.Inc() - leaderSchedulePolicy := cluster.GetOpts().GetLeaderSchedulePolicy() - opInfluence := l.opController.GetOpInfluence(cluster) + leaderSchedulePolicy := cluster.GetSchedulerConfig().GetLeaderSchedulePolicy() + opInfluence := l.OpController.GetOpInfluence(cluster.GetBasicCluster()) kind := constant.NewScheduleKind(constant.LeaderKind, leaderSchedulePolicy) solver := newSolver(basePlan, kind, cluster, opInfluence) @@ -346,8 +344,8 @@ func (l *balanceLeaderScheduler) Schedule(cluster schedule.Cluster, dryRun bool) scoreFunc := func(store *core.StoreInfo) float64 { return store.LeaderScore(solver.kind.Policy, solver.GetOpInfluence(store.GetID())) } - sourceCandidate := newCandidateStores(filter.SelectSourceStores(stores, l.filters, cluster.GetOpts(), collector, l.filterCounter), false, scoreFunc) - targetCandidate := newCandidateStores(filter.SelectTargetStores(stores, l.filters, cluster.GetOpts(), nil, l.filterCounter), true, scoreFunc) + sourceCandidate := newCandidateStores(filter.SelectSourceStores(stores, l.filters, cluster.GetSchedulerConfig(), collector, l.filterCounter), false, scoreFunc) + targetCandidate := newCandidateStores(filter.SelectTargetStores(stores, l.filters, cluster.GetSchedulerConfig(), nil, l.filterCounter), true, scoreFunc) usedRegions := make(map[uint64]struct{}) result := make([]*operator.Operator, 0, batch) @@ -383,16 +381,16 @@ func (l *balanceLeaderScheduler) Schedule(cluster schedule.Cluster, dryRun bool) func createTransferLeaderOperator(cs *candidateStores, dir string, l *balanceLeaderScheduler, ssolver *solver, usedRegions map[uint64]struct{}, collector *plan.Collector) *operator.Operator { store := cs.getStore() - ssolver.step++ - defer func() { ssolver.step-- }() + ssolver.Step++ + defer func() { ssolver.Step-- }() retryLimit := l.retryQuota.GetLimit(store) var creator func(*solver, *plan.Collector) *operator.Operator switch dir { case transferOut: - ssolver.source, ssolver.target = store, nil + ssolver.Source, ssolver.Target = store, nil creator = l.transferLeaderOut case transferIn: - ssolver.source, ssolver.target = nil, store + ssolver.Source, ssolver.Target = nil, store creator = l.transferLeaderIn } var op *operator.Operator @@ -418,10 +416,10 @@ func makeInfluence(op *operator.Operator, plan *solver, usedRegions map[uint64]s usedRegions[op.RegionID()] = struct{}{} candidateUpdateStores := make([][]int, len(candidates)) for id, candidate := range candidates { - storesIDs := candidate.binarySearchStores(plan.source, plan.target) + storesIDs := candidate.binarySearchStores(plan.Source, plan.Target) candidateUpdateStores[id] = storesIDs } - schedule.AddOpInfluence(op, plan.opInfluence, plan.Cluster) + operator.AddOpInfluence(op, plan.opInfluence, plan.SchedulerCluster.GetBasicCluster()) for id, candidate := range candidates { for _, pos := range candidateUpdateStores[id] { candidate.resortStoreWithPos(pos) @@ -433,27 +431,27 @@ func makeInfluence(op *operator.Operator, plan *solver, usedRegions map[uint64]s // It randomly selects a health region from the source store, then picks // the best follower peer and transfers the leader. func (l *balanceLeaderScheduler) transferLeaderOut(solver *solver, collector *plan.Collector) *operator.Operator { - solver.region = filter.SelectOneRegion(solver.RandLeaderRegions(solver.SourceStoreID(), l.conf.Ranges), + solver.Region = filter.SelectOneRegion(solver.RandLeaderRegions(solver.SourceStoreID(), l.conf.Ranges), collector, filter.NewRegionPendingFilter(), filter.NewRegionDownFilter()) - if solver.region == nil { + if solver.Region == nil { log.Debug("store has no leader", zap.String("scheduler", l.GetName()), zap.Uint64("store-id", solver.SourceStoreID())) balanceLeaderNoLeaderRegionCounter.Inc() return nil } - if solver.IsRegionHot(solver.region) { - log.Debug("region is hot region, ignore it", zap.String("scheduler", l.GetName()), zap.Uint64("region-id", solver.region.GetID())) + if solver.IsRegionHot(solver.Region) { + log.Debug("region is hot region, ignore it", zap.String("scheduler", l.GetName()), zap.Uint64("region-id", solver.Region.GetID())) if collector != nil { - collector.Collect(plan.SetResource(solver.region), plan.SetStatus(plan.NewStatus(plan.StatusRegionHot))) + collector.Collect(plan.SetResource(solver.Region), plan.SetStatus(plan.NewStatus(plan.StatusRegionHot))) } balanceLeaderRegionHotCounter.Inc() return nil } - solver.step++ - defer func() { solver.step-- }() - targets := solver.GetFollowerStores(solver.region) + solver.Step++ + defer func() { solver.Step-- }() + targets := solver.GetFollowerStores(solver.Region) finalFilters := l.filters - conf := solver.GetOpts() - if leaderFilter := filter.NewPlacementLeaderSafeguard(l.GetName(), conf, solver.GetBasicCluster(), solver.GetRuleManager(), solver.region, solver.source, false /*allowMoveLeader*/); leaderFilter != nil { + conf := solver.GetSchedulerConfig() + if leaderFilter := filter.NewPlacementLeaderSafeguard(l.GetName(), conf, solver.GetBasicCluster(), solver.GetRuleManager(), solver.Region, solver.Source, false /*allowMoveLeader*/); leaderFilter != nil { finalFilters = append(l.filters, leaderFilter) } targets = filter.SelectTargetStores(targets, finalFilters, conf, collector, l.filterCounter) @@ -463,12 +461,12 @@ func (l *balanceLeaderScheduler) transferLeaderOut(solver *solver, collector *pl jOp := solver.GetOpInfluence(targets[j].GetID()) return targets[i].LeaderScore(leaderSchedulePolicy, iOp) < targets[j].LeaderScore(leaderSchedulePolicy, jOp) }) - for _, solver.target = range targets { + for _, solver.Target = range targets { if op := l.createOperator(solver, collector); op != nil { return op } } - log.Debug("region has no target store", zap.String("scheduler", l.GetName()), zap.Uint64("region-id", solver.region.GetID())) + log.Debug("region has no target store", zap.String("scheduler", l.GetName()), zap.Uint64("region-id", solver.Region.GetID())) balanceLeaderNoTargetStoreCounter.Inc() return nil } @@ -477,39 +475,39 @@ func (l *balanceLeaderScheduler) transferLeaderOut(solver *solver, collector *pl // It randomly selects a health region from the target store, then picks // the worst follower peer and transfers the leader. func (l *balanceLeaderScheduler) transferLeaderIn(solver *solver, collector *plan.Collector) *operator.Operator { - solver.region = filter.SelectOneRegion(solver.RandFollowerRegions(solver.TargetStoreID(), l.conf.Ranges), + solver.Region = filter.SelectOneRegion(solver.RandFollowerRegions(solver.TargetStoreID(), l.conf.Ranges), nil, filter.NewRegionPendingFilter(), filter.NewRegionDownFilter()) - if solver.region == nil { + if solver.Region == nil { log.Debug("store has no follower", zap.String("scheduler", l.GetName()), zap.Uint64("store-id", solver.TargetStoreID())) balanceLeaderNoFollowerRegionCounter.Inc() return nil } - if solver.IsRegionHot(solver.region) { - log.Debug("region is hot region, ignore it", zap.String("scheduler", l.GetName()), zap.Uint64("region-id", solver.region.GetID())) + if solver.IsRegionHot(solver.Region) { + log.Debug("region is hot region, ignore it", zap.String("scheduler", l.GetName()), zap.Uint64("region-id", solver.Region.GetID())) balanceLeaderRegionHotCounter.Inc() return nil } - leaderStoreID := solver.region.GetLeader().GetStoreId() - solver.source = solver.GetStore(leaderStoreID) - if solver.source == nil { + leaderStoreID := solver.Region.GetLeader().GetStoreId() + solver.Source = solver.GetStore(leaderStoreID) + if solver.Source == nil { log.Debug("region has no leader or leader store cannot be found", zap.String("scheduler", l.GetName()), - zap.Uint64("region-id", solver.region.GetID()), + zap.Uint64("region-id", solver.Region.GetID()), zap.Uint64("store-id", leaderStoreID), ) balanceLeaderNoLeaderRegionCounter.Inc() return nil } finalFilters := l.filters - conf := solver.GetOpts() - if leaderFilter := filter.NewPlacementLeaderSafeguard(l.GetName(), conf, solver.GetBasicCluster(), solver.GetRuleManager(), solver.region, solver.source, false /*allowMoveLeader*/); leaderFilter != nil { + conf := solver.GetSchedulerConfig() + if leaderFilter := filter.NewPlacementLeaderSafeguard(l.GetName(), conf, solver.GetBasicCluster(), solver.GetRuleManager(), solver.Region, solver.Source, false /*allowMoveLeader*/); leaderFilter != nil { finalFilters = append(l.filters, leaderFilter) } - target := filter.NewCandidates([]*core.StoreInfo{solver.target}). + target := filter.NewCandidates([]*core.StoreInfo{solver.Target}). FilterTarget(conf, nil, l.filterCounter, finalFilters...). PickFirst() if target == nil { - log.Debug("region has no target store", zap.String("scheduler", l.GetName()), zap.Uint64("region-id", solver.region.GetID())) + log.Debug("region has no target store", zap.String("scheduler", l.GetName()), zap.Uint64("region-id", solver.Region.GetID())) balanceLeaderNoTargetStoreCounter.Inc() return nil } @@ -521,8 +519,8 @@ func (l *balanceLeaderScheduler) transferLeaderIn(solver *solver, collector *pla // no new operator need to be created, otherwise create an operator that transfers // the leader from the source store to the target store for the region. func (l *balanceLeaderScheduler) createOperator(solver *solver, collector *plan.Collector) *operator.Operator { - solver.step++ - defer func() { solver.step-- }() + solver.Step++ + defer func() { solver.Step-- }() solver.sourceScore, solver.targetScore = solver.sourceStoreScore(l.GetName()), solver.targetStoreScore(l.GetName()) if !solver.shouldBalance(l.GetName()) { balanceLeaderSkipCounter.Inc() @@ -531,9 +529,9 @@ func (l *balanceLeaderScheduler) createOperator(solver *solver, collector *plan. } return nil } - solver.step++ - defer func() { solver.step-- }() - op, err := operator.CreateTransferLeaderOperator(BalanceLeaderType, solver, solver.region, solver.region.GetLeader().GetStoreId(), solver.TargetStoreID(), []uint64{}, operator.OpLeader) + solver.Step++ + defer func() { solver.Step-- }() + op, err := operator.CreateTransferLeaderOperator(BalanceLeaderType, solver, solver.Region, solver.Region.GetLeader().GetStoreId(), solver.TargetStoreID(), []uint64{}, operator.OpLeader) if err != nil { log.Debug("fail to create balance leader operator", errs.ZapError(err)) if collector != nil { diff --git a/pkg/schedule/schedulers/balance_plan_test.go b/pkg/schedule/schedulers/balance_plan_test.go deleted file mode 100644 index 0599ee9eadc..00000000000 --- a/pkg/schedule/schedulers/balance_plan_test.go +++ /dev/null @@ -1,272 +0,0 @@ -// Copyright 2022 TiKV Project Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package schedulers - -import ( - "context" - "testing" - - "github.com/pingcap/kvproto/pkg/metapb" - "github.com/stretchr/testify/suite" - "github.com/tikv/pd/pkg/core" - "github.com/tikv/pd/pkg/schedule/plan" -) - -type balanceSchedulerPlanAnalyzeTestSuite struct { - suite.Suite - - stores []*core.StoreInfo - regions []*core.RegionInfo - check func(map[uint64]plan.Status, map[uint64]*plan.Status) bool - ctx context.Context - cancel context.CancelFunc -} - -func TestBalanceSchedulerPlanAnalyzerTestSuite(t *testing.T) { - suite.Run(t, new(balanceSchedulerPlanAnalyzeTestSuite)) -} - -func (suite *balanceSchedulerPlanAnalyzeTestSuite) SetupSuite() { - suite.ctx, suite.cancel = context.WithCancel(context.Background()) - suite.check = func(output map[uint64]plan.Status, expects map[uint64]*plan.Status) bool { - for id, status := range expects { - outputStatus, ok := output[id] - if !ok { - return false - } - if outputStatus != *status { - return false - } - } - return true - } - suite.stores = []*core.StoreInfo{ - core.NewStoreInfo( - &metapb.Store{ - Id: 1, - }, - ), - core.NewStoreInfo( - &metapb.Store{ - Id: 2, - }, - ), - core.NewStoreInfo( - &metapb.Store{ - Id: 3, - }, - ), - core.NewStoreInfo( - &metapb.Store{ - Id: 4, - }, - ), - core.NewStoreInfo( - &metapb.Store{ - Id: 5, - }, - ), - } - suite.regions = []*core.RegionInfo{ - core.NewRegionInfo( - &metapb.Region{ - Id: 1, - }, - &metapb.Peer{ - Id: 1, - StoreId: 1, - }, - ), - core.NewRegionInfo( - &metapb.Region{ - Id: 2, - }, - &metapb.Peer{ - Id: 2, - StoreId: 2, - }, - ), - core.NewRegionInfo( - &metapb.Region{ - Id: 3, - }, - &metapb.Peer{ - Id: 3, - StoreId: 3, - }, - ), - } -} - -func (suite *balanceSchedulerPlanAnalyzeTestSuite) TearDownSuite() { - suite.cancel() -} - -func (suite *balanceSchedulerPlanAnalyzeTestSuite) TestAnalyzerResult1() { - plans := make([]plan.Plan, 0) - plans = append(plans, &balanceSchedulerPlan{source: suite.stores[4], step: 2, target: suite.stores[0], status: plan.NewStatus(plan.StatusStoreScoreDisallowed)}) - plans = append(plans, &balanceSchedulerPlan{source: suite.stores[4], step: 2, target: suite.stores[1], status: plan.NewStatus(plan.StatusStoreScoreDisallowed)}) - plans = append(plans, &balanceSchedulerPlan{source: suite.stores[4], step: 2, target: suite.stores[2], status: plan.NewStatus(plan.StatusStoreScoreDisallowed)}) - plans = append(plans, &balanceSchedulerPlan{source: suite.stores[4], step: 2, target: suite.stores[3], status: plan.NewStatus(plan.StatusStoreNotMatchRule)}) - plans = append(plans, &balanceSchedulerPlan{source: suite.stores[4], step: 2, target: suite.stores[4], status: plan.NewStatus(plan.StatusStoreAlreadyHasPeer)}) - plans = append(plans, &balanceSchedulerPlan{source: suite.stores[3], step: 2, target: suite.stores[0], status: plan.NewStatus(plan.StatusStoreNotMatchRule)}) - plans = append(plans, &balanceSchedulerPlan{source: suite.stores[3], step: 2, target: suite.stores[1], status: plan.NewStatus(plan.StatusStoreNotMatchRule)}) - plans = append(plans, &balanceSchedulerPlan{source: suite.stores[3], step: 2, target: suite.stores[2], status: plan.NewStatus(plan.StatusStoreNotMatchRule)}) - plans = append(plans, &balanceSchedulerPlan{source: suite.stores[3], step: 2, target: suite.stores[3], status: plan.NewStatus(plan.StatusStoreAlreadyHasPeer)}) - plans = append(plans, &balanceSchedulerPlan{source: suite.stores[3], step: 2, target: suite.stores[4], status: plan.NewStatus(plan.StatusStoreNotMatchRule)}) - plans = append(plans, &balanceSchedulerPlan{source: suite.stores[2], step: 2, target: suite.stores[0], status: plan.NewStatus(plan.StatusStoreScoreDisallowed)}) - plans = append(plans, &balanceSchedulerPlan{source: suite.stores[2], step: 2, target: suite.stores[1], status: plan.NewStatus(plan.StatusStoreScoreDisallowed)}) - plans = append(plans, &balanceSchedulerPlan{source: suite.stores[2], step: 2, target: suite.stores[2], status: plan.NewStatus(plan.StatusStoreAlreadyHasPeer)}) - plans = append(plans, &balanceSchedulerPlan{source: suite.stores[2], step: 2, target: suite.stores[3], status: plan.NewStatus(plan.StatusStoreNotMatchRule)}) - plans = append(plans, &balanceSchedulerPlan{source: suite.stores[2], step: 2, target: suite.stores[4], status: plan.NewStatus(plan.StatusStoreScoreDisallowed)}) - plans = append(plans, &balanceSchedulerPlan{source: suite.stores[1], step: 2, target: suite.stores[0], status: plan.NewStatus(plan.StatusStoreScoreDisallowed)}) - plans = append(plans, &balanceSchedulerPlan{source: suite.stores[1], step: 2, target: suite.stores[1], status: plan.NewStatus(plan.StatusStoreAlreadyHasPeer)}) - plans = append(plans, &balanceSchedulerPlan{source: suite.stores[1], step: 2, target: suite.stores[2], status: plan.NewStatus(plan.StatusStoreScoreDisallowed)}) - plans = append(plans, &balanceSchedulerPlan{source: suite.stores[1], step: 2, target: suite.stores[3], status: plan.NewStatus(plan.StatusStoreNotMatchRule)}) - plans = append(plans, &balanceSchedulerPlan{source: suite.stores[1], step: 2, target: suite.stores[4], status: plan.NewStatus(plan.StatusStoreScoreDisallowed)}) - plans = append(plans, &balanceSchedulerPlan{source: suite.stores[0], step: 2, target: suite.stores[0], status: plan.NewStatus(plan.StatusStoreAlreadyHasPeer)}) - plans = append(plans, &balanceSchedulerPlan{source: suite.stores[0], step: 2, target: suite.stores[1], status: plan.NewStatus(plan.StatusStoreScoreDisallowed)}) - plans = append(plans, &balanceSchedulerPlan{source: suite.stores[0], step: 2, target: suite.stores[2], status: plan.NewStatus(plan.StatusStoreScoreDisallowed)}) - plans = append(plans, &balanceSchedulerPlan{source: suite.stores[0], step: 2, target: suite.stores[3], status: plan.NewStatus(plan.StatusStoreNotMatchRule)}) - plans = append(plans, &balanceSchedulerPlan{source: suite.stores[0], step: 2, target: suite.stores[4], status: plan.NewStatus(plan.StatusStoreScoreDisallowed)}) - statuses, isNormal, err := BalancePlanSummary(plans) - suite.NoError(err) - suite.True(isNormal) - suite.True(suite.check(statuses, - map[uint64]*plan.Status{ - 1: plan.NewStatus(plan.StatusStoreNotMatchRule), - 2: plan.NewStatus(plan.StatusStoreNotMatchRule), - 3: plan.NewStatus(plan.StatusStoreNotMatchRule), - 4: plan.NewStatus(plan.StatusStoreNotMatchRule), - 5: plan.NewStatus(plan.StatusStoreNotMatchRule), - })) -} - -func (suite *balanceSchedulerPlanAnalyzeTestSuite) TestAnalyzerResult2() { - plans := make([]plan.Plan, 0) - plans = append(plans, &balanceSchedulerPlan{source: suite.stores[4], step: 0, status: plan.NewStatus(plan.StatusStoreDown)}) - plans = append(plans, &balanceSchedulerPlan{source: suite.stores[3], step: 0, status: plan.NewStatus(plan.StatusStoreDown)}) - plans = append(plans, &balanceSchedulerPlan{source: suite.stores[2], step: 0, status: plan.NewStatus(plan.StatusStoreDown)}) - plans = append(plans, &balanceSchedulerPlan{source: suite.stores[1], step: 0, status: plan.NewStatus(plan.StatusStoreDown)}) - plans = append(plans, &balanceSchedulerPlan{source: suite.stores[0], step: 0, status: plan.NewStatus(plan.StatusStoreDown)}) - statuses, isNormal, err := BalancePlanSummary(plans) - suite.NoError(err) - suite.False(isNormal) - suite.True(suite.check(statuses, - map[uint64]*plan.Status{ - 1: plan.NewStatus(plan.StatusStoreDown), - 2: plan.NewStatus(plan.StatusStoreDown), - 3: plan.NewStatus(plan.StatusStoreDown), - 4: plan.NewStatus(plan.StatusStoreDown), - 5: plan.NewStatus(plan.StatusStoreDown), - })) -} - -func (suite *balanceSchedulerPlanAnalyzeTestSuite) TestAnalyzerResult3() { - plans := make([]plan.Plan, 0) - plans = append(plans, &balanceSchedulerPlan{source: suite.stores[4], step: 0, status: plan.NewStatus(plan.StatusStoreDown)}) - plans = append(plans, &balanceSchedulerPlan{source: suite.stores[3], region: suite.regions[0], step: 1, status: plan.NewStatus(plan.StatusRegionNotMatchRule)}) - plans = append(plans, &balanceSchedulerPlan{source: suite.stores[2], region: suite.regions[0], step: 1, status: plan.NewStatus(plan.StatusRegionNotMatchRule)}) - plans = append(plans, &balanceSchedulerPlan{source: suite.stores[1], region: suite.regions[1], step: 1, status: plan.NewStatus(plan.StatusRegionNotMatchRule)}) - plans = append(plans, &balanceSchedulerPlan{source: suite.stores[0], region: suite.regions[1], step: 1, status: plan.NewStatus(plan.StatusRegionNotMatchRule)}) - statuses, isNormal, err := BalancePlanSummary(plans) - suite.NoError(err) - suite.False(isNormal) - suite.True(suite.check(statuses, - map[uint64]*plan.Status{ - 1: plan.NewStatus(plan.StatusRegionNotMatchRule), - 2: plan.NewStatus(plan.StatusRegionNotMatchRule), - 3: plan.NewStatus(plan.StatusRegionNotMatchRule), - 4: plan.NewStatus(plan.StatusRegionNotMatchRule), - })) -} - -func (suite *balanceSchedulerPlanAnalyzeTestSuite) TestAnalyzerResult4() { - plans := make([]plan.Plan, 0) - plans = append(plans, &balanceSchedulerPlan{source: suite.stores[4], step: 0, status: plan.NewStatus(plan.StatusStoreDown)}) - plans = append(plans, &balanceSchedulerPlan{source: suite.stores[3], region: suite.regions[0], step: 1, status: plan.NewStatus(plan.StatusRegionNotMatchRule)}) - plans = append(plans, &balanceSchedulerPlan{source: suite.stores[2], region: suite.regions[0], step: 1, status: plan.NewStatus(plan.StatusRegionNotMatchRule)}) - plans = append(plans, &balanceSchedulerPlan{source: suite.stores[1], target: suite.stores[0], step: 2, status: plan.NewStatus(plan.StatusStoreScoreDisallowed)}) - plans = append(plans, &balanceSchedulerPlan{source: suite.stores[1], target: suite.stores[1], step: 2, status: plan.NewStatus(plan.StatusStoreAlreadyHasPeer)}) - plans = append(plans, &balanceSchedulerPlan{source: suite.stores[1], target: suite.stores[2], step: 2, status: plan.NewStatus(plan.StatusStoreNotMatchRule)}) - plans = append(plans, &balanceSchedulerPlan{source: suite.stores[1], target: suite.stores[3], step: 2, status: plan.NewStatus(plan.StatusStoreNotMatchRule)}) - plans = append(plans, &balanceSchedulerPlan{source: suite.stores[1], target: suite.stores[4], step: 2, status: plan.NewStatus(plan.StatusStoreDown)}) - plans = append(plans, &balanceSchedulerPlan{source: suite.stores[0], target: suite.stores[0], step: 2, status: plan.NewStatus(plan.StatusStoreAlreadyHasPeer)}) - plans = append(plans, &balanceSchedulerPlan{source: suite.stores[0], target: suite.stores[1], step: 2, status: plan.NewStatus(plan.StatusStoreScoreDisallowed)}) - plans = append(plans, &balanceSchedulerPlan{source: suite.stores[0], target: suite.stores[2], step: 2, status: plan.NewStatus(plan.StatusStoreNotMatchRule)}) - plans = append(plans, &balanceSchedulerPlan{source: suite.stores[0], target: suite.stores[3], step: 2, status: plan.NewStatus(plan.StatusStoreNotMatchRule)}) - plans = append(plans, &balanceSchedulerPlan{source: suite.stores[0], target: suite.stores[4], step: 2, status: plan.NewStatus(plan.StatusStoreDown)}) - statuses, isNormal, err := BalancePlanSummary(plans) - suite.NoError(err) - suite.False(isNormal) - suite.True(suite.check(statuses, - map[uint64]*plan.Status{ - 1: plan.NewStatus(plan.StatusStoreAlreadyHasPeer), - 2: plan.NewStatus(plan.StatusStoreAlreadyHasPeer), - 3: plan.NewStatus(plan.StatusStoreNotMatchRule), - 4: plan.NewStatus(plan.StatusStoreNotMatchRule), - 5: plan.NewStatus(plan.StatusStoreDown), - })) -} - -func (suite *balanceSchedulerPlanAnalyzeTestSuite) TestAnalyzerResult5() { - plans := make([]plan.Plan, 0) - plans = append(plans, &balanceSchedulerPlan{source: suite.stores[4], step: 0, status: plan.NewStatus(plan.StatusStoreRemoveLimitThrottled)}) - plans = append(plans, &balanceSchedulerPlan{source: suite.stores[3], region: suite.regions[0], step: 1, status: plan.NewStatus(plan.StatusRegionNotMatchRule)}) - plans = append(plans, &balanceSchedulerPlan{source: suite.stores[2], region: suite.regions[0], step: 1, status: plan.NewStatus(plan.StatusRegionNotMatchRule)}) - plans = append(plans, &balanceSchedulerPlan{source: suite.stores[1], target: suite.stores[0], step: 2, status: plan.NewStatus(plan.StatusStoreScoreDisallowed)}) - plans = append(plans, &balanceSchedulerPlan{source: suite.stores[1], target: suite.stores[1], step: 2, status: plan.NewStatus(plan.StatusStoreAlreadyHasPeer)}) - plans = append(plans, &balanceSchedulerPlan{source: suite.stores[1], target: suite.stores[2], step: 2, status: plan.NewStatus(plan.StatusStoreNotMatchRule)}) - plans = append(plans, &balanceSchedulerPlan{source: suite.stores[1], target: suite.stores[3], step: 2, status: plan.NewStatus(plan.StatusStoreNotMatchRule)}) - plans = append(plans, &balanceSchedulerPlan{source: suite.stores[0], target: suite.stores[0], step: 2, status: plan.NewStatus(plan.StatusStoreAlreadyHasPeer)}) - plans = append(plans, &balanceSchedulerPlan{source: suite.stores[0], target: suite.stores[1], step: 3, status: plan.NewStatus(plan.StatusStoreScoreDisallowed)}) - plans = append(plans, &balanceSchedulerPlan{source: suite.stores[0], target: suite.stores[2], step: 2, status: plan.NewStatus(plan.StatusStoreNotMatchRule)}) - plans = append(plans, &balanceSchedulerPlan{source: suite.stores[0], target: suite.stores[3], step: 2, status: plan.NewStatus(plan.StatusStoreNotMatchRule)}) - statuses, isNormal, err := BalancePlanSummary(plans) - suite.NoError(err) - suite.False(isNormal) - suite.True(suite.check(statuses, - map[uint64]*plan.Status{ - 1: plan.NewStatus(plan.StatusStoreAlreadyHasPeer), - 2: plan.NewStatus(plan.StatusStoreAlreadyHasPeer), - 3: plan.NewStatus(plan.StatusStoreNotMatchRule), - 4: plan.NewStatus(plan.StatusStoreNotMatchRule), - 5: plan.NewStatus(plan.StatusStoreRemoveLimitThrottled), - })) -} - -func (suite *balanceSchedulerPlanAnalyzeTestSuite) TestAnalyzerResult6() { - basePlan := NewBalanceSchedulerPlan() - collector := plan.NewCollector(basePlan) - collector.Collect(plan.SetResourceWithStep(suite.stores[0], 2), plan.SetStatus(plan.NewStatus(plan.StatusStoreDown))) - collector.Collect(plan.SetResourceWithStep(suite.stores[1], 2), plan.SetStatus(plan.NewStatus(plan.StatusStoreDown))) - collector.Collect(plan.SetResourceWithStep(suite.stores[2], 2), plan.SetStatus(plan.NewStatus(plan.StatusStoreDown))) - collector.Collect(plan.SetResourceWithStep(suite.stores[3], 2), plan.SetStatus(plan.NewStatus(plan.StatusStoreDown))) - collector.Collect(plan.SetResourceWithStep(suite.stores[4], 2), plan.SetStatus(plan.NewStatus(plan.StatusStoreDown))) - basePlan.source = suite.stores[0] - basePlan.step++ - collector.Collect(plan.SetResource(suite.regions[0]), plan.SetStatus(plan.NewStatus(plan.StatusRegionNoLeader))) - statuses, isNormal, err := BalancePlanSummary(collector.GetPlans()) - suite.NoError(err) - suite.False(isNormal) - suite.True(suite.check(statuses, - map[uint64]*plan.Status{ - 1: plan.NewStatus(plan.StatusStoreDown), - 2: plan.NewStatus(plan.StatusStoreDown), - 3: plan.NewStatus(plan.StatusStoreDown), - 4: plan.NewStatus(plan.StatusStoreDown), - 5: plan.NewStatus(plan.StatusStoreDown), - })) -} diff --git a/pkg/schedule/schedulers/balance_region.go b/pkg/schedule/schedulers/balance_region.go index 95e051bc313..d27d18b086c 100644 --- a/pkg/schedule/schedulers/balance_region.go +++ b/pkg/schedule/schedulers/balance_region.go @@ -23,7 +23,7 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/tikv/pd/pkg/core" "github.com/tikv/pd/pkg/core/constant" - "github.com/tikv/pd/pkg/schedule" + sche "github.com/tikv/pd/pkg/schedule/core" "github.com/tikv/pd/pkg/schedule/filter" "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/plan" @@ -58,7 +58,6 @@ type balanceRegionScheduler struct { *BaseScheduler *retryQuota conf *balanceRegionSchedulerConfig - opController *schedule.OperatorController filters []filter.Filter counter *prometheus.CounterVec filterCounter *filter.Counter @@ -66,13 +65,12 @@ type balanceRegionScheduler struct { // newBalanceRegionScheduler creates a scheduler that tends to keep regions on // each store balanced. -func newBalanceRegionScheduler(opController *schedule.OperatorController, conf *balanceRegionSchedulerConfig, opts ...BalanceRegionCreateOption) schedule.Scheduler { +func newBalanceRegionScheduler(opController *operator.Controller, conf *balanceRegionSchedulerConfig, opts ...BalanceRegionCreateOption) Scheduler { base := NewBaseScheduler(opController) scheduler := &balanceRegionScheduler{ BaseScheduler: base, retryQuota: newRetryQuota(), conf: conf, - opController: opController, counter: balanceRegionCounter, filterCounter: filter.NewCounter(filter.BalanceRegion.String()), } @@ -80,7 +78,7 @@ func newBalanceRegionScheduler(opController *schedule.OperatorController, conf * setOption(scheduler) } scheduler.filters = []filter.Filter{ - &filter.StoreStateFilter{ActionScope: scheduler.GetName(), MoveRegion: true}, + &filter.StoreStateFilter{ActionScope: scheduler.GetName(), MoveRegion: true, OperatorLevel: constant.Medium}, filter.NewSpecialUseFilter(scheduler.GetName()), } return scheduler @@ -112,46 +110,47 @@ func (s *balanceRegionScheduler) GetType() string { } func (s *balanceRegionScheduler) EncodeConfig() ([]byte, error) { - return schedule.EncodeConfig(s.conf) + return EncodeConfig(s.conf) } -func (s *balanceRegionScheduler) IsScheduleAllowed(cluster schedule.Cluster) bool { - allowed := s.opController.OperatorCount(operator.OpRegion) < cluster.GetOpts().GetRegionScheduleLimit() +func (s *balanceRegionScheduler) IsScheduleAllowed(cluster sche.SchedulerCluster) bool { + allowed := s.OpController.OperatorCount(operator.OpRegion) < cluster.GetSchedulerConfig().GetRegionScheduleLimit() if !allowed { operator.OperatorLimitCounter.WithLabelValues(s.GetType(), operator.OpRegion.String()).Inc() } return allowed } -func (s *balanceRegionScheduler) Schedule(cluster schedule.Cluster, dryRun bool) ([]*operator.Operator, []plan.Plan) { - basePlan := NewBalanceSchedulerPlan() +func (s *balanceRegionScheduler) Schedule(cluster sche.SchedulerCluster, dryRun bool) ([]*operator.Operator, []plan.Plan) { + basePlan := plan.NewBalanceSchedulerPlan() var collector *plan.Collector if dryRun { collector = plan.NewCollector(basePlan) } balanceRegionScheduleCounter.Inc() stores := cluster.GetStores() - opts := cluster.GetOpts() - faultTargets := filter.SelectUnavailableTargetStores(stores, s.filters, opts, collector, s.filterCounter) - sourceStores := filter.SelectSourceStores(stores, s.filters, opts, collector, s.filterCounter) - opInfluence := s.opController.GetOpInfluence(cluster) - s.OpController.GetFastOpInfluence(cluster, opInfluence) + conf := cluster.GetSchedulerConfig() + snapshotFilter := filter.NewSnapshotSendFilter(stores, constant.Medium) + faultTargets := filter.SelectUnavailableTargetStores(stores, s.filters, conf, collector, s.filterCounter) + sourceStores := filter.SelectSourceStores(stores, s.filters, conf, collector, s.filterCounter) + opInfluence := s.OpController.GetOpInfluence(cluster.GetBasicCluster()) + s.OpController.GetFastOpInfluence(cluster.GetBasicCluster(), opInfluence) kind := constant.NewScheduleKind(constant.RegionKind, constant.BySize) solver := newSolver(basePlan, kind, cluster, opInfluence) sort.Slice(sourceStores, func(i, j int) bool { iOp := solver.GetOpInfluence(sourceStores[i].GetID()) jOp := solver.GetOpInfluence(sourceStores[j].GetID()) - return sourceStores[i].RegionScore(opts.GetRegionScoreFormulaVersion(), opts.GetHighSpaceRatio(), opts.GetLowSpaceRatio(), iOp) > - sourceStores[j].RegionScore(opts.GetRegionScoreFormulaVersion(), opts.GetHighSpaceRatio(), opts.GetLowSpaceRatio(), jOp) + return sourceStores[i].RegionScore(conf.GetRegionScoreFormulaVersion(), conf.GetHighSpaceRatio(), conf.GetLowSpaceRatio(), iOp) > + sourceStores[j].RegionScore(conf.GetRegionScoreFormulaVersion(), conf.GetHighSpaceRatio(), conf.GetLowSpaceRatio(), jOp) }) pendingFilter := filter.NewRegionPendingFilter() downFilter := filter.NewRegionDownFilter() replicaFilter := filter.NewRegionReplicatedFilter(cluster) - baseRegionFilters := []filter.RegionFilter{downFilter, replicaFilter} + baseRegionFilters := []filter.RegionFilter{downFilter, replicaFilter, snapshotFilter} switch cluster.(type) { - case *schedule.RangeCluster: + case *rangeCluster: // allow empty region to be scheduled in range cluster default: baseRegionFilters = append(baseRegionFilters, filter.NewRegionEmptyFilter(cluster)) @@ -161,12 +160,12 @@ func (s *balanceRegionScheduler) Schedule(cluster schedule.Cluster, dryRun bool) collector.Collect(plan.SetResource(sourceStores[0]), plan.SetStatus(plan.NewStatus(plan.StatusStoreScoreDisallowed))) } - solver.step++ + solver.Step++ var sourceIndex int // sourcesStore is sorted by region score desc, so we pick the first store as source store. - for sourceIndex, solver.source = range sourceStores { - retryLimit := s.retryQuota.GetLimit(solver.source) + for sourceIndex, solver.Source = range sourceStores { + retryLimit := s.retryQuota.GetLimit(solver.Source) solver.sourceScore = solver.sourceStoreScore(s.GetName()) if sourceIndex == len(sourceStores)-1 { break @@ -174,58 +173,58 @@ func (s *balanceRegionScheduler) Schedule(cluster schedule.Cluster, dryRun bool) for i := 0; i < retryLimit; i++ { // Priority pick the region that has a pending peer. // Pending region may mean the disk is overload, remove the pending region firstly. - solver.region = filter.SelectOneRegion(cluster.RandPendingRegions(solver.SourceStoreID(), s.conf.Ranges), collector, + solver.Region = filter.SelectOneRegion(cluster.RandPendingRegions(solver.SourceStoreID(), s.conf.Ranges), collector, append(baseRegionFilters, filter.NewRegionWitnessFilter(solver.SourceStoreID()))...) - if solver.region == nil { + if solver.Region == nil { // Then pick the region that has a follower in the source store. - solver.region = filter.SelectOneRegion(cluster.RandFollowerRegions(solver.SourceStoreID(), s.conf.Ranges), collector, + solver.Region = filter.SelectOneRegion(cluster.RandFollowerRegions(solver.SourceStoreID(), s.conf.Ranges), collector, append(baseRegionFilters, filter.NewRegionWitnessFilter(solver.SourceStoreID()), pendingFilter)...) } - if solver.region == nil { + if solver.Region == nil { // Then pick the region has the leader in the source store. - solver.region = filter.SelectOneRegion(cluster.RandLeaderRegions(solver.SourceStoreID(), s.conf.Ranges), collector, + solver.Region = filter.SelectOneRegion(cluster.RandLeaderRegions(solver.SourceStoreID(), s.conf.Ranges), collector, append(baseRegionFilters, filter.NewRegionWitnessFilter(solver.SourceStoreID()), pendingFilter)...) } - if solver.region == nil { + if solver.Region == nil { // Finally, pick learner. - solver.region = filter.SelectOneRegion(cluster.RandLearnerRegions(solver.SourceStoreID(), s.conf.Ranges), collector, + solver.Region = filter.SelectOneRegion(cluster.RandLearnerRegions(solver.SourceStoreID(), s.conf.Ranges), collector, append(baseRegionFilters, filter.NewRegionWitnessFilter(solver.SourceStoreID()), pendingFilter)...) } - if solver.region == nil { + if solver.Region == nil { balanceRegionNoRegionCounter.Inc() continue } - log.Debug("select region", zap.String("scheduler", s.GetName()), zap.Uint64("region-id", solver.region.GetID())) + log.Debug("select region", zap.String("scheduler", s.GetName()), zap.Uint64("region-id", solver.Region.GetID())) // Skip hot regions. - if cluster.IsRegionHot(solver.region) { - log.Debug("region is hot", zap.String("scheduler", s.GetName()), zap.Uint64("region-id", solver.region.GetID())) + if cluster.IsRegionHot(solver.Region) { + log.Debug("region is hot", zap.String("scheduler", s.GetName()), zap.Uint64("region-id", solver.Region.GetID())) if collector != nil { - collector.Collect(plan.SetResource(solver.region), plan.SetStatus(plan.NewStatus(plan.StatusRegionHot))) + collector.Collect(plan.SetResource(solver.Region), plan.SetStatus(plan.NewStatus(plan.StatusRegionHot))) } balanceRegionHotCounter.Inc() continue } // Check region leader - if solver.region.GetLeader() == nil { - log.Warn("region have no leader", zap.String("scheduler", s.GetName()), zap.Uint64("region-id", solver.region.GetID())) + if solver.Region.GetLeader() == nil { + log.Warn("region have no leader", zap.String("scheduler", s.GetName()), zap.Uint64("region-id", solver.Region.GetID())) if collector != nil { - collector.Collect(plan.SetResource(solver.region), plan.SetStatus(plan.NewStatus(plan.StatusRegionNoLeader))) + collector.Collect(plan.SetResource(solver.Region), plan.SetStatus(plan.NewStatus(plan.StatusRegionNoLeader))) } balanceRegionNoLeaderCounter.Inc() continue } - solver.step++ + solver.Step++ // the replica filter will cache the last region fit and the select one will only pict the first one region that // satisfy all the filters, so the region fit must belong the scheduled region. solver.fit = replicaFilter.(*filter.RegionReplicatedFilter).GetFit() if op := s.transferPeer(solver, collector, sourceStores[sourceIndex+1:], faultTargets); op != nil { - s.retryQuota.ResetLimit(solver.source) + s.retryQuota.ResetLimit(solver.Source) op.Counters = append(op.Counters, balanceRegionNewOpCounter) return []*operator.Operator{op}, collector.GetPlans() } - solver.step-- + solver.Step-- } - s.retryQuota.Attenuate(solver.source) + s.retryQuota.Attenuate(solver.Source) } s.filterCounter.Flush() s.retryQuota.GC(stores) @@ -234,29 +233,30 @@ func (s *balanceRegionScheduler) Schedule(cluster schedule.Cluster, dryRun bool) // transferPeer selects the best store to create a new peer to replace the old peer. func (s *balanceRegionScheduler) transferPeer(solver *solver, collector *plan.Collector, dstStores []*core.StoreInfo, faultStores []*core.StoreInfo) *operator.Operator { - excludeTargets := solver.region.GetStoreIDs() + excludeTargets := solver.Region.GetStoreIDs() for _, store := range faultStores { excludeTargets[store.GetID()] = struct{}{} } // the order of the filters should be sorted by the cost of the cpu overhead. // the more expensive the filter is, the later it should be placed. + conf := solver.GetSchedulerConfig() filters := []filter.Filter{ filter.NewExcludedFilter(s.GetName(), nil, excludeTargets), - filter.NewPlacementSafeguard(s.GetName(), solver.GetOpts(), solver.GetBasicCluster(), solver.GetRuleManager(), - solver.region, solver.source, solver.fit), + filter.NewPlacementSafeguard(s.GetName(), conf, solver.GetBasicCluster(), solver.GetRuleManager(), + solver.Region, solver.Source, solver.fit), } - candidates := filter.NewCandidates(dstStores).FilterTarget(solver.GetOpts(), collector, s.filterCounter, filters...) + candidates := filter.NewCandidates(dstStores).FilterTarget(conf, collector, s.filterCounter, filters...) if len(candidates.Stores) != 0 { - solver.step++ + solver.Step++ } // candidates are sorted by region score desc, so we pick the last store as target store. for i := range candidates.Stores { - solver.target = candidates.Stores[len(candidates.Stores)-i-1] + solver.Target = candidates.Stores[len(candidates.Stores)-i-1] solver.targetScore = solver.targetStoreScore(s.GetName()) - regionID := solver.region.GetID() - sourceID := solver.source.GetID() - targetID := solver.target.GetID() + regionID := solver.Region.GetID() + sourceID := solver.Source.GetID() + targetID := solver.Target.GetID() log.Debug("candidate store", zap.Uint64("region-id", regionID), zap.Uint64("source-store", sourceID), zap.Uint64("target-store", targetID)) if !solver.shouldBalance(s.GetName()) { @@ -267,10 +267,10 @@ func (s *balanceRegionScheduler) transferPeer(solver *solver, collector *plan.Co continue } - oldPeer := solver.region.GetStorePeer(sourceID) - newPeer := &metapb.Peer{StoreId: solver.target.GetID(), Role: oldPeer.Role} - solver.step++ - op, err := operator.CreateMovePeerOperator(BalanceRegionType, solver, solver.region, operator.OpRegion, oldPeer.GetStoreId(), newPeer) + oldPeer := solver.Region.GetStorePeer(sourceID) + newPeer := &metapb.Peer{StoreId: solver.Target.GetID(), Role: oldPeer.Role} + solver.Step++ + op, err := operator.CreateMovePeerOperator(BalanceRegionType, solver, solver.Region, operator.OpRegion, oldPeer.GetStoreId(), newPeer) if err != nil { balanceRegionCreateOpFailCounter.Inc() if collector != nil { @@ -281,7 +281,7 @@ func (s *balanceRegionScheduler) transferPeer(solver *solver, collector *plan.Co if collector != nil { collector.Collect() } - solver.step-- + solver.Step-- sourceLabel := strconv.FormatUint(sourceID, 10) targetLabel := strconv.FormatUint(targetID, 10) op.FinishedCounters = append(op.FinishedCounters, @@ -297,7 +297,7 @@ func (s *balanceRegionScheduler) transferPeer(solver *solver, collector *plan.Co balanceRegionNoReplacementCounter.Inc() if len(candidates.Stores) != 0 { - solver.step-- + solver.Step-- } return nil } diff --git a/pkg/schedule/schedulers/balance_test.go b/pkg/schedule/schedulers/balance_test.go index 084698d5d19..4e20b13feaa 100644 --- a/pkg/schedule/schedulers/balance_test.go +++ b/pkg/schedule/schedulers/balance_test.go @@ -28,12 +28,11 @@ import ( "github.com/tikv/pd/pkg/core" "github.com/tikv/pd/pkg/core/constant" "github.com/tikv/pd/pkg/mock/mockcluster" - "github.com/tikv/pd/pkg/schedule" "github.com/tikv/pd/pkg/schedule/config" "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/plan" "github.com/tikv/pd/pkg/storage" - "github.com/tikv/pd/pkg/utils/testutil" + "github.com/tikv/pd/pkg/utils/operatorutil" "github.com/tikv/pd/pkg/versioninfo" ) @@ -53,7 +52,7 @@ func TestInfluenceAmp(t *testing.T) { R := int64(96) kind := constant.NewScheduleKind(constant.RegionKind, constant.BySize) - influence := oc.GetOpInfluence(tc) + influence := oc.GetOpInfluence(tc.GetBasicCluster()) influence.GetStoreInfluence(1).RegionSize = R influence.GetStoreInfluence(2).RegionSize = -R tc.SetTolerantSizeRatio(1) @@ -65,16 +64,16 @@ func TestInfluenceAmp(t *testing.T) { tc.AddLeaderRegion(1, 1, 2) region := tc.GetRegion(1).Clone(core.SetApproximateSize(R)) tc.PutRegion(region) - basePlan := NewBalanceSchedulerPlan() + basePlan := plan.NewBalanceSchedulerPlan() solver := newSolver(basePlan, kind, tc, influence) - solver.source, solver.target, solver.region = tc.GetStore(1), tc.GetStore(2), tc.GetRegion(1) + solver.Source, solver.Target, solver.Region = tc.GetStore(1), tc.GetStore(2), tc.GetRegion(1) solver.sourceScore, solver.targetScore = solver.sourceStoreScore(""), solver.targetStoreScore("") re.True(solver.shouldBalance("")) // It will not schedule if the diff region count is greater than the sum // of TolerantSizeRatio and influenceAmp*2. tc.AddRegionStore(1, int(100+influenceAmp+2)) - solver.source = tc.GetStore(1) + solver.Source = tc.GetStore(1) solver.sourceScore, solver.targetScore = solver.sourceStoreScore(""), solver.targetStoreScore("") re.False(solver.shouldBalance("")) re.Less(solver.sourceScore-solver.targetScore, float64(1)) @@ -150,9 +149,9 @@ func TestShouldBalance(t *testing.T) { tc.PutRegion(region) tc.SetLeaderSchedulePolicy(testCase.kind.String()) kind := constant.NewScheduleKind(constant.LeaderKind, testCase.kind) - basePlan := NewBalanceSchedulerPlan() - solver := newSolver(basePlan, kind, tc, oc.GetOpInfluence(tc)) - solver.source, solver.target, solver.region = tc.GetStore(1), tc.GetStore(2), tc.GetRegion(1) + basePlan := plan.NewBalanceSchedulerPlan() + solver := newSolver(basePlan, kind, tc, oc.GetOpInfluence(tc.GetBasicCluster())) + solver.Source, solver.Target, solver.Region = tc.GetStore(1), tc.GetStore(2), tc.GetRegion(1) solver.sourceScore, solver.targetScore = solver.sourceStoreScore(""), solver.targetStoreScore("") re.Equal(testCase.expectedResult, solver.shouldBalance("")) } @@ -164,9 +163,9 @@ func TestShouldBalance(t *testing.T) { region := tc.GetRegion(1).Clone(core.SetApproximateSize(testCase.regionSize)) tc.PutRegion(region) kind := constant.NewScheduleKind(constant.RegionKind, testCase.kind) - basePlan := NewBalanceSchedulerPlan() - solver := newSolver(basePlan, kind, tc, oc.GetOpInfluence(tc)) - solver.source, solver.target, solver.region = tc.GetStore(1), tc.GetStore(2), tc.GetRegion(1) + basePlan := plan.NewBalanceSchedulerPlan() + solver := newSolver(basePlan, kind, tc, oc.GetOpInfluence(tc.GetBasicCluster())) + solver.Source, solver.Target, solver.Region = tc.GetStore(1), tc.GetStore(2), tc.GetRegion(1) solver.sourceScore, solver.targetScore = solver.sourceStoreScore(""), solver.targetStoreScore("") re.Equal(testCase.expectedResult, solver.shouldBalance("")) } @@ -214,9 +213,9 @@ func TestTolerantRatio(t *testing.T) { } for _, t := range tbl { tc.SetTolerantSizeRatio(t.ratio) - basePlan := NewBalanceSchedulerPlan() + basePlan := plan.NewBalanceSchedulerPlan() solver := newSolver(basePlan, t.kind, tc, operator.OpInfluence{}) - solver.region = region + solver.Region = region sourceScore := t.expectTolerantResource(t.kind) targetScore := solver.getTolerantResource() @@ -228,9 +227,9 @@ type balanceLeaderSchedulerTestSuite struct { suite.Suite cancel context.CancelFunc tc *mockcluster.Cluster - lb schedule.Scheduler - oc *schedule.OperatorController - conf config.Config + lb Scheduler + oc *operator.Controller + conf config.SchedulerConfigProvider } func TestBalanceLeaderSchedulerTestSuite(t *testing.T) { @@ -239,7 +238,7 @@ func TestBalanceLeaderSchedulerTestSuite(t *testing.T) { func (suite *balanceLeaderSchedulerTestSuite) SetupTest() { suite.cancel, suite.conf, suite.tc, suite.oc = prepareSchedulersTest() - lb, err := schedule.CreateScheduler(BalanceLeaderType, suite.oc, storage.NewStorageWithMemoryBackend(), schedule.ConfigSliceDecoder(BalanceLeaderType, []string{"", ""})) + lb, err := CreateScheduler(BalanceLeaderType, suite.oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(BalanceLeaderType, []string{"", ""})) suite.NoError(err) suite.lb = lb } @@ -420,16 +419,16 @@ func (suite *balanceLeaderSchedulerTestSuite) TestBalanceFilter() { suite.tc.AddLeaderStore(4, 16) suite.tc.AddLeaderRegion(1, 4, 1, 2, 3) - testutil.CheckTransferLeader(suite.Require(), suite.schedule()[0], operator.OpKind(0), 4, 1) + operatorutil.CheckTransferLeader(suite.Require(), suite.schedule()[0], operator.OpKind(0), 4, 1) // Test stateFilter. // if store 4 is offline, we should consider it // because it still provides services suite.tc.SetStoreOffline(4) - testutil.CheckTransferLeader(suite.Require(), suite.schedule()[0], operator.OpKind(0), 4, 1) + operatorutil.CheckTransferLeader(suite.Require(), suite.schedule()[0], operator.OpKind(0), 4, 1) // If store 1 is down, it will be filtered, // store 2 becomes the store with least leaders. suite.tc.SetStoreDown(1) - testutil.CheckTransferLeader(suite.Require(), suite.schedule()[0], operator.OpKind(0), 4, 2) + operatorutil.CheckTransferLeader(suite.Require(), suite.schedule()[0], operator.OpKind(0), 4, 2) plans := suite.dryRun() suite.NotEmpty(plans) suite.Equal(0, plans[0].GetStep()) @@ -440,7 +439,7 @@ func (suite *balanceLeaderSchedulerTestSuite) TestBalanceFilter() { // If store 2 is busy, it will be filtered, // store 3 becomes the store with least leaders. suite.tc.SetStoreBusy(2, true) - testutil.CheckTransferLeader(suite.Require(), suite.schedule()[0], operator.OpKind(0), 4, 3) + operatorutil.CheckTransferLeader(suite.Require(), suite.schedule()[0], operator.OpKind(0), 4, 3) // Test disconnectFilter. // If store 3 is disconnected, no operator can be created. @@ -462,9 +461,9 @@ func (suite *balanceLeaderSchedulerTestSuite) TestLeaderWeight() { suite.tc.UpdateStoreLeaderWeight(3, 1) suite.tc.UpdateStoreLeaderWeight(4, 2) suite.tc.AddLeaderRegion(1, 1, 2, 3, 4) - testutil.CheckTransferLeader(suite.Require(), suite.schedule()[0], operator.OpKind(0), 1, 4) + operatorutil.CheckTransferLeader(suite.Require(), suite.schedule()[0], operator.OpKind(0), 1, 4) suite.tc.UpdateLeaderCount(4, 30) - testutil.CheckTransferLeader(suite.Require(), suite.schedule()[0], operator.OpKind(0), 1, 3) + operatorutil.CheckTransferLeader(suite.Require(), suite.schedule()[0], operator.OpKind(0), 1, 3) } func (suite *balanceLeaderSchedulerTestSuite) TestBalancePolicy() { @@ -478,9 +477,9 @@ func (suite *balanceLeaderSchedulerTestSuite) TestBalancePolicy() { suite.tc.AddLeaderRegion(1, 2, 1, 3, 4) suite.tc.AddLeaderRegion(2, 1, 2, 3, 4) suite.tc.SetLeaderSchedulePolicy("count") - testutil.CheckTransferLeader(suite.Require(), suite.schedule()[0], operator.OpKind(0), 2, 3) + operatorutil.CheckTransferLeader(suite.Require(), suite.schedule()[0], operator.OpKind(0), 2, 3) suite.tc.SetLeaderSchedulePolicy("size") - testutil.CheckTransferLeader(suite.Require(), suite.schedule()[0], operator.OpKind(0), 1, 4) + operatorutil.CheckTransferLeader(suite.Require(), suite.schedule()[0], operator.OpKind(0), 1, 4) } func (suite *balanceLeaderSchedulerTestSuite) TestBalanceSelector() { @@ -496,7 +495,7 @@ func (suite *balanceLeaderSchedulerTestSuite) TestBalanceSelector() { suite.tc.AddLeaderRegion(2, 3, 1, 2) // store4 has max leader score, store1 has min leader score. // The scheduler try to move a leader out of 16 first. - testutil.CheckTransferLeader(suite.Require(), suite.schedule()[0], operator.OpKind(0), 4, 2) + operatorutil.CheckTransferLeader(suite.Require(), suite.schedule()[0], operator.OpKind(0), 4, 2) // Stores: 1 2 3 4 // Leaders: 1 14 15 16 @@ -505,7 +504,7 @@ func (suite *balanceLeaderSchedulerTestSuite) TestBalanceSelector() { suite.tc.UpdateLeaderCount(2, 14) suite.tc.UpdateLeaderCount(3, 15) // Cannot move leader out of store4, move a leader into store1. - testutil.CheckTransferLeader(suite.Require(), suite.schedule()[0], operator.OpKind(0), 3, 1) + operatorutil.CheckTransferLeader(suite.Require(), suite.schedule()[0], operator.OpKind(0), 3, 1) // Stores: 1 2 3 4 // Leaders: 1 2 15 16 @@ -515,7 +514,7 @@ func (suite *balanceLeaderSchedulerTestSuite) TestBalanceSelector() { suite.tc.AddLeaderRegion(1, 3, 2, 4) suite.tc.AddLeaderRegion(2, 1, 2, 3) // No leader in store16, no follower in store1. Now source and target are store3 and store2. - testutil.CheckTransferLeader(suite.Require(), suite.schedule()[0], operator.OpKind(0), 3, 2) + operatorutil.CheckTransferLeader(suite.Require(), suite.schedule()[0], operator.OpKind(0), 3, 2) // Stores: 1 2 3 4 // Leaders: 9 10 10 11 @@ -538,14 +537,14 @@ func (suite *balanceLeaderSchedulerTestSuite) TestBalanceSelector() { suite.tc.AddLeaderStore(2, 13) suite.tc.AddLeaderStore(3, 0) suite.tc.AddLeaderStore(4, 16) - testutil.CheckTransferLeader(suite.Require(), suite.schedule()[0], operator.OpKind(0), 4, 3) + operatorutil.CheckTransferLeader(suite.Require(), suite.schedule()[0], operator.OpKind(0), 4, 3) } type balanceLeaderRangeSchedulerTestSuite struct { suite.Suite cancel context.CancelFunc tc *mockcluster.Cluster - oc *schedule.OperatorController + oc *operator.Controller } func TestBalanceLeaderRangeSchedulerTestSuite(t *testing.T) { @@ -573,34 +572,34 @@ func (suite *balanceLeaderRangeSchedulerTestSuite) TestSingleRangeBalance() { suite.tc.UpdateStoreLeaderWeight(3, 1) suite.tc.UpdateStoreLeaderWeight(4, 2) suite.tc.AddLeaderRegionWithRange(1, "a", "g", 1, 2, 3, 4) - lb, err := schedule.CreateScheduler(BalanceLeaderType, suite.oc, storage.NewStorageWithMemoryBackend(), schedule.ConfigSliceDecoder(BalanceLeaderType, []string{"", ""})) + lb, err := CreateScheduler(BalanceLeaderType, suite.oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(BalanceLeaderType, []string{"", ""})) suite.NoError(err) ops, _ := lb.Schedule(suite.tc, false) suite.NotEmpty(ops) suite.Len(ops, 1) suite.Len(ops[0].Counters, 1) suite.Len(ops[0].FinishedCounters, 3) - lb, err = schedule.CreateScheduler(BalanceLeaderType, suite.oc, storage.NewStorageWithMemoryBackend(), schedule.ConfigSliceDecoder(BalanceLeaderType, []string{"h", "n"})) + lb, err = CreateScheduler(BalanceLeaderType, suite.oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(BalanceLeaderType, []string{"h", "n"})) suite.NoError(err) ops, _ = lb.Schedule(suite.tc, false) suite.Empty(ops) - lb, err = schedule.CreateScheduler(BalanceLeaderType, suite.oc, storage.NewStorageWithMemoryBackend(), schedule.ConfigSliceDecoder(BalanceLeaderType, []string{"b", "f"})) + lb, err = CreateScheduler(BalanceLeaderType, suite.oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(BalanceLeaderType, []string{"b", "f"})) suite.NoError(err) ops, _ = lb.Schedule(suite.tc, false) suite.Empty(ops) - lb, err = schedule.CreateScheduler(BalanceLeaderType, suite.oc, storage.NewStorageWithMemoryBackend(), schedule.ConfigSliceDecoder(BalanceLeaderType, []string{"", "a"})) + lb, err = CreateScheduler(BalanceLeaderType, suite.oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(BalanceLeaderType, []string{"", "a"})) suite.NoError(err) ops, _ = lb.Schedule(suite.tc, false) suite.Empty(ops) - lb, err = schedule.CreateScheduler(BalanceLeaderType, suite.oc, storage.NewStorageWithMemoryBackend(), schedule.ConfigSliceDecoder(BalanceLeaderType, []string{"g", ""})) + lb, err = CreateScheduler(BalanceLeaderType, suite.oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(BalanceLeaderType, []string{"g", ""})) suite.NoError(err) ops, _ = lb.Schedule(suite.tc, false) suite.Empty(ops) - lb, err = schedule.CreateScheduler(BalanceLeaderType, suite.oc, storage.NewStorageWithMemoryBackend(), schedule.ConfigSliceDecoder(BalanceLeaderType, []string{"", "f"})) + lb, err = CreateScheduler(BalanceLeaderType, suite.oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(BalanceLeaderType, []string{"", "f"})) suite.NoError(err) ops, _ = lb.Schedule(suite.tc, false) suite.Empty(ops) - lb, err = schedule.CreateScheduler(BalanceLeaderType, suite.oc, storage.NewStorageWithMemoryBackend(), schedule.ConfigSliceDecoder(BalanceLeaderType, []string{"b", ""})) + lb, err = CreateScheduler(BalanceLeaderType, suite.oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(BalanceLeaderType, []string{"b", ""})) suite.NoError(err) ops, _ = lb.Schedule(suite.tc, false) suite.Empty(ops) @@ -619,7 +618,7 @@ func (suite *balanceLeaderRangeSchedulerTestSuite) TestMultiRangeBalance() { suite.tc.UpdateStoreLeaderWeight(3, 1) suite.tc.UpdateStoreLeaderWeight(4, 2) suite.tc.AddLeaderRegionWithRange(1, "a", "g", 1, 2, 3, 4) - lb, err := schedule.CreateScheduler(BalanceLeaderType, suite.oc, storage.NewStorageWithMemoryBackend(), schedule.ConfigSliceDecoder(BalanceLeaderType, []string{"", "g", "o", "t"})) + lb, err := CreateScheduler(BalanceLeaderType, suite.oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(BalanceLeaderType, []string{"", "g", "o", "t"})) suite.NoError(err) ops, _ := lb.Schedule(suite.tc, false) suite.Equal(uint64(1), ops[0].RegionID()) @@ -657,7 +656,7 @@ func (suite *balanceLeaderRangeSchedulerTestSuite) TestBatchBalance() { suite.tc.AddLeaderRegionWithRange(uint64(102), "102a", "102z", 1, 2, 3) suite.tc.AddLeaderRegionWithRange(uint64(103), "103a", "103z", 4, 5, 6) - lb, err := schedule.CreateScheduler(BalanceLeaderType, suite.oc, storage.NewStorageWithMemoryBackend(), schedule.ConfigSliceDecoder(BalanceLeaderType, []string{"", ""})) + lb, err := CreateScheduler(BalanceLeaderType, suite.oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(BalanceLeaderType, []string{"", ""})) suite.NoError(err) ops, _ := lb.Schedule(suite.tc, false) suite.Len(ops, 2) @@ -748,7 +747,7 @@ func checkBalanceRegionSchedule1(re *require.Assertions, enablePlacementRules bo tc.SetClusterVersion(versioninfo.MinSupportedVersion(versioninfo.Version4_0)) tc.SetEnablePlacementRules(enablePlacementRules) tc.SetMaxReplicasWithLabel(enablePlacementRules, 1) - sb, err := schedule.CreateScheduler(BalanceRegionType, oc, storage.NewStorageWithMemoryBackend(), schedule.ConfigSliceDecoder(BalanceRegionType, []string{"", ""})) + sb, err := CreateScheduler(BalanceRegionType, oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(BalanceRegionType, []string{"", ""})) re.NoError(err) // Add stores 1,2,3,4. tc.AddRegionStore(1, 6) @@ -759,7 +758,7 @@ func checkBalanceRegionSchedule1(re *require.Assertions, enablePlacementRules bo tc.AddLeaderRegion(1, 4) ops, _ := sb.Schedule(tc, false) op := ops[0] - testutil.CheckTransferPeerWithLeaderTransfer(re, op, operator.OpKind(0), 4, 1) + operatorutil.CheckTransferPeerWithLeaderTransfer(re, op, operator.OpKind(0), 4, 1) // Test stateFilter. tc.SetStoreOffline(1) @@ -769,7 +768,7 @@ func checkBalanceRegionSchedule1(re *require.Assertions, enablePlacementRules bo // store 2 becomes the store with least regions. ops, _ = sb.Schedule(tc, false) op = ops[0] - testutil.CheckTransferPeerWithLeaderTransfer(re, op, operator.OpKind(0), 4, 2) + operatorutil.CheckTransferPeerWithLeaderTransfer(re, op, operator.OpKind(0), 4, 2) tc.SetStoreUp(1) // test region replicate not match tc.SetMaxReplicasWithLabel(enablePlacementRules, 3) @@ -803,7 +802,7 @@ func checkReplica3(re *require.Assertions, enablePlacementRules bool) { tc.SetEnablePlacementRules(enablePlacementRules) tc.SetMaxReplicasWithLabel(enablePlacementRules, 3) - sb, err := schedule.CreateScheduler(BalanceRegionType, oc, storage.NewStorageWithMemoryBackend(), schedule.ConfigSliceDecoder(BalanceRegionType, []string{"", ""})) + sb, err := CreateScheduler(BalanceRegionType, oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(BalanceRegionType, []string{"", ""})) re.NoError(err) // Store 1 has the largest region score, so the balance scheduler tries to replace peer in store 1. tc.AddLabelsStore(1, 16, map[string]string{"zone": "z1", "rack": "r1", "host": "h1"}) @@ -819,37 +818,37 @@ func checkReplica3(re *require.Assertions, enablePlacementRules bool) { tc.AddLabelsStore(4, 2, map[string]string{"zone": "z1", "rack": "r2", "host": "h1"}) ops, _ = sb.Schedule(tc, false) op := ops[0] - testutil.CheckTransferPeer(re, op, operator.OpKind(0), 2, 4) + operatorutil.CheckTransferPeer(re, op, operator.OpKind(0), 2, 4) // Store 5 has smaller region score than store 1. tc.AddLabelsStore(5, 2, map[string]string{"zone": "z1", "rack": "r1", "host": "h1"}) ops, _ = sb.Schedule(tc, false) op = ops[0] - testutil.CheckTransferPeer(re, op, operator.OpKind(0), 1, 5) + operatorutil.CheckTransferPeer(re, op, operator.OpKind(0), 1, 5) // Store 6 has smaller region score than store 5. tc.AddLabelsStore(6, 1, map[string]string{"zone": "z1", "rack": "r1", "host": "h1"}) ops, _ = sb.Schedule(tc, false) op = ops[0] - testutil.CheckTransferPeer(re, op, operator.OpKind(0), 1, 6) + operatorutil.CheckTransferPeer(re, op, operator.OpKind(0), 1, 6) // Store 7 has smaller region score with store 6. tc.AddLabelsStore(7, 0, map[string]string{"zone": "z1", "rack": "r1", "host": "h2"}) ops, _ = sb.Schedule(tc, false) op = ops[0] - testutil.CheckTransferPeer(re, op, operator.OpKind(0), 1, 7) + operatorutil.CheckTransferPeer(re, op, operator.OpKind(0), 1, 7) // If store 7 is not available, will choose store 6. tc.SetStoreDown(7) ops, _ = sb.Schedule(tc, false) op = ops[0] - testutil.CheckTransferPeer(re, op, operator.OpKind(0), 1, 6) + operatorutil.CheckTransferPeer(re, op, operator.OpKind(0), 1, 6) // Store 8 has smaller region score than store 7, but the distinct score decrease. tc.AddLabelsStore(8, 1, map[string]string{"zone": "z1", "rack": "r2", "host": "h3"}) ops, _ = sb.Schedule(tc, false) op = ops[0] - testutil.CheckTransferPeer(re, op, operator.OpKind(0), 1, 6) + operatorutil.CheckTransferPeer(re, op, operator.OpKind(0), 1, 6) // Take down 4,5,6,7 tc.SetStoreDown(4) @@ -877,7 +876,7 @@ func checkReplica5(re *require.Assertions, enablePlacementRules bool) { tc.SetEnablePlacementRules(enablePlacementRules) tc.SetMaxReplicasWithLabel(enablePlacementRules, 5) - sb, err := schedule.CreateScheduler(BalanceRegionType, oc, storage.NewStorageWithMemoryBackend(), schedule.ConfigSliceDecoder(BalanceRegionType, []string{"", ""})) + sb, err := CreateScheduler(BalanceRegionType, oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(BalanceRegionType, []string{"", ""})) re.NoError(err) tc.AddLabelsStore(1, 4, map[string]string{"zone": "z1", "rack": "r1", "host": "h1"}) tc.AddLabelsStore(2, 5, map[string]string{"zone": "z2", "rack": "r1", "host": "h1"}) @@ -891,19 +890,19 @@ func checkReplica5(re *require.Assertions, enablePlacementRules bool) { tc.AddLabelsStore(6, 1, map[string]string{"zone": "z5", "rack": "r2", "host": "h1"}) ops, _ := sb.Schedule(tc, false) op := ops[0] - testutil.CheckTransferPeer(re, op, operator.OpKind(0), 5, 6) + operatorutil.CheckTransferPeer(re, op, operator.OpKind(0), 5, 6) // Store 7 has larger region score and same distinct score with store 6. tc.AddLabelsStore(7, 5, map[string]string{"zone": "z6", "rack": "r1", "host": "h1"}) ops, _ = sb.Schedule(tc, false) op = ops[0] - testutil.CheckTransferPeer(re, op, operator.OpKind(0), 5, 6) + operatorutil.CheckTransferPeer(re, op, operator.OpKind(0), 5, 6) // Store 1 has smaller region score and higher distinct score. tc.AddLeaderRegion(1, 2, 3, 4, 5, 6) ops, _ = sb.Schedule(tc, false) op = ops[0] - testutil.CheckTransferPeer(re, op, operator.OpKind(0), 5, 1) + operatorutil.CheckTransferPeer(re, op, operator.OpKind(0), 5, 1) // Store 6 has smaller region score and higher distinct score. tc.AddLabelsStore(11, 29, map[string]string{"zone": "z1", "rack": "r2", "host": "h1"}) @@ -912,7 +911,7 @@ func checkReplica5(re *require.Assertions, enablePlacementRules bool) { tc.AddLeaderRegion(1, 2, 3, 11, 12, 13) ops, _ = sb.Schedule(tc, false) op = ops[0] - testutil.CheckTransferPeer(re, op, operator.OpKind(0), 11, 6) + operatorutil.CheckTransferPeer(re, op, operator.OpKind(0), 11, 6) } // TestBalanceRegionSchedule2 for corner case 1: @@ -978,7 +977,7 @@ func checkBalanceRegionSchedule2(re *require.Assertions, enablePlacementRules bo core.SetApproximateKeys(200), ) - sb, err := schedule.CreateScheduler(BalanceRegionType, oc, storage.NewStorageWithMemoryBackend(), schedule.ConfigSliceDecoder(BalanceRegionType, []string{"", ""})) + sb, err := CreateScheduler(BalanceRegionType, oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(BalanceRegionType, []string{"", ""})) re.NoError(err) tc.AddRegionStore(1, 11) @@ -1001,7 +1000,7 @@ func checkBalanceRegionSchedule2(re *require.Assertions, enablePlacementRules bo // if the space of store 5 is normal, we can balance region to store 5 ops1, _ = sb.Schedule(tc, false) op = ops1[0] - testutil.CheckTransferPeer(re, op, operator.OpKind(0), 1, 5) + operatorutil.CheckTransferPeer(re, op, operator.OpKind(0), 1, 5) // the used size of store 5 reach (highSpace, lowSpace) origin := tc.GetStore(5) @@ -1019,7 +1018,7 @@ func checkBalanceRegionSchedule2(re *require.Assertions, enablePlacementRules bo // Then it will try store 4. ops1, _ = sb.Schedule(tc, false) op = ops1[0] - testutil.CheckTransferPeer(re, op, operator.OpKind(0), 1, 4) + operatorutil.CheckTransferPeer(re, op, operator.OpKind(0), 1, 4) } func TestBalanceRegionStoreWeight(t *testing.T) { @@ -1034,7 +1033,7 @@ func checkBalanceRegionStoreWeight(re *require.Assertions, enablePlacementRules tc.SetClusterVersion(versioninfo.MinSupportedVersion(versioninfo.Version4_0)) tc.SetEnablePlacementRules(enablePlacementRules) tc.SetMaxReplicasWithLabel(enablePlacementRules, 1) - sb, err := schedule.CreateScheduler(BalanceRegionType, oc, storage.NewStorageWithMemoryBackend(), schedule.ConfigSliceDecoder(BalanceRegionType, []string{"", ""})) + sb, err := CreateScheduler(BalanceRegionType, oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(BalanceRegionType, []string{"", ""})) re.NoError(err) tc.AddRegionStore(1, 10) @@ -1049,12 +1048,12 @@ func checkBalanceRegionStoreWeight(re *require.Assertions, enablePlacementRules tc.AddLeaderRegion(1, 1) ops, _ := sb.Schedule(tc, false) op := ops[0] - testutil.CheckTransferPeer(re, op, operator.OpKind(0), 1, 4) + operatorutil.CheckTransferPeer(re, op, operator.OpKind(0), 1, 4) tc.UpdateRegionCount(4, 30) ops, _ = sb.Schedule(tc, false) op = ops[0] - testutil.CheckTransferPeer(re, op, operator.OpKind(0), 1, 3) + operatorutil.CheckTransferPeer(re, op, operator.OpKind(0), 1, 3) } func TestBalanceRegionOpInfluence(t *testing.T) { @@ -1069,7 +1068,7 @@ func checkBalanceRegionOpInfluence(re *require.Assertions, enablePlacementRules tc.SetClusterVersion(versioninfo.MinSupportedVersion(versioninfo.Version4_0)) tc.SetEnablePlacementRules(enablePlacementRules) tc.SetMaxReplicasWithLabel(enablePlacementRules, 1) - sb, err := schedule.CreateScheduler(BalanceRegionType, oc, storage.NewStorageWithMemoryBackend(), schedule.ConfigSliceDecoder(BalanceRegionType, []string{"", ""})) + sb, err := CreateScheduler(BalanceRegionType, oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(BalanceRegionType, []string{"", ""})) re.NoError(err) // Add stores 1,2,3,4. tc.AddRegionStoreWithLeader(1, 2) @@ -1090,7 +1089,7 @@ func checkBalanceRegionOpInfluence(re *require.Assertions, enablePlacementRules } ops, _ := sb.Schedule(tc, false) op := ops[0] - testutil.CheckTransferPeerWithLeaderTransfer(re, op, operator.OpKind(0), 2, 1) + operatorutil.CheckTransferPeerWithLeaderTransfer(re, op, operator.OpKind(0), 2, 1) } func TestBalanceRegionReplacePendingRegion(t *testing.T) { @@ -1105,7 +1104,7 @@ func checkReplacePendingRegion(re *require.Assertions, enablePlacementRules bool tc.SetClusterVersion(versioninfo.MinSupportedVersion(versioninfo.Version4_0)) tc.SetEnablePlacementRules(enablePlacementRules) tc.SetMaxReplicasWithLabel(enablePlacementRules, 3) - sb, err := schedule.CreateScheduler(BalanceRegionType, oc, storage.NewStorageWithMemoryBackend(), schedule.ConfigSliceDecoder(BalanceRegionType, []string{"", ""})) + sb, err := CreateScheduler(BalanceRegionType, oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(BalanceRegionType, []string{"", ""})) re.NoError(err) // Store 1 has the largest region score, so the balance scheduler try to replace peer in store 1. tc.AddLabelsStore(1, 16, map[string]string{"zone": "z1", "rack": "r1", "host": "h1"}) @@ -1127,7 +1126,7 @@ func checkReplacePendingRegion(re *require.Assertions, enablePlacementRules bool re.Equal(uint64(3), op.RegionID()) ops, _ = sb.Schedule(tc, false) op = ops[0] - testutil.CheckTransferPeer(re, op, operator.OpKind(0), 1, 4) + operatorutil.CheckTransferPeer(re, op, operator.OpKind(0), 1, 4) } func TestBalanceRegionShouldNotBalance(t *testing.T) { @@ -1135,7 +1134,7 @@ func TestBalanceRegionShouldNotBalance(t *testing.T) { cancel, _, tc, oc := prepareSchedulersTest() defer cancel() tc.SetClusterVersion(versioninfo.MinSupportedVersion(versioninfo.Version4_0)) - sb, err := schedule.CreateScheduler(BalanceRegionType, oc, storage.NewStorageWithMemoryBackend(), schedule.ConfigSliceDecoder(BalanceRegionType, []string{"", ""})) + sb, err := CreateScheduler(BalanceRegionType, oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(BalanceRegionType, []string{"", ""})) re.NoError(err) region := tc.MockRegionInfo(1, 0, []uint64{2, 3, 4}, nil, nil) tc.PutRegion(region) @@ -1148,7 +1147,7 @@ func TestBalanceRegionEmptyRegion(t *testing.T) { cancel, _, tc, oc := prepareSchedulersTest() defer cancel() tc.SetClusterVersion(versioninfo.MinSupportedVersion(versioninfo.Version4_0)) - sb, err := schedule.CreateScheduler(BalanceRegionType, oc, storage.NewStorageWithMemoryBackend(), schedule.ConfigSliceDecoder(BalanceRegionType, []string{"", ""})) + sb, err := CreateScheduler(BalanceRegionType, oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(BalanceRegionType, []string{"", ""})) re.NoError(err) tc.AddRegionStore(1, 10) tc.AddRegionStore(2, 9) @@ -1194,7 +1193,7 @@ func checkRandomMergeSchedule(re *require.Assertions, enablePlacementRules bool) tc.SetMaxReplicasWithLabel(enablePlacementRules, 3) tc.SetMergeScheduleLimit(1) - mb, err := schedule.CreateScheduler(RandomMergeType, oc, storage.NewStorageWithMemoryBackend(), schedule.ConfigSliceDecoder(RandomMergeType, []string{"", ""})) + mb, err := CreateScheduler(RandomMergeType, oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(RandomMergeType, []string{"", ""})) re.NoError(err) tc.AddRegionStore(1, 4) @@ -1276,7 +1275,7 @@ func checkScatterRangeBalance(re *require.Assertions, enablePlacementRules bool) tc.UpdateStoreStatus(uint64(i)) } - hb, err := schedule.CreateScheduler(ScatterRangeType, oc, storage.NewStorageWithMemoryBackend(), schedule.ConfigSliceDecoder(ScatterRangeType, []string{"s_00", "s_50", "t"})) + hb, err := CreateScheduler(ScatterRangeType, oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(ScatterRangeType, []string{"s_00", "s_50", "t"})) re.NoError(err) scheduleAndApplyOperator(tc, hb, 100) @@ -1350,7 +1349,7 @@ func checkBalanceLeaderLimit(re *require.Assertions, enablePlacementRules bool) // test not allow schedule leader tc.SetLeaderScheduleLimit(0) - hb, err := schedule.CreateScheduler(ScatterRangeType, oc, storage.NewStorageWithMemoryBackend(), schedule.ConfigSliceDecoder(ScatterRangeType, []string{"s_00", "s_50", "t"})) + hb, err := CreateScheduler(ScatterRangeType, oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(ScatterRangeType, []string{"s_00", "s_50", "t"})) re.NoError(err) scheduleAndApplyOperator(tc, hb, 100) @@ -1374,7 +1373,7 @@ func TestConcurrencyUpdateConfig(t *testing.T) { re := require.New(t) cancel, _, tc, oc := prepareSchedulersTest() defer cancel() - hb, err := schedule.CreateScheduler(ScatterRangeType, oc, storage.NewStorageWithMemoryBackend(), schedule.ConfigSliceDecoder(ScatterRangeType, []string{"s_00", "s_50", "t"})) + hb, err := CreateScheduler(ScatterRangeType, oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(ScatterRangeType, []string{"s_00", "s_50", "t"})) sche := hb.(*scatterRangeScheduler) re.NoError(err) ch := make(chan struct{}) @@ -1447,14 +1446,14 @@ func TestBalanceWhenRegionNotHeartbeat(t *testing.T) { tc.UpdateStoreStatus(uint64(i)) } - hb, err := schedule.CreateScheduler(ScatterRangeType, oc, storage.NewStorageWithMemoryBackend(), schedule.ConfigSliceDecoder(ScatterRangeType, []string{"s_00", "s_09", "t"})) + hb, err := CreateScheduler(ScatterRangeType, oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(ScatterRangeType, []string{"s_00", "s_09", "t"})) re.NoError(err) scheduleAndApplyOperator(tc, hb, 100) } // scheduleAndApplyOperator will try to schedule for `count` times and apply the operator if the operator is created. -func scheduleAndApplyOperator(tc *mockcluster.Cluster, hb schedule.Scheduler, count int) { +func scheduleAndApplyOperator(tc *mockcluster.Cluster, hb Scheduler, count int) { limit := 0 for { if limit > count { @@ -1465,6 +1464,6 @@ func scheduleAndApplyOperator(tc *mockcluster.Cluster, hb schedule.Scheduler, co limit++ continue } - schedule.ApplyOperator(tc, ops[0]) + operator.ApplyOperator(tc, ops[0]) } } diff --git a/pkg/schedule/schedulers/balance_witness.go b/pkg/schedule/schedulers/balance_witness.go index 1db5149f419..55674356313 100644 --- a/pkg/schedule/schedulers/balance_witness.go +++ b/pkg/schedule/schedulers/balance_witness.go @@ -28,7 +28,7 @@ import ( "github.com/tikv/pd/pkg/core" "github.com/tikv/pd/pkg/core/constant" "github.com/tikv/pd/pkg/errs" - "github.com/tikv/pd/pkg/schedule" + sche "github.com/tikv/pd/pkg/schedule/core" "github.com/tikv/pd/pkg/schedule/filter" "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/plan" @@ -105,7 +105,7 @@ func (conf *balanceWitnessSchedulerConfig) Clone() *balanceWitnessSchedulerConfi } func (conf *balanceWitnessSchedulerConfig) persistLocked() error { - data, err := schedule.EncodeConfig(conf) + data, err := EncodeConfig(conf) if err != nil { return err } @@ -146,7 +146,6 @@ type balanceWitnessScheduler struct { name string conf *balanceWitnessSchedulerConfig handler http.Handler - opController *schedule.OperatorController filters []filter.Filter counter *prometheus.CounterVec filterCounter *filter.Counter @@ -154,7 +153,7 @@ type balanceWitnessScheduler struct { // newBalanceWitnessScheduler creates a scheduler that tends to keep witnesses on // each store balanced. -func newBalanceWitnessScheduler(opController *schedule.OperatorController, conf *balanceWitnessSchedulerConfig, options ...BalanceWitnessCreateOption) schedule.Scheduler { +func newBalanceWitnessScheduler(opController *operator.Controller, conf *balanceWitnessSchedulerConfig, options ...BalanceWitnessCreateOption) Scheduler { base := NewBaseScheduler(opController) s := &balanceWitnessScheduler{ BaseScheduler: base, @@ -162,7 +161,6 @@ func newBalanceWitnessScheduler(opController *schedule.OperatorController, conf name: BalanceWitnessName, conf: conf, handler: newbalanceWitnessHandler(conf), - opController: opController, counter: balanceWitnessCounter, filterCounter: filter.NewCounter(filter.BalanceWitness.String()), } @@ -170,7 +168,7 @@ func newBalanceWitnessScheduler(opController *schedule.OperatorController, conf option(s) } s.filters = []filter.Filter{ - &filter.StoreStateFilter{ActionScope: s.GetName(), MoveRegion: true}, + &filter.StoreStateFilter{ActionScope: s.GetName(), MoveRegion: true, OperatorLevel: constant.Medium}, filter.NewSpecialUseFilter(s.GetName()), } return s @@ -208,21 +206,21 @@ func (b *balanceWitnessScheduler) GetType() string { func (b *balanceWitnessScheduler) EncodeConfig() ([]byte, error) { b.conf.mu.RLock() defer b.conf.mu.RUnlock() - return schedule.EncodeConfig(b.conf) + return EncodeConfig(b.conf) } -func (b *balanceWitnessScheduler) IsScheduleAllowed(cluster schedule.Cluster) bool { - allowed := b.opController.OperatorCount(operator.OpWitness) < cluster.GetOpts().GetWitnessScheduleLimit() +func (b *balanceWitnessScheduler) IsScheduleAllowed(cluster sche.SchedulerCluster) bool { + allowed := b.OpController.OperatorCount(operator.OpWitness) < cluster.GetSchedulerConfig().GetWitnessScheduleLimit() if !allowed { operator.OperatorLimitCounter.WithLabelValues(b.GetType(), operator.OpWitness.String()).Inc() } return allowed } -func (b *balanceWitnessScheduler) Schedule(cluster schedule.Cluster, dryRun bool) ([]*operator.Operator, []plan.Plan) { +func (b *balanceWitnessScheduler) Schedule(cluster sche.SchedulerCluster, dryRun bool) ([]*operator.Operator, []plan.Plan) { b.conf.mu.RLock() defer b.conf.mu.RUnlock() - basePlan := NewBalanceSchedulerPlan() + basePlan := plan.NewBalanceSchedulerPlan() var collector *plan.Collector if dryRun { collector = plan.NewCollector(basePlan) @@ -230,7 +228,7 @@ func (b *balanceWitnessScheduler) Schedule(cluster schedule.Cluster, dryRun bool batch := b.conf.Batch schedulerCounter.WithLabelValues(b.GetName(), "schedule").Inc() - opInfluence := b.opController.GetOpInfluence(cluster) + opInfluence := b.OpController.GetOpInfluence(cluster.GetBasicCluster()) kind := constant.NewScheduleKind(constant.WitnessKind, constant.ByCount) solver := newSolver(basePlan, kind, cluster, opInfluence) @@ -238,7 +236,7 @@ func (b *balanceWitnessScheduler) Schedule(cluster schedule.Cluster, dryRun bool scoreFunc := func(store *core.StoreInfo) float64 { return store.WitnessScore(solver.GetOpInfluence(store.GetID())) } - sourceCandidate := newCandidateStores(filter.SelectSourceStores(stores, b.filters, cluster.GetOpts(), collector, b.filterCounter), false, scoreFunc) + sourceCandidate := newCandidateStores(filter.SelectSourceStores(stores, b.filters, cluster.GetSchedulerConfig(), collector, b.filterCounter), false, scoreFunc) usedRegions := make(map[uint64]struct{}) result := make([]*operator.Operator, 0, batch) @@ -259,10 +257,10 @@ func (b *balanceWitnessScheduler) Schedule(cluster schedule.Cluster, dryRun bool func createTransferWitnessOperator(cs *candidateStores, b *balanceWitnessScheduler, ssolver *solver, usedRegions map[uint64]struct{}, collector *plan.Collector) *operator.Operator { store := cs.getStore() - ssolver.step++ - defer func() { ssolver.step-- }() + ssolver.Step++ + defer func() { ssolver.Step-- }() retryLimit := b.retryQuota.GetLimit(store) - ssolver.source, ssolver.target = store, nil + ssolver.Source, ssolver.Target = store, nil var op *operator.Operator for i := 0; i < retryLimit; i++ { schedulerCounter.WithLabelValues(b.GetName(), "total").Inc() @@ -287,33 +285,33 @@ func createTransferWitnessOperator(cs *candidateStores, b *balanceWitnessSchedul // It randomly selects a health region from the source store, then picks // the best follower peer and transfers the witness. func (b *balanceWitnessScheduler) transferWitnessOut(solver *solver, collector *plan.Collector) *operator.Operator { - solver.region = filter.SelectOneRegion(solver.RandWitnessRegions(solver.SourceStoreID(), b.conf.Ranges), + solver.Region = filter.SelectOneRegion(solver.RandWitnessRegions(solver.SourceStoreID(), b.conf.Ranges), collector, filter.NewRegionPendingFilter(), filter.NewRegionDownFilter()) - if solver.region == nil { + if solver.Region == nil { log.Debug("store has no witness", zap.String("scheduler", b.GetName()), zap.Uint64("store-id", solver.SourceStoreID())) schedulerCounter.WithLabelValues(b.GetName(), "no-witness-region").Inc() return nil } - solver.step++ - defer func() { solver.step-- }() - targets := solver.GetNonWitnessVoterStores(solver.region) + solver.Step++ + defer func() { solver.Step-- }() + targets := solver.GetNonWitnessVoterStores(solver.Region) finalFilters := b.filters - opts := solver.GetOpts() - if witnessFilter := filter.NewPlacementWitnessSafeguard(b.GetName(), opts, solver.GetBasicCluster(), solver.GetRuleManager(), solver.region, solver.source, solver.fit); witnessFilter != nil { + conf := solver.GetSchedulerConfig() + if witnessFilter := filter.NewPlacementWitnessSafeguard(b.GetName(), conf, solver.GetBasicCluster(), solver.GetRuleManager(), solver.Region, solver.Source, solver.fit); witnessFilter != nil { finalFilters = append(b.filters, witnessFilter) } - targets = filter.SelectTargetStores(targets, finalFilters, opts, collector, b.filterCounter) + targets = filter.SelectTargetStores(targets, finalFilters, conf, collector, b.filterCounter) sort.Slice(targets, func(i, j int) bool { iOp := solver.GetOpInfluence(targets[i].GetID()) jOp := solver.GetOpInfluence(targets[j].GetID()) return targets[i].WitnessScore(iOp) < targets[j].WitnessScore(jOp) }) - for _, solver.target = range targets { + for _, solver.Target = range targets { if op := b.createOperator(solver, collector); op != nil { return op } } - log.Debug("region has no target store", zap.String("scheduler", b.GetName()), zap.Uint64("region-id", solver.region.GetID())) + log.Debug("region has no target store", zap.String("scheduler", b.GetName()), zap.Uint64("region-id", solver.Region.GetID())) schedulerCounter.WithLabelValues(b.GetName(), "no-target-store").Inc() return nil } @@ -323,8 +321,8 @@ func (b *balanceWitnessScheduler) transferWitnessOut(solver *solver, collector * // no new operator need to be created, otherwise create an operator that transfers // the witness from the source store to the target store for the region. func (b *balanceWitnessScheduler) createOperator(solver *solver, collector *plan.Collector) *operator.Operator { - solver.step++ - defer func() { solver.step-- }() + solver.Step++ + defer func() { solver.Step-- }() solver.sourceScore, solver.targetScore = solver.sourceStoreScore(b.GetName()), solver.targetStoreScore(b.GetName()) if !solver.shouldBalance(b.GetName()) { schedulerCounter.WithLabelValues(b.GetName(), "skip").Inc() @@ -333,9 +331,9 @@ func (b *balanceWitnessScheduler) createOperator(solver *solver, collector *plan } return nil } - solver.step++ - defer func() { solver.step-- }() - op, err := operator.CreateMoveWitnessOperator(BalanceWitnessType, solver, solver.region, solver.SourceStoreID(), solver.TargetStoreID()) + solver.Step++ + defer func() { solver.Step-- }() + op, err := operator.CreateMoveWitnessOperator(BalanceWitnessType, solver, solver.Region, solver.SourceStoreID(), solver.TargetStoreID()) if err != nil { log.Debug("fail to create balance witness operator", errs.ZapError(err)) return nil diff --git a/pkg/schedule/schedulers/balance_witness_test.go b/pkg/schedule/schedulers/balance_witness_test.go index d1342c3dc97..abd4a3b3bba 100644 --- a/pkg/schedule/schedulers/balance_witness_test.go +++ b/pkg/schedule/schedulers/balance_witness_test.go @@ -20,7 +20,6 @@ import ( "github.com/stretchr/testify/suite" "github.com/tikv/pd/pkg/mock/mockcluster" - "github.com/tikv/pd/pkg/schedule" "github.com/tikv/pd/pkg/schedule/config" "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/placement" @@ -35,9 +34,9 @@ type balanceWitnessSchedulerTestSuite struct { suite.Suite cancel context.CancelFunc tc *mockcluster.Cluster - lb schedule.Scheduler - oc *schedule.OperatorController - conf config.Config + lb Scheduler + oc *operator.Controller + conf config.SchedulerConfigProvider } func (suite *balanceWitnessSchedulerTestSuite) SetupTest() { @@ -50,7 +49,7 @@ func (suite *balanceWitnessSchedulerTestSuite) SetupTest() { Count: 4, }, }) - lb, err := schedule.CreateScheduler(BalanceWitnessType, suite.oc, storage.NewStorageWithMemoryBackend(), schedule.ConfigSliceDecoder(BalanceWitnessType, []string{"", ""})) + lb, err := CreateScheduler(BalanceWitnessType, suite.oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(BalanceWitnessType, []string{"", ""}), nil) suite.NoError(err) suite.lb = lb } diff --git a/pkg/schedule/schedulers/base_scheduler.go b/pkg/schedule/schedulers/base_scheduler.go index 9da9b55b04e..8da0f13626b 100644 --- a/pkg/schedule/schedulers/base_scheduler.go +++ b/pkg/schedule/schedulers/base_scheduler.go @@ -21,7 +21,8 @@ import ( "github.com/pingcap/log" "github.com/tikv/pd/pkg/errs" - "github.com/tikv/pd/pkg/schedule" + sche "github.com/tikv/pd/pkg/schedule/core" + "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/utils/typeutil" ) @@ -59,11 +60,11 @@ func intervalGrow(x time.Duration, maxInterval time.Duration, typ intervalGrowth // BaseScheduler is a basic scheduler for all other complex scheduler type BaseScheduler struct { - OpController *schedule.OperatorController + OpController *operator.Controller } // NewBaseScheduler returns a basic scheduler -func NewBaseScheduler(opController *schedule.OperatorController) *BaseScheduler { +func NewBaseScheduler(opController *operator.Controller) *BaseScheduler { return &BaseScheduler{OpController: opController} } @@ -78,7 +79,7 @@ func (s *BaseScheduler) GetMinInterval() time.Duration { // EncodeConfig encode config for the scheduler func (s *BaseScheduler) EncodeConfig() ([]byte, error) { - return schedule.EncodeConfig(nil) + return EncodeConfig(nil) } // GetNextInterval return the next interval for the scheduler @@ -87,7 +88,7 @@ func (s *BaseScheduler) GetNextInterval(interval time.Duration) time.Duration { } // Prepare does some prepare work -func (s *BaseScheduler) Prepare(cluster schedule.Cluster) error { return nil } +func (s *BaseScheduler) Prepare(cluster sche.SchedulerCluster) error { return nil } // Cleanup does some cleanup work -func (s *BaseScheduler) Cleanup(cluster schedule.Cluster) {} +func (s *BaseScheduler) Cleanup(cluster sche.SchedulerCluster) {} diff --git a/server/cluster/diagnostic_manager.go b/pkg/schedule/schedulers/diagnostic_recorder.go similarity index 57% rename from server/cluster/diagnostic_manager.go rename to pkg/schedule/schedulers/diagnostic_recorder.go index dc6ae042995..b990bdc8f22 100644 --- a/server/cluster/diagnostic_manager.go +++ b/pkg/schedule/schedulers/diagnostic_recorder.go @@ -1,4 +1,4 @@ -// Copyright 2022 TiKV Project Authors. +// Copyright 2023 TiKV Project Authors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,120 +12,77 @@ // See the License for the specific language governing permissions and // limitations under the License. -package cluster +package schedulers import ( "fmt" "time" - "github.com/pingcap/log" "github.com/tikv/pd/pkg/cache" - "github.com/tikv/pd/pkg/errs" "github.com/tikv/pd/pkg/movingaverage" + sc "github.com/tikv/pd/pkg/schedule/config" "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/plan" - "github.com/tikv/pd/pkg/schedule/schedulers" - "go.uber.org/zap" ) const ( - // disabled means the current scheduler is unavailable or removed - disabled = "disabled" - // paused means the current scheduler is paused - paused = "paused" - // scheduling means the current scheduler is generating. - scheduling = "scheduling" - // pending means the current scheduler cannot generate scheduling operator - pending = "pending" - // normal means that there is no need to create operators since everything is fine. - normal = "normal" + maxDiagnosticResultNum = 10 ) const ( - maxDiagnosticResultNum = 10 + // Disabled means the current scheduler is unavailable or removed + Disabled = "disabled" + // Paused means the current scheduler is paused + Paused = "paused" + // Halted means the current scheduler is halted + Halted = "halted" + // Scheduling means the current scheduler is generating. + Scheduling = "scheduling" + // Pending means the current scheduler cannot generate scheduling operator + Pending = "pending" + // Normal means that there is no need to create operators since everything is fine. + Normal = "normal" ) // DiagnosableSummaryFunc includes all implementations of plan.Summary. // And it also includes all schedulers which pd support to diagnose. var DiagnosableSummaryFunc = map[string]plan.Summary{ - schedulers.BalanceRegionName: schedulers.BalancePlanSummary, - schedulers.BalanceLeaderName: schedulers.BalancePlanSummary, -} - -type diagnosticManager struct { - cluster *RaftCluster - recorders map[string]*diagnosticRecorder -} - -func newDiagnosticManager(cluster *RaftCluster) *diagnosticManager { - recorders := make(map[string]*diagnosticRecorder) - for name := range DiagnosableSummaryFunc { - recorders[name] = newDiagnosticRecorder(name, cluster) - } - return &diagnosticManager{ - cluster: cluster, - recorders: recorders, - } -} - -func (d *diagnosticManager) getDiagnosticResult(name string) (*DiagnosticResult, error) { - if !d.cluster.opt.IsDiagnosticAllowed() { - return nil, errs.ErrDiagnosticDisabled - } - - isSchedulerExisted, _ := d.cluster.IsSchedulerExisted(name) - isDisabled, _ := d.cluster.IsSchedulerDisabled(name) - if !isSchedulerExisted || isDisabled { - ts := uint64(time.Now().Unix()) - res := &DiagnosticResult{Name: name, Timestamp: ts, Status: disabled} - return res, nil - } - - recorder := d.getRecorder(name) - if recorder == nil { - return nil, errs.ErrSchedulerUndiagnosable.FastGenByArgs(name) - } - result := recorder.getLastResult() - if result == nil { - return nil, errs.ErrNoDiagnosticResult.FastGenByArgs(name) - } - return result, nil -} - -func (d *diagnosticManager) getRecorder(name string) *diagnosticRecorder { - return d.recorders[name] + BalanceRegionName: plan.BalancePlanSummary, + BalanceLeaderName: plan.BalancePlanSummary, } -// diagnosticRecorder is used to manage diagnostic for one scheduler. -type diagnosticRecorder struct { +// DiagnosticRecorder is used to manage diagnostic for one scheduler. +type DiagnosticRecorder struct { schedulerName string - cluster *RaftCluster + config sc.SchedulerConfigProvider summaryFunc plan.Summary results *cache.FIFO } -func newDiagnosticRecorder(name string, cluster *RaftCluster) *diagnosticRecorder { +// NewDiagnosticRecorder creates a new DiagnosticRecorder. +func NewDiagnosticRecorder(name string, config sc.SchedulerConfigProvider) *DiagnosticRecorder { summaryFunc, ok := DiagnosableSummaryFunc[name] if !ok { - log.Error("can't find summary function", zap.String("scheduler-name", name)) return nil } - return &diagnosticRecorder{ - cluster: cluster, + return &DiagnosticRecorder{ schedulerName: name, + config: config, summaryFunc: summaryFunc, results: cache.NewFIFO(maxDiagnosticResultNum), } } -func (d *diagnosticRecorder) isAllowed() bool { +// IsAllowed is used to check whether the diagnostic is allowed. +func (d *DiagnosticRecorder) IsAllowed() bool { if d == nil { return false } - return d.cluster.opt.IsDiagnosticAllowed() + return d.config.IsDiagnosticAllowed() } -func (d *diagnosticRecorder) getLastResult() *DiagnosticResult { +// GetLastResult is used to get the last diagnostic result. +func (d *DiagnosticRecorder) GetLastResult() *DiagnosticResult { if d.results.Len() == 0 { return nil } @@ -143,7 +100,7 @@ func (d *diagnosticRecorder) getLastResult() *DiagnosticResult { var resStr string firstStatus := items[0].Value.(*DiagnosticResult).Status - if firstStatus == pending || firstStatus == normal { + if firstStatus == Pending || firstStatus == Normal { wa := movingaverage.NewWeightAllocator(length, 3) counter := make(map[uint64]map[plan.Status]float64) for i := 0; i < length; i++ { @@ -172,7 +129,7 @@ func (d *diagnosticRecorder) getLastResult() *DiagnosticResult { for k, v := range statusCounter { resStr += fmt.Sprintf("%d store(s) %s; ", v, k.String()) } - } else if firstStatus == pending { + } else if firstStatus == Pending { // This is used to handle pending status because of reach limit in `IsScheduleAllowed` resStr = fmt.Sprintf("%s reach limit", d.schedulerName) } @@ -185,7 +142,8 @@ func (d *diagnosticRecorder) getLastResult() *DiagnosticResult { } } -func (d *diagnosticRecorder) setResultFromStatus(status string) { +// SetResultFromStatus is used to set result from status. +func (d *DiagnosticRecorder) SetResultFromStatus(status string) { if d == nil { return } @@ -193,7 +151,8 @@ func (d *diagnosticRecorder) setResultFromStatus(status string) { d.results.Put(result.Timestamp, result) } -func (d *diagnosticRecorder) setResultFromPlans(ops []*operator.Operator, plans []plan.Plan) { +// SetResultFromPlans is used to set result from plans. +func (d *DiagnosticRecorder) SetResultFromPlans(ops []*operator.Operator, plans []plan.Plan) { if d == nil { return } @@ -201,22 +160,22 @@ func (d *diagnosticRecorder) setResultFromPlans(ops []*operator.Operator, plans d.results.Put(result.Timestamp, result) } -func (d *diagnosticRecorder) analyze(ops []*operator.Operator, plans []plan.Plan, ts uint64) *DiagnosticResult { - res := &DiagnosticResult{Name: d.schedulerName, Timestamp: ts, Status: normal} +func (d *DiagnosticRecorder) analyze(ops []*operator.Operator, plans []plan.Plan, ts uint64) *DiagnosticResult { + res := &DiagnosticResult{Name: d.schedulerName, Timestamp: ts, Status: Normal} name := d.schedulerName // TODO: support more schedulers and checkers switch name { - case schedulers.BalanceRegionName, schedulers.BalanceLeaderName: + case BalanceRegionName, BalanceLeaderName: if len(ops) != 0 { - res.Status = scheduling + res.Status = Scheduling return res } - res.Status = pending + res.Status = Pending if d.summaryFunc != nil { isAllNormal := false res.StoreStatus, isAllNormal, _ = d.summaryFunc(plans) if isAllNormal { - res.Status = normal + res.Status = Normal } } return res diff --git a/pkg/schedule/schedulers/evict_leader.go b/pkg/schedule/schedulers/evict_leader.go index 0db548bc93c..b307bf5fb73 100644 --- a/pkg/schedule/schedulers/evict_leader.go +++ b/pkg/schedule/schedulers/evict_leader.go @@ -25,7 +25,7 @@ import ( "github.com/tikv/pd/pkg/core" "github.com/tikv/pd/pkg/core/constant" "github.com/tikv/pd/pkg/errs" - "github.com/tikv/pd/pkg/schedule" + sche "github.com/tikv/pd/pkg/schedule/core" "github.com/tikv/pd/pkg/schedule/filter" "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/plan" @@ -59,7 +59,8 @@ type evictLeaderSchedulerConfig struct { mu syncutil.RWMutex storage endpoint.ConfigStorage StoreIDWithRanges map[uint64][]core.KeyRange `json:"store-id-ranges"` - cluster schedule.Cluster + cluster *core.BasicCluster + removeSchedulerCb func(string) error } func (conf *evictLeaderSchedulerConfig) getStores() []uint64 { @@ -107,7 +108,7 @@ func (conf *evictLeaderSchedulerConfig) Persist() error { name := conf.getSchedulerName() conf.mu.RLock() defer conf.mu.RUnlock() - data, err := schedule.EncodeConfig(conf) + data, err := EncodeConfig(conf) failpoint.Inject("persistFail", func() { err = errors.New("fail to persist") }) @@ -170,7 +171,7 @@ type evictLeaderScheduler struct { // newEvictLeaderScheduler creates an admin scheduler that transfers all leaders // out of a store. -func newEvictLeaderScheduler(opController *schedule.OperatorController, conf *evictLeaderSchedulerConfig) schedule.Scheduler { +func newEvictLeaderScheduler(opController *operator.Controller, conf *evictLeaderSchedulerConfig) Scheduler { base := NewBaseScheduler(opController) handler := newEvictLeaderHandler(conf) return &evictLeaderScheduler{ @@ -200,10 +201,10 @@ func (s *evictLeaderScheduler) GetType() string { func (s *evictLeaderScheduler) EncodeConfig() ([]byte, error) { s.conf.mu.RLock() defer s.conf.mu.RUnlock() - return schedule.EncodeConfig(s.conf) + return EncodeConfig(s.conf) } -func (s *evictLeaderScheduler) Prepare(cluster schedule.Cluster) error { +func (s *evictLeaderScheduler) Prepare(cluster sche.SchedulerCluster) error { s.conf.mu.RLock() defer s.conf.mu.RUnlock() var res error @@ -215,7 +216,7 @@ func (s *evictLeaderScheduler) Prepare(cluster schedule.Cluster) error { return res } -func (s *evictLeaderScheduler) Cleanup(cluster schedule.Cluster) { +func (s *evictLeaderScheduler) Cleanup(cluster sche.SchedulerCluster) { s.conf.mu.RLock() defer s.conf.mu.RUnlock() for id := range s.conf.StoreIDWithRanges { @@ -223,15 +224,15 @@ func (s *evictLeaderScheduler) Cleanup(cluster schedule.Cluster) { } } -func (s *evictLeaderScheduler) IsScheduleAllowed(cluster schedule.Cluster) bool { - allowed := s.OpController.OperatorCount(operator.OpLeader) < cluster.GetOpts().GetLeaderScheduleLimit() +func (s *evictLeaderScheduler) IsScheduleAllowed(cluster sche.SchedulerCluster) bool { + allowed := s.OpController.OperatorCount(operator.OpLeader) < cluster.GetSchedulerConfig().GetLeaderScheduleLimit() if !allowed { operator.OperatorLimitCounter.WithLabelValues(s.GetType(), operator.OpLeader.String()).Inc() } return allowed } -func (s *evictLeaderScheduler) Schedule(cluster schedule.Cluster, dryRun bool) ([]*operator.Operator, []plan.Plan) { +func (s *evictLeaderScheduler) Schedule(cluster sche.SchedulerCluster, dryRun bool) ([]*operator.Operator, []plan.Plan) { evictLeaderCounter.Inc() return scheduleEvictLeaderBatch(s.GetName(), s.GetType(), cluster, s.conf, EvictLeaderBatchSize), nil } @@ -256,7 +257,7 @@ type evictLeaderStoresConf interface { getKeyRangesByID(id uint64) []core.KeyRange } -func scheduleEvictLeaderBatch(name, typ string, cluster schedule.Cluster, conf evictLeaderStoresConf, batchSize int) []*operator.Operator { +func scheduleEvictLeaderBatch(name, typ string, cluster sche.SchedulerCluster, conf evictLeaderStoresConf, batchSize int) []*operator.Operator { var ops []*operator.Operator for i := 0; i < batchSize; i++ { once := scheduleEvictLeaderOnce(name, typ, cluster, conf) @@ -273,7 +274,7 @@ func scheduleEvictLeaderBatch(name, typ string, cluster schedule.Cluster, conf e return ops } -func scheduleEvictLeaderOnce(name, typ string, cluster schedule.Cluster, conf evictLeaderStoresConf) []*operator.Operator { +func scheduleEvictLeaderOnce(name, typ string, cluster sche.SchedulerCluster, conf evictLeaderStoresConf) []*operator.Operator { stores := conf.getStores() ops := make([]*operator.Operator, 0, len(stores)) for _, storeID := range stores { @@ -303,9 +304,9 @@ func scheduleEvictLeaderOnce(name, typ string, cluster schedule.Cluster, conf ev filters = append(filters, filter.NewExcludedFilter(name, nil, unhealthyPeerStores)) } - filters = append(filters, &filter.StoreStateFilter{ActionScope: name, TransferLeader: true}) + filters = append(filters, &filter.StoreStateFilter{ActionScope: name, TransferLeader: true, OperatorLevel: constant.Urgent}) candidates := filter.NewCandidates(cluster.GetFollowerStores(region)). - FilterTarget(cluster.GetOpts(), nil, nil, filters...) + FilterTarget(cluster.GetSchedulerConfig(), nil, nil, filters...) // Compatible with old TiKV transfer leader logic. target := candidates.RandomPick() targets := candidates.PickAll() @@ -399,7 +400,7 @@ func (handler *evictLeaderHandler) DeleteConfig(w http.ResponseWriter, r *http.R return } if last { - if err := handler.config.cluster.RemoveScheduler(EvictLeaderName); err != nil { + if err := handler.config.removeSchedulerCb(EvictLeaderName); err != nil { if errors.ErrorEqual(err, errs.ErrSchedulerNotFound.FastGenByArgs()) { handler.rd.JSON(w, http.StatusNotFound, err.Error()) } else { diff --git a/pkg/schedule/schedulers/evict_leader_test.go b/pkg/schedule/schedulers/evict_leader_test.go index 37262e81593..d804561f11c 100644 --- a/pkg/schedule/schedulers/evict_leader_test.go +++ b/pkg/schedule/schedulers/evict_leader_test.go @@ -22,10 +22,9 @@ import ( "github.com/pingcap/kvproto/pkg/pdpb" "github.com/stretchr/testify/require" "github.com/tikv/pd/pkg/core" - "github.com/tikv/pd/pkg/schedule" "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/storage" - "github.com/tikv/pd/pkg/utils/testutil" + "github.com/tikv/pd/pkg/utils/operatorutil" ) func TestEvictLeader(t *testing.T) { @@ -42,11 +41,11 @@ func TestEvictLeader(t *testing.T) { tc.AddLeaderRegion(2, 2, 1) tc.AddLeaderRegion(3, 3, 1) - sl, err := schedule.CreateScheduler(EvictLeaderType, oc, storage.NewStorageWithMemoryBackend(), schedule.ConfigSliceDecoder(EvictLeaderType, []string{"1"})) + sl, err := CreateScheduler(EvictLeaderType, oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(EvictLeaderType, []string{"1"}), func(string) error { return nil }) re.NoError(err) re.True(sl.IsScheduleAllowed(tc)) ops, _ := sl.Schedule(tc, false) - testutil.CheckMultiTargetTransferLeader(re, ops[0], operator.OpLeader, 1, []uint64{2, 3}) + operatorutil.CheckMultiTargetTransferLeader(re, ops[0], operator.OpLeader, 1, []uint64{2, 3}) re.False(ops[0].Step(0).(operator.TransferLeader).IsFinish(tc.MockRegionInfo(1, 1, []uint64{2, 3}, []uint64{}, &metapb.RegionEpoch{ConfVer: 0, Version: 0}))) re.True(ops[0].Step(0).(operator.TransferLeader).IsFinish(tc.MockRegionInfo(1, 2, []uint64{1, 3}, []uint64{}, &metapb.RegionEpoch{ConfVer: 0, Version: 0}))) } @@ -55,7 +54,7 @@ func TestEvictLeaderWithUnhealthyPeer(t *testing.T) { re := require.New(t) cancel, _, tc, oc := prepareSchedulersTest() defer cancel() - sl, err := schedule.CreateScheduler(EvictLeaderType, oc, storage.NewStorageWithMemoryBackend(), schedule.ConfigSliceDecoder(EvictLeaderType, []string{"1"})) + sl, err := CreateScheduler(EvictLeaderType, oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(EvictLeaderType, []string{"1"}), func(string) error { return nil }) re.NoError(err) // Add stores 1, 2, 3 @@ -74,11 +73,11 @@ func TestEvictLeaderWithUnhealthyPeer(t *testing.T) { // only pending tc.PutRegion(region.Clone(withPendingPeer)) ops, _ := sl.Schedule(tc, false) - testutil.CheckMultiTargetTransferLeader(re, ops[0], operator.OpLeader, 1, []uint64{3}) + operatorutil.CheckMultiTargetTransferLeader(re, ops[0], operator.OpLeader, 1, []uint64{3}) // only down tc.PutRegion(region.Clone(withDownPeer)) ops, _ = sl.Schedule(tc, false) - testutil.CheckMultiTargetTransferLeader(re, ops[0], operator.OpLeader, 1, []uint64{2}) + operatorutil.CheckMultiTargetTransferLeader(re, ops[0], operator.OpLeader, 1, []uint64{2}) // pending + down tc.PutRegion(region.Clone(withPendingPeer, withDownPeer)) ops, _ = sl.Schedule(tc, false) diff --git a/pkg/schedule/schedulers/evict_slow_store.go b/pkg/schedule/schedulers/evict_slow_store.go index 70dde65a38c..ead92378398 100644 --- a/pkg/schedule/schedulers/evict_slow_store.go +++ b/pkg/schedule/schedulers/evict_slow_store.go @@ -19,7 +19,7 @@ import ( "github.com/pingcap/failpoint" "github.com/pingcap/log" "github.com/tikv/pd/pkg/core" - "github.com/tikv/pd/pkg/schedule" + sche "github.com/tikv/pd/pkg/schedule/core" "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/plan" "github.com/tikv/pd/pkg/storage/endpoint" @@ -46,7 +46,7 @@ type evictSlowStoreSchedulerConfig struct { func (conf *evictSlowStoreSchedulerConfig) Persist() error { name := conf.getSchedulerName() - data, err := schedule.EncodeConfig(conf) + data, err := EncodeConfig(conf) failpoint.Inject("persistFail", func() { err = errors.New("fail to persist") }) @@ -106,10 +106,10 @@ func (s *evictSlowStoreScheduler) GetType() string { } func (s *evictSlowStoreScheduler) EncodeConfig() ([]byte, error) { - return schedule.EncodeConfig(s.conf) + return EncodeConfig(s.conf) } -func (s *evictSlowStoreScheduler) Prepare(cluster schedule.Cluster) error { +func (s *evictSlowStoreScheduler) Prepare(cluster sche.SchedulerCluster) error { evictStore := s.conf.evictStore() if evictStore != 0 { return cluster.SlowStoreEvicted(evictStore) @@ -117,11 +117,11 @@ func (s *evictSlowStoreScheduler) Prepare(cluster schedule.Cluster) error { return nil } -func (s *evictSlowStoreScheduler) Cleanup(cluster schedule.Cluster) { +func (s *evictSlowStoreScheduler) Cleanup(cluster sche.SchedulerCluster) { s.cleanupEvictLeader(cluster) } -func (s *evictSlowStoreScheduler) prepareEvictLeader(cluster schedule.Cluster, storeID uint64) error { +func (s *evictSlowStoreScheduler) prepareEvictLeader(cluster sche.SchedulerCluster, storeID uint64) error { err := s.conf.setStoreAndPersist(storeID) if err != nil { log.Info("evict-slow-store-scheduler persist config failed", zap.Uint64("store-id", storeID)) @@ -131,7 +131,7 @@ func (s *evictSlowStoreScheduler) prepareEvictLeader(cluster schedule.Cluster, s return cluster.SlowStoreEvicted(storeID) } -func (s *evictSlowStoreScheduler) cleanupEvictLeader(cluster schedule.Cluster) { +func (s *evictSlowStoreScheduler) cleanupEvictLeader(cluster sche.SchedulerCluster) { evictSlowStore, err := s.conf.clearAndPersist() if err != nil { log.Info("evict-slow-store-scheduler persist config failed", zap.Uint64("store-id", evictSlowStore)) @@ -142,13 +142,13 @@ func (s *evictSlowStoreScheduler) cleanupEvictLeader(cluster schedule.Cluster) { cluster.SlowStoreRecovered(evictSlowStore) } -func (s *evictSlowStoreScheduler) schedulerEvictLeader(cluster schedule.Cluster) []*operator.Operator { +func (s *evictSlowStoreScheduler) schedulerEvictLeader(cluster sche.SchedulerCluster) []*operator.Operator { return scheduleEvictLeaderBatch(s.GetName(), s.GetType(), cluster, s.conf, EvictLeaderBatchSize) } -func (s *evictSlowStoreScheduler) IsScheduleAllowed(cluster schedule.Cluster) bool { +func (s *evictSlowStoreScheduler) IsScheduleAllowed(cluster sche.SchedulerCluster) bool { if s.conf.evictStore() != 0 { - allowed := s.OpController.OperatorCount(operator.OpLeader) < cluster.GetOpts().GetLeaderScheduleLimit() + allowed := s.OpController.OperatorCount(operator.OpLeader) < cluster.GetSchedulerConfig().GetLeaderScheduleLimit() if !allowed { operator.OperatorLimitCounter.WithLabelValues(s.GetType(), operator.OpLeader.String()).Inc() } @@ -157,7 +157,7 @@ func (s *evictSlowStoreScheduler) IsScheduleAllowed(cluster schedule.Cluster) bo return true } -func (s *evictSlowStoreScheduler) Schedule(cluster schedule.Cluster, dryRun bool) ([]*operator.Operator, []plan.Plan) { +func (s *evictSlowStoreScheduler) Schedule(cluster sche.SchedulerCluster, dryRun bool) ([]*operator.Operator, []plan.Plan) { evictSlowStoreCounter.Inc() var ops []*operator.Operator @@ -210,7 +210,7 @@ func (s *evictSlowStoreScheduler) Schedule(cluster schedule.Cluster, dryRun bool } // newEvictSlowStoreScheduler creates a scheduler that detects and evicts slow stores. -func newEvictSlowStoreScheduler(opController *schedule.OperatorController, conf *evictSlowStoreSchedulerConfig) schedule.Scheduler { +func newEvictSlowStoreScheduler(opController *operator.Controller, conf *evictSlowStoreSchedulerConfig) Scheduler { base := NewBaseScheduler(opController) s := &evictSlowStoreScheduler{ diff --git a/pkg/schedule/schedulers/evict_slow_store_test.go b/pkg/schedule/schedulers/evict_slow_store_test.go index 845b3b1b93c..58a44118048 100644 --- a/pkg/schedule/schedulers/evict_slow_store_test.go +++ b/pkg/schedule/schedulers/evict_slow_store_test.go @@ -24,19 +24,18 @@ import ( "github.com/stretchr/testify/suite" "github.com/tikv/pd/pkg/core" "github.com/tikv/pd/pkg/mock/mockcluster" - "github.com/tikv/pd/pkg/schedule" "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/storage" - "github.com/tikv/pd/pkg/utils/testutil" + "github.com/tikv/pd/pkg/utils/operatorutil" ) type evictSlowStoreTestSuite struct { suite.Suite cancel context.CancelFunc tc *mockcluster.Cluster - es schedule.Scheduler - bs schedule.Scheduler - oc *schedule.OperatorController + es Scheduler + bs Scheduler + oc *operator.Controller } func TestEvictSlowStoreTestSuite(t *testing.T) { @@ -57,9 +56,9 @@ func (suite *evictSlowStoreTestSuite) SetupTest() { storage := storage.NewStorageWithMemoryBackend() var err error - suite.es, err = schedule.CreateScheduler(EvictSlowStoreType, suite.oc, storage, schedule.ConfigSliceDecoder(EvictSlowStoreType, []string{})) + suite.es, err = CreateScheduler(EvictSlowStoreType, suite.oc, storage, ConfigSliceDecoder(EvictSlowStoreType, []string{}), nil) suite.NoError(err) - suite.bs, err = schedule.CreateScheduler(BalanceLeaderType, suite.oc, storage, schedule.ConfigSliceDecoder(BalanceLeaderType, []string{})) + suite.bs, err = CreateScheduler(BalanceLeaderType, suite.oc, storage, ConfigSliceDecoder(BalanceLeaderType, []string{}), nil) suite.NoError(err) } @@ -76,7 +75,7 @@ func (suite *evictSlowStoreTestSuite) TestEvictSlowStore() { suite.True(suite.es.IsScheduleAllowed(suite.tc)) // Add evict leader scheduler to store 1 ops, _ := suite.es.Schedule(suite.tc, false) - testutil.CheckMultiTargetTransferLeader(suite.Require(), ops[0], operator.OpLeader, 1, []uint64{2}) + operatorutil.CheckMultiTargetTransferLeader(suite.Require(), ops[0], operator.OpLeader, 1, []uint64{2}) suite.Equal(EvictSlowStoreType, ops[0].Desc()) // Cannot balance leaders to store 1 ops, _ = suite.bs.Schedule(suite.tc, false) @@ -89,7 +88,7 @@ func (suite *evictSlowStoreTestSuite) TestEvictSlowStore() { ops, _ = suite.es.Schedule(suite.tc, false) suite.Empty(ops) ops, _ = suite.bs.Schedule(suite.tc, false) - testutil.CheckTransferLeader(suite.Require(), ops[0], operator.OpLeader, 2, 1) + operatorutil.CheckTransferLeader(suite.Require(), ops[0], operator.OpLeader, 2, 1) // no slow store need to evict. ops, _ = suite.es.Schedule(suite.tc, false) diff --git a/pkg/schedule/schedulers/evict_slow_trend.go b/pkg/schedule/schedulers/evict_slow_trend.go index 85e29f2fd36..067f9e517f2 100644 --- a/pkg/schedule/schedulers/evict_slow_trend.go +++ b/pkg/schedule/schedulers/evict_slow_trend.go @@ -18,16 +18,15 @@ import ( "strconv" "time" - "go.uber.org/zap" - "github.com/pingcap/errors" "github.com/pingcap/failpoint" "github.com/pingcap/log" "github.com/tikv/pd/pkg/core" - "github.com/tikv/pd/pkg/schedule" + sche "github.com/tikv/pd/pkg/schedule/core" "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/plan" "github.com/tikv/pd/pkg/storage/endpoint" + "go.uber.org/zap" ) const ( @@ -48,7 +47,7 @@ type evictSlowTrendSchedulerConfig struct { func (conf *evictSlowTrendSchedulerConfig) Persist() error { name := conf.getSchedulerName() - data, err := schedule.EncodeConfig(conf) + data, err := EncodeConfig(conf) failpoint.Inject("persistFail", func() { err = errors.New("fail to persist") }) @@ -84,6 +83,10 @@ func (conf *evictSlowTrendSchedulerConfig) candidate() uint64 { return conf.evictCandidate } +func (conf *evictSlowTrendSchedulerConfig) captureTS() time.Time { + return conf.candidateCaptureTime +} + func (conf *evictSlowTrendSchedulerConfig) candidateCapturedSecs() uint64 { return uint64(time.Since(conf.candidateCaptureTime).Seconds()) } @@ -104,7 +107,7 @@ func (conf *evictSlowTrendSchedulerConfig) setStoreAndPersist(id uint64) error { return conf.Persist() } -func (conf *evictSlowTrendSchedulerConfig) clearAndPersist(cluster schedule.Cluster) (oldID uint64, err error) { +func (conf *evictSlowTrendSchedulerConfig) clearAndPersist(cluster sche.SchedulerCluster) (oldID uint64, err error) { oldID = conf.evictedStore() if oldID == 0 { return @@ -133,10 +136,10 @@ func (s *evictSlowTrendScheduler) GetType() string { } func (s *evictSlowTrendScheduler) EncodeConfig() ([]byte, error) { - return schedule.EncodeConfig(s.conf) + return EncodeConfig(s.conf) } -func (s *evictSlowTrendScheduler) Prepare(cluster schedule.Cluster) error { +func (s *evictSlowTrendScheduler) Prepare(cluster sche.SchedulerCluster) error { evictedStoreID := s.conf.evictedStore() if evictedStoreID == 0 { return nil @@ -144,11 +147,11 @@ func (s *evictSlowTrendScheduler) Prepare(cluster schedule.Cluster) error { return cluster.SlowTrendEvicted(evictedStoreID) } -func (s *evictSlowTrendScheduler) Cleanup(cluster schedule.Cluster) { +func (s *evictSlowTrendScheduler) Cleanup(cluster sche.SchedulerCluster) { s.cleanupEvictLeader(cluster) } -func (s *evictSlowTrendScheduler) prepareEvictLeader(cluster schedule.Cluster, storeID uint64) error { +func (s *evictSlowTrendScheduler) prepareEvictLeader(cluster sche.SchedulerCluster, storeID uint64) error { err := s.conf.setStoreAndPersist(storeID) if err != nil { log.Info("evict-slow-trend-scheduler persist config failed", zap.Uint64("store-id", storeID)) @@ -157,7 +160,7 @@ func (s *evictSlowTrendScheduler) prepareEvictLeader(cluster schedule.Cluster, s return cluster.SlowTrendEvicted(storeID) } -func (s *evictSlowTrendScheduler) cleanupEvictLeader(cluster schedule.Cluster) { +func (s *evictSlowTrendScheduler) cleanupEvictLeader(cluster sche.SchedulerCluster) { evictedStoreID, err := s.conf.clearAndPersist(cluster) if err != nil { log.Info("evict-slow-trend-scheduler persist config failed", zap.Uint64("store-id", evictedStoreID)) @@ -167,7 +170,7 @@ func (s *evictSlowTrendScheduler) cleanupEvictLeader(cluster schedule.Cluster) { } } -func (s *evictSlowTrendScheduler) scheduleEvictLeader(cluster schedule.Cluster) []*operator.Operator { +func (s *evictSlowTrendScheduler) scheduleEvictLeader(cluster sche.SchedulerCluster) []*operator.Operator { store := cluster.GetStore(s.conf.evictedStore()) if store == nil { return nil @@ -176,18 +179,18 @@ func (s *evictSlowTrendScheduler) scheduleEvictLeader(cluster schedule.Cluster) return scheduleEvictLeaderBatch(s.GetName(), s.GetType(), cluster, s.conf, EvictLeaderBatchSize) } -func (s *evictSlowTrendScheduler) IsScheduleAllowed(cluster schedule.Cluster) bool { +func (s *evictSlowTrendScheduler) IsScheduleAllowed(cluster sche.SchedulerCluster) bool { if s.conf.evictedStore() == 0 { return true } - allowed := s.OpController.OperatorCount(operator.OpLeader) < cluster.GetOpts().GetLeaderScheduleLimit() + allowed := s.OpController.OperatorCount(operator.OpLeader) < cluster.GetSchedulerConfig().GetLeaderScheduleLimit() if !allowed { operator.OperatorLimitCounter.WithLabelValues(s.GetType(), operator.OpLeader.String()).Inc() } return allowed } -func (s *evictSlowTrendScheduler) Schedule(cluster schedule.Cluster, dryRun bool) ([]*operator.Operator, []plan.Plan) { +func (s *evictSlowTrendScheduler) Schedule(cluster sche.SchedulerCluster, dryRun bool) ([]*operator.Operator, []plan.Plan) { schedulerCounter.WithLabelValues(s.GetName(), "schedule").Inc() var ops []*operator.Operator @@ -237,7 +240,8 @@ func (s *evictSlowTrendScheduler) Schedule(cluster schedule.Cluster, dryRun bool storeSlowTrendActionStatusGauge.WithLabelValues("cand.cancel:too-faster").Inc() return ops, nil } - if !checkStoresAreUpdated(cluster, slowStore) { + slowStoreRecordTS := s.conf.captureTS() + if !checkStoresAreUpdated(cluster, slowStoreID, slowStoreRecordTS) { log.Info("slow store candidate waiting for other stores to update heartbeats", zap.Uint64("store-id", slowStoreID)) storeSlowTrendActionStatusGauge.WithLabelValues("cand.wait").Inc() @@ -259,14 +263,14 @@ func (s *evictSlowTrendScheduler) Schedule(cluster schedule.Cluster, dryRun bool return s.scheduleEvictLeader(cluster), nil } -func newEvictSlowTrendScheduler(opController *schedule.OperatorController, conf *evictSlowTrendSchedulerConfig) schedule.Scheduler { +func newEvictSlowTrendScheduler(opController *operator.Controller, conf *evictSlowTrendSchedulerConfig) Scheduler { return &evictSlowTrendScheduler{ BaseScheduler: NewBaseScheduler(opController), conf: conf, } } -func chooseEvictCandidate(cluster schedule.Cluster) (slowStore *core.StoreInfo) { +func chooseEvictCandidate(cluster sche.SchedulerCluster) (slowStore *core.StoreInfo) { stores := cluster.GetStores() if len(stores) < 3 { storeSlowTrendActionStatusGauge.WithLabelValues("cand.none:too-few").Inc() @@ -308,7 +312,7 @@ func chooseEvictCandidate(cluster schedule.Cluster) (slowStore *core.StoreInfo) } store := candidates[0] - affectedStoreThreshold := int(float64(len(stores)) * cluster.GetOpts().GetSlowStoreEvictingAffectedStoreRatioThreshold()) + affectedStoreThreshold := int(float64(len(stores)) * cluster.GetSchedulerConfig().GetSlowStoreEvictingAffectedStoreRatioThreshold()) if affectedStoreCount < affectedStoreThreshold { log.Info("evict-slow-trend-scheduler failed to confirm candidate: it only affect a few stores", zap.Uint64("store-id", store.GetID())) storeSlowTrendActionStatusGauge.WithLabelValues("cand.none:affect-a-few").Inc() @@ -322,18 +326,17 @@ func chooseEvictCandidate(cluster schedule.Cluster) (slowStore *core.StoreInfo) } storeSlowTrendActionStatusGauge.WithLabelValues("cand.add").Inc() - log.Info("evict-slow-trend-scheduler canptured candidate", zap.Uint64("store-id", store.GetID())) + log.Info("evict-slow-trend-scheduler captured candidate", zap.Uint64("store-id", store.GetID())) return store } -func checkStoresAreUpdated(cluster schedule.Cluster, baseline *core.StoreInfo) bool { +func checkStoresAreUpdated(cluster sche.SchedulerCluster, slowStoreID uint64, slowStoreRecordTS time.Time) bool { stores := cluster.GetStores() if len(stores) <= 1 { return false } expected := (len(stores) + 1) / 2 updatedStores := 0 - baselineTS := baseline.GetLastHeartbeatTS() for _, store := range stores { if store.IsRemoved() { updatedStores += 1 @@ -343,11 +346,11 @@ func checkStoresAreUpdated(cluster schedule.Cluster, baseline *core.StoreInfo) b updatedStores += 1 continue } - if store.GetID() == baseline.GetID() { + if store.GetID() == slowStoreID { updatedStores += 1 continue } - if baselineTS.Before(store.GetLastHeartbeatTS()) { + if slowStoreRecordTS.Before(store.GetLastHeartbeatTS()) { updatedStores += 1 } } @@ -356,7 +359,7 @@ func checkStoresAreUpdated(cluster schedule.Cluster, baseline *core.StoreInfo) b return updatedStores >= expected } -func checkStoreSlowerThanOthers(cluster schedule.Cluster, target *core.StoreInfo) bool { +func checkStoreSlowerThanOthers(cluster sche.SchedulerCluster, target *core.StoreInfo) bool { stores := cluster.GetStores() expected := (len(stores)*2 + 1) / 3 targetSlowTrend := target.GetSlowTrend() @@ -387,7 +390,7 @@ func checkStoreSlowerThanOthers(cluster schedule.Cluster, target *core.StoreInfo return slowerThanStoresNum >= expected } -func checkStoreCanRecover(cluster schedule.Cluster, target *core.StoreInfo) bool { +func checkStoreCanRecover(cluster sche.SchedulerCluster, target *core.StoreInfo) bool { /* // // This might not be necessary, @@ -410,7 +413,7 @@ func checkStoreCanRecover(cluster schedule.Cluster, target *core.StoreInfo) bool return checkStoreFasterThanOthers(cluster, target) } -func checkStoreFasterThanOthers(cluster schedule.Cluster, target *core.StoreInfo) bool { +func checkStoreFasterThanOthers(cluster sche.SchedulerCluster, target *core.StoreInfo) bool { stores := cluster.GetStores() expected := (len(stores) + 1) / 2 targetSlowTrend := target.GetSlowTrend() diff --git a/pkg/schedule/schedulers/evict_slow_trend_test.go b/pkg/schedule/schedulers/evict_slow_trend_test.go index 3227d285014..c4ff7e501e8 100644 --- a/pkg/schedule/schedulers/evict_slow_trend_test.go +++ b/pkg/schedule/schedulers/evict_slow_trend_test.go @@ -24,19 +24,18 @@ import ( "github.com/stretchr/testify/suite" "github.com/tikv/pd/pkg/core" "github.com/tikv/pd/pkg/mock/mockcluster" - "github.com/tikv/pd/pkg/schedule" "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/storage" - "github.com/tikv/pd/pkg/utils/testutil" + "github.com/tikv/pd/pkg/utils/operatorutil" ) type evictSlowTrendTestSuite struct { suite.Suite cancel context.CancelFunc tc *mockcluster.Cluster - es schedule.Scheduler - bs schedule.Scheduler - oc *schedule.OperatorController + es Scheduler + bs Scheduler + oc *operator.Controller } func TestEvictSlowTrendTestSuite(t *testing.T) { @@ -69,9 +68,9 @@ func (suite *evictSlowTrendTestSuite) SetupTest() { storage := storage.NewStorageWithMemoryBackend() var err error - suite.es, err = schedule.CreateScheduler(EvictSlowTrendType, suite.oc, storage, schedule.ConfigSliceDecoder(EvictSlowTrendType, []string{})) + suite.es, err = CreateScheduler(EvictSlowTrendType, suite.oc, storage, ConfigSliceDecoder(EvictSlowTrendType, []string{})) suite.NoError(err) - suite.bs, err = schedule.CreateScheduler(BalanceLeaderType, suite.oc, storage, schedule.ConfigSliceDecoder(BalanceLeaderType, []string{})) + suite.bs, err = CreateScheduler(BalanceLeaderType, suite.oc, storage, ConfigSliceDecoder(BalanceLeaderType, []string{})) suite.NoError(err) } @@ -111,7 +110,7 @@ func (suite *evictSlowTrendTestSuite) TestEvictSlowTrend() { suite.tc.PutStore(newStoreInfo) } ops, _ = suite.es.Schedule(suite.tc, false) - testutil.CheckMultiTargetTransferLeader(suite.Require(), ops[0], operator.OpLeader, 1, []uint64{2, 3}) + operatorutil.CheckMultiTargetTransferLeader(suite.Require(), ops[0], operator.OpLeader, 1, []uint64{2, 3}) suite.Equal(EvictSlowTrendType, ops[0].Desc()) suite.Equal(es2.conf.candidate(), uint64(0)) suite.Equal(es2.conf.evictedStore(), uint64(1)) @@ -134,7 +133,7 @@ func (suite *evictSlowTrendTestSuite) TestEvictSlowTrend() { suite.Empty(ops) suite.Zero(es2.conf.evictedStore()) ops, _ = suite.bs.Schedule(suite.tc, false) - testutil.CheckTransferLeader(suite.Require(), ops[0], operator.OpLeader, 3, 1) + operatorutil.CheckTransferLeader(suite.Require(), ops[0], operator.OpLeader, 3, 1) // no slow store need to evict. ops, _ = suite.es.Schedule(suite.tc, false) diff --git a/pkg/schedule/schedulers/grant_hot_region.go b/pkg/schedule/schedulers/grant_hot_region.go index e04d91b1a9b..d0c0cc9a1be 100644 --- a/pkg/schedule/schedulers/grant_hot_region.go +++ b/pkg/schedule/schedulers/grant_hot_region.go @@ -23,9 +23,10 @@ import ( "github.com/gorilla/mux" "github.com/pingcap/kvproto/pkg/metapb" "github.com/pingcap/log" + "github.com/tikv/pd/pkg/core" "github.com/tikv/pd/pkg/core/constant" "github.com/tikv/pd/pkg/errs" - "github.com/tikv/pd/pkg/schedule" + sche "github.com/tikv/pd/pkg/schedule/core" "github.com/tikv/pd/pkg/schedule/filter" "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/plan" @@ -54,7 +55,7 @@ var ( type grantHotRegionSchedulerConfig struct { mu syncutil.RWMutex storage endpoint.ConfigStorage - cluster schedule.Cluster + cluster *core.BasicCluster StoreIDs []uint64 `json:"store-id"` StoreLeaderID uint64 `json:"store-leader-id"` } @@ -99,7 +100,7 @@ func (conf *grantHotRegionSchedulerConfig) Persist() error { name := conf.getSchedulerName() conf.mu.RLock() defer conf.mu.RUnlock() - data, err := schedule.EncodeConfig(conf) + data, err := EncodeConfig(conf) if err != nil { return err } @@ -126,7 +127,7 @@ type grantHotRegionScheduler struct { } // newGrantHotRegionScheduler creates an admin scheduler that transfers hot region peer to fixed store and hot region leader to one store. -func newGrantHotRegionScheduler(opController *schedule.OperatorController, conf *grantHotRegionSchedulerConfig) *grantHotRegionScheduler { +func newGrantHotRegionScheduler(opController *operator.Controller, conf *grantHotRegionSchedulerConfig) *grantHotRegionScheduler { base := newBaseHotScheduler(opController) handler := newGrantHotRegionHandler(conf) ret := &grantHotRegionScheduler{ @@ -146,14 +147,15 @@ func (s *grantHotRegionScheduler) GetType() string { } func (s *grantHotRegionScheduler) EncodeConfig() ([]byte, error) { - return schedule.EncodeConfig(s.conf) + return EncodeConfig(s.conf) } // IsScheduleAllowed returns whether the scheduler is allowed to schedule. // TODO it should check if there is any scheduler such as evict or hot region scheduler -func (s *grantHotRegionScheduler) IsScheduleAllowed(cluster schedule.Cluster) bool { - regionAllowed := s.OpController.OperatorCount(operator.OpRegion) < cluster.GetOpts().GetRegionScheduleLimit() - leaderAllowed := s.OpController.OperatorCount(operator.OpLeader) < cluster.GetOpts().GetLeaderScheduleLimit() +func (s *grantHotRegionScheduler) IsScheduleAllowed(cluster sche.SchedulerCluster) bool { + conf := cluster.GetSchedulerConfig() + regionAllowed := s.OpController.OperatorCount(operator.OpRegion) < conf.GetRegionScheduleLimit() + leaderAllowed := s.OpController.OperatorCount(operator.OpLeader) < conf.GetLeaderScheduleLimit() if !regionAllowed { operator.OperatorLimitCounter.WithLabelValues(s.GetType(), operator.OpRegion.String()).Inc() } @@ -225,14 +227,14 @@ func newGrantHotRegionHandler(config *grantHotRegionSchedulerConfig) http.Handle return router } -func (s *grantHotRegionScheduler) Schedule(cluster schedule.Cluster, dryRun bool) ([]*operator.Operator, []plan.Plan) { +func (s *grantHotRegionScheduler) Schedule(cluster sche.SchedulerCluster, dryRun bool) ([]*operator.Operator, []plan.Plan) { grantHotRegionCounter.Inc() rw := s.randomRWType() s.prepareForBalance(rw, cluster) return s.dispatch(rw, cluster), nil } -func (s *grantHotRegionScheduler) dispatch(typ statistics.RWType, cluster schedule.Cluster) []*operator.Operator { +func (s *grantHotRegionScheduler) dispatch(typ statistics.RWType, cluster sche.SchedulerCluster) []*operator.Operator { stLoadInfos := s.stLoadInfos[buildResourceType(typ, constant.RegionKind)] infos := make([]*statistics.StoreLoadDetail, len(stLoadInfos)) index := 0 @@ -246,7 +248,7 @@ func (s *grantHotRegionScheduler) dispatch(typ statistics.RWType, cluster schedu return s.randomSchedule(cluster, infos) } -func (s *grantHotRegionScheduler) randomSchedule(cluster schedule.Cluster, srcStores []*statistics.StoreLoadDetail) (ops []*operator.Operator) { +func (s *grantHotRegionScheduler) randomSchedule(cluster sche.SchedulerCluster, srcStores []*statistics.StoreLoadDetail) (ops []*operator.Operator) { isLeader := s.r.Int()%2 == 1 for _, srcStore := range srcStores { srcStoreID := srcStore.GetID() @@ -277,7 +279,7 @@ func (s *grantHotRegionScheduler) randomSchedule(cluster schedule.Cluster, srcSt return nil } -func (s *grantHotRegionScheduler) transfer(cluster schedule.Cluster, regionID uint64, srcStoreID uint64, isLeader bool) (op *operator.Operator, err error) { +func (s *grantHotRegionScheduler) transfer(cluster sche.SchedulerCluster, regionID uint64, srcStoreID uint64, isLeader bool) (op *operator.Operator, err error) { srcRegion := cluster.GetRegion(regionID) if srcRegion == nil || len(srcRegion.GetDownPeers()) != 0 || len(srcRegion.GetPendingPeers()) != 0 { return nil, errs.ErrRegionRuleNotFound @@ -288,22 +290,22 @@ func (s *grantHotRegionScheduler) transfer(cluster schedule.Cluster, regionID ui return nil, errs.ErrStoreNotFound } filters := []filter.Filter{ - filter.NewPlacementSafeguard(s.GetName(), cluster.GetOpts(), cluster.GetBasicCluster(), cluster.GetRuleManager(), srcRegion, srcStore, nil), + filter.NewPlacementSafeguard(s.GetName(), cluster.GetSchedulerConfig(), cluster.GetBasicCluster(), cluster.GetRuleManager(), srcRegion, srcStore, nil), } destStoreIDs := make([]uint64, 0, len(s.conf.StoreIDs)) var candidate []uint64 if isLeader { - filters = append(filters, &filter.StoreStateFilter{ActionScope: s.GetName(), TransferLeader: true}) + filters = append(filters, &filter.StoreStateFilter{ActionScope: s.GetName(), TransferLeader: true, OperatorLevel: constant.High}) candidate = []uint64{s.conf.GetStoreLeaderID()} } else { - filters = append(filters, &filter.StoreStateFilter{ActionScope: s.GetName(), MoveRegion: true}, + filters = append(filters, &filter.StoreStateFilter{ActionScope: s.GetName(), MoveRegion: true, OperatorLevel: constant.High}, filter.NewExcludedFilter(s.GetName(), srcRegion.GetStoreIDs(), srcRegion.GetStoreIDs())) candidate = s.conf.StoreIDs } for _, storeID := range candidate { store := cluster.GetStore(storeID) - if !filter.Target(cluster.GetOpts(), store, filters) { + if !filter.Target(cluster.GetSchedulerConfig(), store, filters) { continue } destStoreIDs = append(destStoreIDs, storeID) diff --git a/pkg/schedule/schedulers/grant_leader.go b/pkg/schedule/schedulers/grant_leader.go index 5f504d05172..08856d101cc 100644 --- a/pkg/schedule/schedulers/grant_leader.go +++ b/pkg/schedule/schedulers/grant_leader.go @@ -24,7 +24,7 @@ import ( "github.com/tikv/pd/pkg/core" "github.com/tikv/pd/pkg/core/constant" "github.com/tikv/pd/pkg/errs" - "github.com/tikv/pd/pkg/schedule" + sche "github.com/tikv/pd/pkg/schedule/core" "github.com/tikv/pd/pkg/schedule/filter" "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/plan" @@ -52,7 +52,8 @@ type grantLeaderSchedulerConfig struct { mu syncutil.RWMutex storage endpoint.ConfigStorage StoreIDWithRanges map[uint64][]core.KeyRange `json:"store-id-ranges"` - cluster schedule.Cluster + cluster *core.BasicCluster + removeSchedulerCb func(name string) error } func (conf *grantLeaderSchedulerConfig) BuildWithArgs(args []string) error { @@ -90,7 +91,7 @@ func (conf *grantLeaderSchedulerConfig) Persist() error { name := conf.getSchedulerName() conf.mu.RLock() defer conf.mu.RUnlock() - data, err := schedule.EncodeConfig(conf) + data, err := EncodeConfig(conf) if err != nil { return err } @@ -151,7 +152,7 @@ type grantLeaderScheduler struct { // newGrantLeaderScheduler creates an admin scheduler that transfers all leaders // to a store. -func newGrantLeaderScheduler(opController *schedule.OperatorController, conf *grantLeaderSchedulerConfig) schedule.Scheduler { +func newGrantLeaderScheduler(opController *operator.Controller, conf *grantLeaderSchedulerConfig) Scheduler { base := NewBaseScheduler(opController) handler := newGrantLeaderHandler(conf) return &grantLeaderScheduler{ @@ -174,10 +175,10 @@ func (s *grantLeaderScheduler) GetType() string { } func (s *grantLeaderScheduler) EncodeConfig() ([]byte, error) { - return schedule.EncodeConfig(s.conf) + return EncodeConfig(s.conf) } -func (s *grantLeaderScheduler) Prepare(cluster schedule.Cluster) error { +func (s *grantLeaderScheduler) Prepare(cluster sche.SchedulerCluster) error { s.conf.mu.RLock() defer s.conf.mu.RUnlock() var res error @@ -189,7 +190,7 @@ func (s *grantLeaderScheduler) Prepare(cluster schedule.Cluster) error { return res } -func (s *grantLeaderScheduler) Cleanup(cluster schedule.Cluster) { +func (s *grantLeaderScheduler) Cleanup(cluster sche.SchedulerCluster) { s.conf.mu.RLock() defer s.conf.mu.RUnlock() for id := range s.conf.StoreIDWithRanges { @@ -197,15 +198,15 @@ func (s *grantLeaderScheduler) Cleanup(cluster schedule.Cluster) { } } -func (s *grantLeaderScheduler) IsScheduleAllowed(cluster schedule.Cluster) bool { - allowed := s.OpController.OperatorCount(operator.OpLeader) < cluster.GetOpts().GetLeaderScheduleLimit() +func (s *grantLeaderScheduler) IsScheduleAllowed(cluster sche.SchedulerCluster) bool { + allowed := s.OpController.OperatorCount(operator.OpLeader) < cluster.GetSchedulerConfig().GetLeaderScheduleLimit() if !allowed { operator.OperatorLimitCounter.WithLabelValues(s.GetType(), operator.OpLeader.String()).Inc() } return allowed } -func (s *grantLeaderScheduler) Schedule(cluster schedule.Cluster, dryRun bool) ([]*operator.Operator, []plan.Plan) { +func (s *grantLeaderScheduler) Schedule(cluster sche.SchedulerCluster, dryRun bool) ([]*operator.Operator, []plan.Plan) { grantLeaderCounter.Inc() s.conf.mu.RLock() defer s.conf.mu.RUnlock() @@ -301,7 +302,7 @@ func (handler *grantLeaderHandler) DeleteConfig(w http.ResponseWriter, r *http.R return } if last { - if err := handler.config.cluster.RemoveScheduler(GrantLeaderName); err != nil { + if err := handler.config.removeSchedulerCb(GrantLeaderName); err != nil { if errors.ErrorEqual(err, errs.ErrSchedulerNotFound.FastGenByArgs()) { handler.rd.JSON(w, http.StatusNotFound, err.Error()) } else { diff --git a/pkg/schedule/schedulers/hot_region.go b/pkg/schedule/schedulers/hot_region.go index 49e302cfd46..1100e980843 100644 --- a/pkg/schedule/schedulers/hot_region.go +++ b/pkg/schedule/schedulers/hot_region.go @@ -24,34 +24,48 @@ import ( "time" "github.com/pingcap/kvproto/pkg/metapb" + "github.com/pingcap/kvproto/pkg/pdpb" "github.com/pingcap/log" "github.com/prometheus/client_golang/prometheus" "github.com/tikv/pd/pkg/core" "github.com/tikv/pd/pkg/core/constant" "github.com/tikv/pd/pkg/errs" - "github.com/tikv/pd/pkg/schedule" + sche "github.com/tikv/pd/pkg/schedule/core" "github.com/tikv/pd/pkg/schedule/filter" "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/plan" "github.com/tikv/pd/pkg/slice" "github.com/tikv/pd/pkg/statistics" + "github.com/tikv/pd/pkg/statistics/buckets" + "github.com/tikv/pd/pkg/utils/keyutil" "github.com/tikv/pd/pkg/utils/syncutil" "go.uber.org/zap" ) var ( + topnPosition = 10 statisticsInterval = time.Second // WithLabelValues is a heavy operation, define variable to avoid call it every time. - hotSchedulerCounter = schedulerCounter.WithLabelValues(HotRegionName, "schedule") - hotSchedulerSkipCounter = schedulerCounter.WithLabelValues(HotRegionName, "skip") - hotSchedulerNeedSplitBeforeScheduleCounter = schedulerCounter.WithLabelValues(HotRegionName, "need_split_before_move_peer") - hotSchedulerSearchRevertRegionsCounter = schedulerCounter.WithLabelValues(HotRegionName, "search_revert_regions") - hotSchedulerNotSameEngineCounter = schedulerCounter.WithLabelValues(HotRegionName, "not_same_engine") - hotSchedulerNoRegionCounter = schedulerCounter.WithLabelValues(HotRegionName, "no_region") - hotSchedulerUnhealthyReplicaCounter = schedulerCounter.WithLabelValues(HotRegionName, "unhealthy_replica") - hotSchedulerAbnormalReplicaCounter = schedulerCounter.WithLabelValues(HotRegionName, "abnormal_replica") - hotSchedulerCreateOperatorFailedCounter = schedulerCounter.WithLabelValues(HotRegionName, "create_operator_failed") - hotSchedulerNewOperatorCounter = schedulerCounter.WithLabelValues(HotRegionName, "new_operator") + hotSchedulerCounter = schedulerCounter.WithLabelValues(HotRegionName, "schedule") + hotSchedulerSkipCounter = schedulerCounter.WithLabelValues(HotRegionName, "skip") + hotSchedulerSearchRevertRegionsCounter = schedulerCounter.WithLabelValues(HotRegionName, "search_revert_regions") + hotSchedulerNotSameEngineCounter = schedulerCounter.WithLabelValues(HotRegionName, "not_same_engine") + hotSchedulerNoRegionCounter = schedulerCounter.WithLabelValues(HotRegionName, "no_region") + hotSchedulerUnhealthyReplicaCounter = schedulerCounter.WithLabelValues(HotRegionName, "unhealthy_replica") + hotSchedulerAbnormalReplicaCounter = schedulerCounter.WithLabelValues(HotRegionName, "abnormal_replica") + hotSchedulerCreateOperatorFailedCounter = schedulerCounter.WithLabelValues(HotRegionName, "create_operator_failed") + hotSchedulerNewOperatorCounter = schedulerCounter.WithLabelValues(HotRegionName, "new_operator") + hotSchedulerSnapshotSenderLimitCounter = schedulerCounter.WithLabelValues(HotRegionName, "snapshot_sender_limit") + + // counter related with the split region + hotSchedulerNotFoundSplitKeysCounter = schedulerCounter.WithLabelValues(HotRegionName, "not_found_split_keys") + hotSchedulerRegionBucketsNotHotCounter = schedulerCounter.WithLabelValues(HotRegionName, "region_buckets_not_hot") + hotSchedulerOnlyOneBucketsHotCounter = schedulerCounter.WithLabelValues(HotRegionName, "only_one_buckets_hot") + hotSchedulerHotBucketNotValidCounter = schedulerCounter.WithLabelValues(HotRegionName, "hot_buckets_not_valid") + hotSchedulerRegionBucketsSingleHotSpotCounter = schedulerCounter.WithLabelValues(HotRegionName, "region_buckets_single_hot_spot") + hotSchedulerSplitSuccessCounter = schedulerCounter.WithLabelValues(HotRegionName, "split_success") + hotSchedulerNeedSplitBeforeScheduleCounter = schedulerCounter.WithLabelValues(HotRegionName, "need_split_before_move_peer") + hotSchedulerRegionTooHotNeedSplitCounter = schedulerCounter.WithLabelValues(HotRegionName, "region_is_too_hot_need_split") hotSchedulerMoveLeaderCounter = schedulerCounter.WithLabelValues(HotRegionName, moveLeader.String()) hotSchedulerMovePeerCounter = schedulerCounter.WithLabelValues(HotRegionName, movePeer.String()) @@ -76,7 +90,8 @@ type baseHotScheduler struct { stInfos map[uint64]*statistics.StoreSummaryInfo // temporary states but exported to API or metrics // Every time `Schedule()` will recalculate it. - stLoadInfos [resourceTypeLen]map[uint64]*statistics.StoreLoadDetail + stLoadInfos [resourceTypeLen]map[uint64]*statistics.StoreLoadDetail + stHistoryLoads *statistics.StoreHistoryLoads // temporary states // Every time `Schedule()` will recalculate it. storesLoads map[uint64][]float64 @@ -90,12 +105,13 @@ type baseHotScheduler struct { updateWriteTime time.Time } -func newBaseHotScheduler(opController *schedule.OperatorController) *baseHotScheduler { +func newBaseHotScheduler(opController *operator.Controller) *baseHotScheduler { base := NewBaseScheduler(opController) ret := &baseHotScheduler{ BaseScheduler: base, types: []statistics.RWType{statistics.Write, statistics.Read}, regionPendings: make(map[uint64]*pendingInfluence), + stHistoryLoads: statistics.NewStoreHistoryLoads(statistics.DimLen), r: rand.New(rand.NewSource(time.Now().UnixNano())), } for ty := resourceType(0); ty < resourceTypeLen; ty++ { @@ -106,17 +122,18 @@ func newBaseHotScheduler(opController *schedule.OperatorController) *baseHotSche // prepareForBalance calculate the summary of pending Influence for each store and prepare the load detail for // each store, only update read or write load detail -func (h *baseHotScheduler) prepareForBalance(rw statistics.RWType, cluster schedule.Cluster) { +func (h *baseHotScheduler) prepareForBalance(rw statistics.RWType, cluster sche.SchedulerCluster) { h.stInfos = statistics.SummaryStoreInfos(cluster.GetStores()) - h.summaryPendingInfluence(cluster) + h.summaryPendingInfluence() h.storesLoads = cluster.GetStoresLoads() - isTraceRegionFlow := cluster.GetOpts().IsTraceRegionFlow() + isTraceRegionFlow := cluster.GetSchedulerConfig().IsTraceRegionFlow() prepare := func(regionStats map[uint64][]*statistics.HotPeerStat, resource constant.ResourceKind) { ty := buildResourceType(rw, resource) h.stLoadInfos[ty] = statistics.SummaryStoresLoad( h.stInfos, h.storesLoads, + h.stHistoryLoads, regionStats, isTraceRegionFlow, rw, resource) @@ -144,35 +161,42 @@ func (h *baseHotScheduler) prepareForBalance(rw statistics.RWType, cluster sched // summaryPendingInfluence calculate the summary of pending Influence for each store // and clean the region from regionInfluence if they have ended operator. // It makes each dim rate or count become `weight` times to the origin value. -func (h *baseHotScheduler) summaryPendingInfluence(cluster schedule.Cluster) { +func (h *baseHotScheduler) summaryPendingInfluence() { for id, p := range h.regionPendings { - from := h.stInfos[p.from] - to := h.stInfos[p.to] - maxZombieDur := p.maxZombieDuration - weight, needGC := calcPendingInfluence(p.op, maxZombieDur) - - if needGC { - delete(h.regionPendings, id) - continue - } + for _, from := range p.froms { + from := h.stInfos[from] + to := h.stInfos[p.to] + maxZombieDur := p.maxZombieDuration + weight, needGC := calcPendingInfluence(p.op, maxZombieDur) + + if needGC { + delete(h.regionPendings, id) + continue + } - if from != nil && weight > 0 { - from.AddInfluence(&p.origin, -weight) - } - if to != nil && weight > 0 { - to.AddInfluence(&p.origin, weight) + if from != nil && weight > 0 { + from.AddInfluence(&p.origin, -weight) + } + if to != nil && weight > 0 { + to.AddInfluence(&p.origin, weight) + } } } for storeID, info := range h.stInfos { storeLabel := strconv.FormatUint(storeID, 10) if infl := info.PendingSum; infl != nil { statistics.ForeachRegionStats(func(rwTy statistics.RWType, dim int, kind statistics.RegionStatKind) { - cluster.SetHotPendingInfluenceMetrics(storeLabel, rwTy.String(), statistics.DimToString(dim), infl.Loads[kind]) + setHotPendingInfluenceMetrics(storeLabel, rwTy.String(), statistics.DimToString(dim), infl.Loads[kind]) }) } } } +// setHotPendingInfluenceMetrics sets pending influence in hot scheduler. +func setHotPendingInfluenceMetrics(storeLabel, rwTy, dim string, load float64) { + HotPendingSum.WithLabelValues(storeLabel, rwTy, dim).Set(load) +} + func (h *baseHotScheduler) randomRWType() statistics.RWType { return h.types[h.r.Int()%len(h.types)] } @@ -195,6 +219,10 @@ var ( // If the distribution of a dimension is below the corresponding stddev threshold, then scheduling will no longer be based on this dimension, // as it implies that this dimension is sufficiently uniform. stddevThreshold = 0.1 + + splitHotReadBuckets = "split-hot-read-region" + splitHotWriteBuckets = "split-hot-write-region" + splitProgressiveRank = int64(-5) ) type hotScheduler struct { @@ -206,7 +234,7 @@ type hotScheduler struct { searchRevertRegions [resourceTypeLen]bool // Whether to search revert regions. } -func newHotScheduler(opController *schedule.OperatorController, conf *hotRegionSchedulerConfig) *hotScheduler { +func newHotScheduler(opController *operator.Controller, conf *hotRegionSchedulerConfig) *hotScheduler { base := newBaseHotScheduler(opController) ret := &hotScheduler{ name: HotRegionName, @@ -243,21 +271,21 @@ func (h *hotScheduler) GetNextInterval(interval time.Duration) time.Duration { return intervalGrow(h.GetMinInterval(), maxHotScheduleInterval, exponentialGrowth) } -func (h *hotScheduler) IsScheduleAllowed(cluster schedule.Cluster) bool { - allowed := h.OpController.OperatorCount(operator.OpHotRegion) < cluster.GetOpts().GetHotRegionScheduleLimit() +func (h *hotScheduler) IsScheduleAllowed(cluster sche.SchedulerCluster) bool { + allowed := h.OpController.OperatorCount(operator.OpHotRegion) < cluster.GetSchedulerConfig().GetHotRegionScheduleLimit() if !allowed { operator.OperatorLimitCounter.WithLabelValues(h.GetType(), operator.OpHotRegion.String()).Inc() } return allowed } -func (h *hotScheduler) Schedule(cluster schedule.Cluster, dryRun bool) ([]*operator.Operator, []plan.Plan) { +func (h *hotScheduler) Schedule(cluster sche.SchedulerCluster, dryRun bool) ([]*operator.Operator, []plan.Plan) { hotSchedulerCounter.Inc() rw := h.randomRWType() return h.dispatch(rw, cluster), nil } -func (h *hotScheduler) dispatch(typ statistics.RWType, cluster schedule.Cluster) []*operator.Operator { +func (h *hotScheduler) dispatch(typ statistics.RWType, cluster sche.SchedulerCluster) []*operator.Operator { h.Lock() defer h.Unlock() h.prepareForBalance(typ, cluster) @@ -265,7 +293,6 @@ func (h *hotScheduler) dispatch(typ statistics.RWType, cluster schedule.Cluster) if h.conf.IsForbidRWType(typ) { return nil } - switch typ { case statistics.Read: return h.balanceHotReadRegions(cluster) @@ -275,7 +302,7 @@ func (h *hotScheduler) dispatch(typ statistics.RWType, cluster schedule.Cluster) return nil } -func (h *hotScheduler) tryAddPendingInfluence(op *operator.Operator, srcStore, dstStore uint64, infl statistics.Influence, maxZombieDur time.Duration) bool { +func (h *hotScheduler) tryAddPendingInfluence(op *operator.Operator, srcStore []uint64, dstStore uint64, infl statistics.Influence, maxZombieDur time.Duration) bool { regionID := op.RegionID() _, ok := h.regionPendings[regionID] if ok { @@ -292,7 +319,7 @@ func (h *hotScheduler) tryAddPendingInfluence(op *operator.Operator, srcStore, d return true } -func (h *hotScheduler) balanceHotReadRegions(cluster schedule.Cluster) []*operator.Operator { +func (h *hotScheduler) balanceHotReadRegions(cluster sche.SchedulerCluster) []*operator.Operator { leaderSolver := newBalanceSolver(h, cluster, statistics.Read, transferLeader) leaderOps := leaderSolver.solve() peerSolver := newBalanceSolver(h, cluster, statistics.Read, movePeer) @@ -335,7 +362,7 @@ func (h *hotScheduler) balanceHotReadRegions(cluster schedule.Cluster) []*operat return nil } -func (h *hotScheduler) balanceHotWriteRegions(cluster schedule.Cluster) []*operator.Operator { +func (h *hotScheduler) balanceHotWriteRegions(cluster sche.SchedulerCluster) []*operator.Operator { // prefer to balance by peer s := h.r.Intn(100) switch { @@ -424,12 +451,14 @@ func isAvailableV1(s *solution) bool { } type balanceSolver struct { - schedule.Cluster - sche *hotScheduler - stLoadDetail map[uint64]*statistics.StoreLoadDetail - rwTy statistics.RWType - opTy opType - resourceTy resourceType + sche.SchedulerCluster + sche *hotScheduler + stLoadDetail map[uint64]*statistics.StoreLoadDetail + filteredHotPeers map[uint64][]*statistics.HotPeerStat // storeID -> hotPeers(filtered) + nthHotPeer map[uint64][]*statistics.HotPeerStat // storeID -> [dimLen]hotPeers + rwTy statistics.RWType + opTy opType + resourceTy resourceType cur *solution @@ -462,11 +491,24 @@ type balanceSolver struct { betterThan func(*solution) bool rankToDimString func() string checkByPriorityAndTolerance func(loads []float64, f func(int) bool) bool + checkHistoryLoadsByPriority func(loads [][]float64, f func(int) bool) bool } func (bs *balanceSolver) init() { - // Init store load detail according to the type. + // Load the configuration items of the scheduler. bs.resourceTy = toResourceType(bs.rwTy, bs.opTy) + bs.maxPeerNum = bs.sche.conf.GetMaxPeerNumber() + bs.minHotDegree = bs.GetSchedulerConfig().GetHotRegionCacheHitsThreshold() + bs.firstPriority, bs.secondPriority = prioritiesToDim(bs.getPriorities()) + bs.greatDecRatio, bs.minorDecRatio = bs.sche.conf.GetGreatDecRatio(), bs.sche.conf.GetMinorDecRatio() + switch bs.sche.conf.GetRankFormulaVersion() { + case "v1": + bs.initRankV1() + default: + bs.initRankV2() + } + + // Init store load detail according to the type. bs.stLoadDetail = bs.sche.stLoadInfos[bs.resourceTy] bs.maxSrc = &statistics.StoreLoad{Loads: make([]float64, statistics.DimLen)} @@ -479,10 +521,14 @@ func (bs *balanceSolver) init() { } maxCur := &statistics.StoreLoad{Loads: make([]float64, statistics.DimLen)} + bs.filteredHotPeers = make(map[uint64][]*statistics.HotPeerStat) + bs.nthHotPeer = make(map[uint64][]*statistics.HotPeerStat) for _, detail := range bs.stLoadDetail { bs.maxSrc = statistics.MaxLoad(bs.maxSrc, detail.LoadPred.Min()) bs.minDst = statistics.MinLoad(bs.minDst, detail.LoadPred.Max()) maxCur = statistics.MaxLoad(maxCur, &detail.LoadPred.Current) + bs.nthHotPeer[detail.GetID()] = make([]*statistics.HotPeerStat, statistics.DimLen) + bs.filteredHotPeers[detail.GetID()] = bs.filterHotPeers(detail) } rankStepRatios := []float64{ @@ -497,18 +543,6 @@ func (bs *balanceSolver) init() { Loads: stepLoads, Count: maxCur.Count * bs.sche.conf.GetCountRankStepRatio(), } - - bs.firstPriority, bs.secondPriority = prioritiesToDim(bs.getPriorities()) - bs.greatDecRatio, bs.minorDecRatio = bs.sche.conf.GetGreatDecRatio(), bs.sche.conf.GetMinorDecRatio() - bs.maxPeerNum = bs.sche.conf.GetMaxPeerNumber() - bs.minHotDegree = bs.GetOpts().GetHotRegionCacheHitsThreshold() - - switch bs.sche.conf.GetRankFormulaVersion() { - case "v1": - bs.initRankV1() - default: - bs.initRankV2() - } } func (bs *balanceSolver) initRankV1() { @@ -526,10 +560,13 @@ func (bs *balanceSolver) pickCheckPolicyV1() { switch { case bs.resourceTy == writeLeader: bs.checkByPriorityAndTolerance = bs.checkByPriorityAndToleranceFirstOnly + bs.checkHistoryLoadsByPriority = bs.checkHistoryLoadsByPriorityAndToleranceFirstOnly case bs.sche.conf.IsStrictPickingStoreEnabled(): bs.checkByPriorityAndTolerance = bs.checkByPriorityAndToleranceAllOf + bs.checkHistoryLoadsByPriority = bs.checkHistoryLoadsByPriorityAndToleranceAllOf default: bs.checkByPriorityAndTolerance = bs.checkByPriorityAndToleranceFirstOnly + bs.checkHistoryLoadsByPriority = bs.checkHistoryLoadsByPriorityAndToleranceFirstOnly } } @@ -538,7 +575,7 @@ func (bs *balanceSolver) isSelectedDim(dim int) bool { } func (bs *balanceSolver) getPriorities() []string { - querySupport := bs.sche.conf.checkQuerySupport(bs.Cluster) + querySupport := bs.sche.conf.checkQuerySupport(bs.SchedulerCluster) // For read, transfer-leader and move-peer have the same priority config // For write, they are different switch bs.resourceTy { @@ -553,19 +590,19 @@ func (bs *balanceSolver) getPriorities() []string { return []string{} } -func newBalanceSolver(sche *hotScheduler, cluster schedule.Cluster, rwTy statistics.RWType, opTy opType) *balanceSolver { +func newBalanceSolver(sche *hotScheduler, cluster sche.SchedulerCluster, rwTy statistics.RWType, opTy opType) *balanceSolver { bs := &balanceSolver{ - Cluster: cluster, - sche: sche, - rwTy: rwTy, - opTy: opTy, + SchedulerCluster: cluster, + sche: sche, + rwTy: rwTy, + opTy: opTy, } bs.init() return bs } func (bs *balanceSolver) isValid() bool { - if bs.Cluster == nil || bs.sche == nil || bs.stLoadDetail == nil { + if bs.SchedulerCluster == nil || bs.sche == nil || bs.stLoadDetail == nil { return false } return true @@ -575,7 +612,7 @@ func (bs *balanceSolver) filterUniformStoreV1() (string, bool) { if !bs.enableExpectation() { return "", false } - // Because region is available for src and dst, so stddev is the same for both, only need to calcurate one. + // Because region is available for src and dst, so stddev is the same for both, only need to calculate one. isUniformFirstPriority, isUniformSecondPriority := bs.isUniformFirstPriority(bs.cur.srcStore), bs.isUniformSecondPriority(bs.cur.srcStore) if isUniformFirstPriority && isUniformSecondPriority { // If both dims are enough uniform, any schedule is unnecessary. @@ -598,7 +635,6 @@ func (bs *balanceSolver) solve() []*operator.Operator { if !bs.isValid() { return nil } - bs.cur = &solution{} tryUpdateBestSolution := func() { if label, ok := bs.filterUniformStore(); ok { @@ -625,18 +661,31 @@ func (bs *balanceSolver) solve() []*operator.Operator { return region.GetStorePeer(srcStoreID) == nil } } - + snapshotFilter := filter.NewSnapshotSendFilter(bs.GetStores(), constant.Medium) + splitThresholds := bs.sche.conf.getSplitThresholds() for _, srcStore := range bs.filterSrcStores() { bs.cur.srcStore = srcStore srcStoreID := srcStore.GetID() - for _, mainPeerStat := range bs.filterHotPeers(srcStore) { + for _, mainPeerStat := range bs.filteredHotPeers[srcStoreID] { if bs.cur.region = bs.getRegion(mainPeerStat, srcStoreID); bs.cur.region == nil { continue - } else if bs.opTy == movePeer && bs.cur.region.GetApproximateSize() > bs.GetOpts().GetMaxMovableHotPeerSize() { - hotSchedulerNeedSplitBeforeScheduleCounter.Inc() - continue + } else if bs.opTy == movePeer { + if !snapshotFilter.Select(bs.cur.region).IsOK() { + hotSchedulerSnapshotSenderLimitCounter.Inc() + continue + } } bs.cur.mainPeerStat = mainPeerStat + if bs.GetStoreConfig().IsEnableRegionBucket() && bs.tooHotNeedSplit(srcStore, mainPeerStat, splitThresholds) { + hotSchedulerRegionTooHotNeedSplitCounter.Inc() + ops := bs.createSplitOperator([]*core.RegionInfo{bs.cur.region}, byLoad) + if len(ops) > 0 { + bs.ops = ops + bs.cur.calcPeersRate(bs.firstPriority, bs.secondPriority) + bs.best = bs.cur + return ops + } + } for _, dstStore := range bs.filterDstStores() { bs.cur.dstStore = dstStore @@ -645,7 +694,7 @@ func (bs *balanceSolver) solve() []*operator.Operator { if bs.needSearchRevertRegions() { hotSchedulerSearchRevertRegionsCounter.Inc() dstStoreID := dstStore.GetID() - for _, revertPeerStat := range bs.filterHotPeers(bs.cur.dstStore) { + for _, revertPeerStat := range bs.filteredHotPeers[dstStoreID] { revertRegion := bs.getRegion(revertPeerStat, dstStoreID) if revertRegion == nil || revertRegion.GetID() == bs.cur.region.GetID() || !allowRevertRegion(revertRegion, srcStoreID) { @@ -696,7 +745,8 @@ func (bs *balanceSolver) tryAddPendingInfluence() bool { if bs.best == nil || len(bs.ops) == 0 { return false } - if bs.best.srcStore.IsTiFlash() != bs.best.dstStore.IsTiFlash() { + isSplit := bs.ops[0].Kind() == operator.OpSplit + if !isSplit && bs.best.srcStore.IsTiFlash() != bs.best.dstStore.IsTiFlash() { hotSchedulerNotSameEngineCounter.Inc() return false } @@ -704,16 +754,29 @@ func (bs *balanceSolver) tryAddPendingInfluence() bool { // TODO: Process operators atomically. // main peer - srcStoreID := bs.best.srcStore.GetID() - dstStoreID := bs.best.dstStore.GetID() + + srcStoreIDs := make([]uint64, 0) + dstStoreID := uint64(0) + if isSplit { + region := bs.GetRegion(bs.ops[0].RegionID()) + for id := range region.GetStoreIDs() { + srcStoreIDs = append(srcStoreIDs, id) + } + } else { + srcStoreIDs = append(srcStoreIDs, bs.best.srcStore.GetID()) + dstStoreID = bs.best.dstStore.GetID() + } infl := bs.collectPendingInfluence(bs.best.mainPeerStat) - if !bs.sche.tryAddPendingInfluence(bs.ops[0], srcStoreID, dstStoreID, infl, maxZombieDur) { + if !bs.sche.tryAddPendingInfluence(bs.ops[0], srcStoreIDs, dstStoreID, infl, maxZombieDur) { return false } + if isSplit { + return true + } // revert peers - if bs.best.revertPeerStat != nil { + if bs.best.revertPeerStat != nil && len(bs.ops) > 1 { infl := bs.collectPendingInfluence(bs.best.revertPeerStat) - if !bs.sche.tryAddPendingInfluence(bs.ops[1], dstStoreID, srcStoreID, infl, maxZombieDur) { + if !bs.sche.tryAddPendingInfluence(bs.ops[1], srcStoreIDs, dstStoreID, infl, maxZombieDur) { return false } } @@ -775,12 +838,17 @@ func (bs *balanceSolver) filterSrcStores() map[uint64]*statistics.StoreLoadDetai continue } - if bs.checkSrcByPriorityAndTolerance(detail.LoadPred.Min(), &detail.LoadPred.Expect, srcToleranceRatio) { - ret[id] = detail - hotSchedulerResultCounter.WithLabelValues("src-store-succ", strconv.FormatUint(id, 10)).Inc() - } else { - hotSchedulerResultCounter.WithLabelValues("src-store-failed", strconv.FormatUint(id, 10)).Inc() + if !bs.checkSrcByPriorityAndTolerance(detail.LoadPred.Min(), &detail.LoadPred.Expect, srcToleranceRatio) { + hotSchedulerResultCounter.WithLabelValues("src-store-failed-"+bs.resourceTy.String(), strconv.FormatUint(id, 10)).Inc() + continue } + if !bs.checkSrcHistoryLoadsByPriorityAndTolerance(&detail.LoadPred.Current, &detail.LoadPred.Expect, srcToleranceRatio) { + hotSchedulerResultCounter.WithLabelValues("src-store-history-loads-failed-"+bs.resourceTy.String(), strconv.FormatUint(id, 10)).Inc() + continue + } + + ret[id] = detail + hotSchedulerResultCounter.WithLabelValues("src-store-succ-"+bs.resourceTy.String(), strconv.FormatUint(id, 10)).Inc() } return ret } @@ -791,9 +859,22 @@ func (bs *balanceSolver) checkSrcByPriorityAndTolerance(minLoad, expectLoad *sta }) } +func (bs *balanceSolver) checkSrcHistoryLoadsByPriorityAndTolerance(current, expectLoad *statistics.StoreLoad, toleranceRatio float64) bool { + if len(current.HistoryLoads) == 0 { + return true + } + return bs.checkHistoryLoadsByPriority(current.HistoryLoads, func(i int) bool { + return slice.AllOf(current.HistoryLoads[i], func(j int) bool { + return current.HistoryLoads[i][j] > toleranceRatio*expectLoad.HistoryLoads[i][j] + }) + }) +} + // filterHotPeers filtered hot peers from statistics.HotPeerStat and deleted the peer if its region is in pending status. // The returned hotPeer count in controlled by `max-peer-number`. -func (bs *balanceSolver) filterHotPeers(storeLoad *statistics.StoreLoadDetail) (ret []*statistics.HotPeerStat) { +func (bs *balanceSolver) filterHotPeers(storeLoad *statistics.StoreLoadDetail) []*statistics.HotPeerStat { + hotPeers := storeLoad.HotPeers + ret := make([]*statistics.HotPeerStat, 0, len(hotPeers)) appendItem := func(item *statistics.HotPeerStat) { if _, ok := bs.sche.regionPendings[item.ID()]; !ok && !item.IsNeedCoolDownTransferLeader(bs.minHotDegree, bs.rwTy) { // no in pending operator and no need cool down after transfer leader @@ -801,36 +882,42 @@ func (bs *balanceSolver) filterHotPeers(storeLoad *statistics.StoreLoadDetail) ( } } - src := storeLoad.HotPeers - // At most MaxPeerNum peers, to prevent balanceSolver.solve() too slow. - if len(src) <= bs.maxPeerNum { - ret = make([]*statistics.HotPeerStat, 0, len(src)) - for _, peer := range src { - appendItem(peer) - } - } else { - union := bs.sortHotPeers(src) + var firstSort, secondSort []*statistics.HotPeerStat + if len(hotPeers) >= topnPosition || len(hotPeers) > bs.maxPeerNum { + firstSort = make([]*statistics.HotPeerStat, len(hotPeers)) + copy(firstSort, hotPeers) + sort.Slice(firstSort, func(i, j int) bool { + return firstSort[i].GetLoad(bs.firstPriority) > firstSort[j].GetLoad(bs.firstPriority) + }) + secondSort = make([]*statistics.HotPeerStat, len(hotPeers)) + copy(secondSort, hotPeers) + sort.Slice(secondSort, func(i, j int) bool { + return secondSort[i].GetLoad(bs.secondPriority) > secondSort[j].GetLoad(bs.secondPriority) + }) + } + if len(hotPeers) >= topnPosition { + storeID := storeLoad.GetID() + bs.nthHotPeer[storeID][bs.firstPriority] = firstSort[topnPosition-1] + bs.nthHotPeer[storeID][bs.secondPriority] = secondSort[topnPosition-1] + } + if len(hotPeers) > bs.maxPeerNum { + union := bs.sortHotPeers(firstSort, secondSort) ret = make([]*statistics.HotPeerStat, 0, len(union)) for peer := range union { appendItem(peer) } + return ret } - return + for _, peer := range hotPeers { + appendItem(peer) + } + return ret } -func (bs *balanceSolver) sortHotPeers(ret []*statistics.HotPeerStat) map[*statistics.HotPeerStat]struct{} { - firstSort := make([]*statistics.HotPeerStat, len(ret)) - copy(firstSort, ret) - sort.Slice(firstSort, func(i, j int) bool { - return firstSort[i].GetLoad(bs.firstPriority) > firstSort[j].GetLoad(bs.firstPriority) - }) - secondSort := make([]*statistics.HotPeerStat, len(ret)) - copy(secondSort, ret) - sort.Slice(secondSort, func(i, j int) bool { - return secondSort[i].GetLoad(bs.secondPriority) > secondSort[j].GetLoad(bs.secondPriority) - }) +func (bs *balanceSolver) sortHotPeers(firstSort, secondSort []*statistics.HotPeerStat) map[*statistics.HotPeerStat]struct{} { union := make(map[*statistics.HotPeerStat]struct{}, bs.maxPeerNum) + // At most MaxPeerNum peers, to prevent balanceSolver.solve() too slow. for len(union) < bs.maxPeerNum { for len(firstSort) > 0 { peer := firstSort[0] @@ -864,7 +951,7 @@ func (bs *balanceSolver) isRegionAvailable(region *core.RegionInfo) bool { return false } - if !filter.IsRegionReplicated(bs.Cluster, region) { + if !filter.IsRegionReplicated(bs.SchedulerCluster, region) { log.Debug("region has abnormal replica count", zap.String("scheduler", bs.sche.GetName()), zap.Uint64("region-id", region.GetID())) hotSchedulerAbnormalReplicaCounter.Inc() return false @@ -915,10 +1002,10 @@ func (bs *balanceSolver) filterDstStores() map[uint64]*statistics.StoreLoadDetai return nil } filters = []filter.Filter{ - &filter.StoreStateFilter{ActionScope: bs.sche.GetName(), MoveRegion: true}, + &filter.StoreStateFilter{ActionScope: bs.sche.GetName(), MoveRegion: true, OperatorLevel: constant.High}, filter.NewExcludedFilter(bs.sche.GetName(), bs.cur.region.GetStoreIDs(), bs.cur.region.GetStoreIDs()), filter.NewSpecialUseFilter(bs.sche.GetName(), filter.SpecialUseHotRegion), - filter.NewPlacementSafeguard(bs.sche.GetName(), bs.GetOpts(), bs.GetBasicCluster(), bs.GetRuleManager(), bs.cur.region, srcStore, nil), + filter.NewPlacementSafeguard(bs.sche.GetName(), bs.GetSchedulerConfig(), bs.GetBasicCluster(), bs.GetRuleManager(), bs.cur.region, srcStore, nil), } for _, detail := range bs.stLoadDetail { candidates = append(candidates, detail) @@ -929,13 +1016,13 @@ func (bs *balanceSolver) filterDstStores() map[uint64]*statistics.StoreLoadDetai return nil } filters = []filter.Filter{ - &filter.StoreStateFilter{ActionScope: bs.sche.GetName(), TransferLeader: true}, + &filter.StoreStateFilter{ActionScope: bs.sche.GetName(), TransferLeader: true, OperatorLevel: constant.High}, filter.NewSpecialUseFilter(bs.sche.GetName(), filter.SpecialUseHotRegion), } if bs.rwTy == statistics.Read { peers := bs.cur.region.GetPeers() - moveLeaderFilters := []filter.Filter{&filter.StoreStateFilter{ActionScope: bs.sche.GetName(), MoveRegion: true}} - if leaderFilter := filter.NewPlacementLeaderSafeguard(bs.sche.GetName(), bs.GetOpts(), bs.GetBasicCluster(), bs.GetRuleManager(), bs.cur.region, srcStore, true /*allowMoveLeader*/); leaderFilter != nil { + moveLeaderFilters := []filter.Filter{&filter.StoreStateFilter{ActionScope: bs.sche.GetName(), MoveRegion: true, OperatorLevel: constant.High}} + if leaderFilter := filter.NewPlacementLeaderSafeguard(bs.sche.GetName(), bs.GetSchedulerConfig(), bs.GetBasicCluster(), bs.GetRuleManager(), bs.cur.region, srcStore, true /*allowMoveLeader*/); leaderFilter != nil { filters = append(filters, leaderFilter) } for storeID, detail := range bs.stLoadDetail { @@ -950,12 +1037,12 @@ func (bs *balanceSolver) filterDstStores() map[uint64]*statistics.StoreLoadDetai continue } // move leader - if filter.Target(bs.GetOpts(), detail.StoreInfo, moveLeaderFilters) { + if filter.Target(bs.GetSchedulerConfig(), detail.StoreInfo, moveLeaderFilters) { candidates = append(candidates, detail) } } } else { - if leaderFilter := filter.NewPlacementLeaderSafeguard(bs.sche.GetName(), bs.GetOpts(), bs.GetBasicCluster(), bs.GetRuleManager(), bs.cur.region, srcStore, false /*allowMoveLeader*/); leaderFilter != nil { + if leaderFilter := filter.NewPlacementLeaderSafeguard(bs.sche.GetName(), bs.GetSchedulerConfig(), bs.GetBasicCluster(), bs.GetRuleManager(), bs.cur.region, srcStore, false /*allowMoveLeader*/); leaderFilter != nil { filters = append(filters, leaderFilter) } for _, peer := range bs.cur.region.GetFollowers() { @@ -987,14 +1074,19 @@ func (bs *balanceSolver) pickDstStores(filters []filter.Filter, candidates []*st } dstToleranceRatio += tiflashToleranceRatioCorrection } - if filter.Target(bs.GetOpts(), store, filters) { + if filter.Target(bs.GetSchedulerConfig(), store, filters) { id := store.GetID() - if bs.checkDstByPriorityAndTolerance(detail.LoadPred.Max(), &detail.LoadPred.Expect, dstToleranceRatio) { - ret[id] = detail - hotSchedulerResultCounter.WithLabelValues("dst-store-succ", strconv.FormatUint(id, 10)).Inc() - } else { - hotSchedulerResultCounter.WithLabelValues("dst-store-failed", strconv.FormatUint(id, 10)).Inc() + if !bs.checkDstByPriorityAndTolerance(detail.LoadPred.Max(), &detail.LoadPred.Expect, dstToleranceRatio) { + hotSchedulerResultCounter.WithLabelValues("dst-store-failed-"+bs.resourceTy.String(), strconv.FormatUint(id, 10)).Inc() + continue } + if !bs.checkDstHistoryLoadsByPriorityAndTolerance(&detail.LoadPred.Current, &detail.LoadPred.Expect, dstToleranceRatio) { + hotSchedulerResultCounter.WithLabelValues("dst-store-history-loads-failed-"+bs.resourceTy.String(), strconv.FormatUint(id, 10)).Inc() + continue + } + + hotSchedulerResultCounter.WithLabelValues("dst-store-succ-"+bs.resourceTy.String(), strconv.FormatUint(id, 10)).Inc() + ret[id] = detail } } return ret @@ -1006,6 +1098,17 @@ func (bs *balanceSolver) checkDstByPriorityAndTolerance(maxLoad, expect *statist }) } +func (bs *balanceSolver) checkDstHistoryLoadsByPriorityAndTolerance(current, expect *statistics.StoreLoad, toleranceRatio float64) bool { + if len(current.HistoryLoads) == 0 { + return true + } + return bs.checkHistoryLoadsByPriority(current.HistoryLoads, func(i int) bool { + return slice.AllOf(current.HistoryLoads[i], func(j int) bool { + return current.HistoryLoads[i][j]*toleranceRatio < expect.HistoryLoads[i][j] + }) + }) +} + func (bs *balanceSolver) checkByPriorityAndToleranceAllOf(loads []float64, f func(int) bool) bool { return slice.AllOf(loads, func(i int) bool { if bs.isSelectedDim(i) { @@ -1015,6 +1118,15 @@ func (bs *balanceSolver) checkByPriorityAndToleranceAllOf(loads []float64, f fun }) } +func (bs *balanceSolver) checkHistoryLoadsByPriorityAndToleranceAllOf(loads [][]float64, f func(int) bool) bool { + return slice.AllOf(loads, func(i int) bool { + if bs.isSelectedDim(i) { + return f(i) + } + return true + }) +} + func (bs *balanceSolver) checkByPriorityAndToleranceAnyOf(loads []float64, f func(int) bool) bool { return slice.AnyOf(loads, func(i int) bool { if bs.isSelectedDim(i) { @@ -1024,10 +1136,23 @@ func (bs *balanceSolver) checkByPriorityAndToleranceAnyOf(loads []float64, f fun }) } +func (bs *balanceSolver) checkHistoryByPriorityAndToleranceAnyOf(loads [][]float64, f func(int) bool) bool { + return slice.AnyOf(loads, func(i int) bool { + if bs.isSelectedDim(i) { + return f(i) + } + return false + }) +} + func (bs *balanceSolver) checkByPriorityAndToleranceFirstOnly(loads []float64, f func(int) bool) bool { return f(bs.firstPriority) } +func (bs *balanceSolver) checkHistoryLoadsByPriorityAndToleranceFirstOnly(_ [][]float64, f func(int) bool) bool { + return f(bs.firstPriority) +} + func (bs *balanceSolver) enableExpectation() bool { return bs.sche.conf.GetDstToleranceRatio() > 0 && bs.sche.conf.GetSrcToleranceRatio() > 0 } @@ -1156,7 +1281,7 @@ func (bs *balanceSolver) getMinRate(dim int) float64 { // betterThan checks if `bs.cur` is a better solution than `old`. func (bs *balanceSolver) betterThanV1(old *solution) bool { - if old == nil { + if old == nil || bs.cur.progressiveRank <= splitProgressiveRank { return true } if bs.cur.progressiveRank != old.progressiveRank { @@ -1335,6 +1460,22 @@ func (bs *balanceSolver) buildOperators() (ops []*operator.Operator) { return nil } + splitRegions := make([]*core.RegionInfo, 0) + if bs.opTy == movePeer { + for _, region := range []*core.RegionInfo{bs.cur.region, bs.cur.revertRegion} { + if region == nil { + continue + } + if region.GetApproximateSize() > bs.GetSchedulerConfig().GetMaxMovableHotPeerSize() { + hotSchedulerNeedSplitBeforeScheduleCounter.Inc() + splitRegions = append(splitRegions, region) + } + } + } + if len(splitRegions) > 0 { + return bs.createSplitOperator(splitRegions, bySize) + } + srcStoreID := bs.cur.srcStore.GetID() dstStoreID := bs.cur.dstStore.GetID() sourceLabel := strconv.FormatUint(srcStoreID, 10) @@ -1371,6 +1512,155 @@ func (bs *balanceSolver) buildOperators() (ops []*operator.Operator) { return } +// bucketFirstStat returns the first priority statistics of the bucket. +// if the first priority is query rate, it will return the second priority . +func (bs *balanceSolver) bucketFirstStat() statistics.RegionStatKind { + base := statistics.RegionReadBytes + if bs.rwTy == statistics.Write { + base = statistics.RegionWriteBytes + } + offset := bs.firstPriority + // todo: remove it if bucket's qps has been supported. + if bs.firstPriority == statistics.QueryDim { + offset = bs.secondPriority + } + return base + statistics.RegionStatKind(offset) +} + +func (bs *balanceSolver) splitBucketsOperator(region *core.RegionInfo, keys [][]byte) *operator.Operator { + splitKeys := make([][]byte, 0, len(keys)) + for _, key := range keys { + // make sure that this split key is in the region + if keyutil.Between(region.GetStartKey(), region.GetEndKey(), key) { + splitKeys = append(splitKeys, key) + } + } + if len(splitKeys) == 0 { + hotSchedulerNotFoundSplitKeysCounter.Inc() + return nil + } + desc := splitHotReadBuckets + if bs.rwTy == statistics.Write { + desc = splitHotWriteBuckets + } + + op, err := operator.CreateSplitRegionOperator(desc, region, operator.OpSplit, pdpb.CheckPolicy_USEKEY, splitKeys) + if err != nil { + log.Error("fail to create split operator", + zap.Stringer("resource-type", bs.resourceTy), + errs.ZapError(err)) + return nil + } + hotSchedulerSplitSuccessCounter.Inc() + return op +} + +func (bs *balanceSolver) splitBucketsByLoad(region *core.RegionInfo, bucketStats []*buckets.BucketStat) *operator.Operator { + // bucket key range maybe not match the region key range, so we should filter the invalid buckets. + // filter some buckets key range not match the region start key and end key. + stats := make([]*buckets.BucketStat, 0, len(bucketStats)) + startKey, endKey := region.GetStartKey(), region.GetEndKey() + for _, stat := range bucketStats { + if keyutil.Between(startKey, endKey, stat.StartKey) || keyutil.Between(startKey, endKey, stat.EndKey) { + stats = append(stats, stat) + } + } + if len(stats) == 0 { + hotSchedulerHotBucketNotValidCounter.Inc() + return nil + } + + // if this region has only one buckets, we can't split it into two hot region, so skip it. + if len(stats) == 1 { + hotSchedulerOnlyOneBucketsHotCounter.Inc() + return nil + } + totalLoads := uint64(0) + dim := bs.bucketFirstStat() + for _, stat := range stats { + totalLoads += stat.Loads[dim] + } + + // find the half point of the total loads. + acc, splitIdx := uint64(0), 0 + for ; acc < totalLoads/2 && splitIdx < len(stats); splitIdx++ { + acc += stats[splitIdx].Loads[dim] + } + if splitIdx <= 0 { + hotSchedulerRegionBucketsSingleHotSpotCounter.Inc() + return nil + } + splitKey := stats[splitIdx-1].EndKey + // if the split key is not in the region, we should use the start key of the bucket. + if !keyutil.Between(region.GetStartKey(), region.GetEndKey(), splitKey) { + splitKey = stats[splitIdx-1].StartKey + } + op := bs.splitBucketsOperator(region, [][]byte{splitKey}) + if op != nil { + op.AdditionalInfos["accLoads"] = strconv.FormatUint(acc-stats[splitIdx-1].Loads[dim], 10) + op.AdditionalInfos["totalLoads"] = strconv.FormatUint(totalLoads, 10) + } + return op +} + +// splitBucketBySize splits the region order by bucket count if the region is too big. +func (bs *balanceSolver) splitBucketBySize(region *core.RegionInfo) *operator.Operator { + splitKeys := make([][]byte, 0) + for _, key := range region.GetBuckets().GetKeys() { + if keyutil.Between(region.GetStartKey(), region.GetEndKey(), key) { + splitKeys = append(splitKeys, key) + } + } + if len(splitKeys) == 0 { + return nil + } + splitKey := splitKeys[len(splitKeys)/2] + return bs.splitBucketsOperator(region, [][]byte{splitKey}) +} + +// createSplitOperator creates split operators for the given regions. +func (bs *balanceSolver) createSplitOperator(regions []*core.RegionInfo, strategy splitStrategy) []*operator.Operator { + if len(regions) == 0 { + return nil + } + ids := make([]uint64, len(regions)) + for i, region := range regions { + ids[i] = region.GetID() + } + operators := make([]*operator.Operator, 0) + var hotBuckets map[uint64][]*buckets.BucketStat + + createFunc := func(region *core.RegionInfo) { + switch strategy { + case bySize: + if op := bs.splitBucketBySize(region); op != nil { + operators = append(operators, op) + } + case byLoad: + if hotBuckets == nil { + hotBuckets = bs.SchedulerCluster.BucketsStats(bs.minHotDegree, ids...) + } + stats, ok := hotBuckets[region.GetID()] + if !ok { + hotSchedulerRegionBucketsNotHotCounter.Inc() + return + } + if op := bs.splitBucketsByLoad(region, stats); op != nil { + operators = append(operators, op) + } + } + } + + for _, region := range regions { + createFunc(region) + } + // the split bucket's priority is highest + if len(operators) > 0 { + bs.cur.progressiveRank = splitProgressiveRank + } + return operators +} + func (bs *balanceSolver) createReadOperator(region *core.RegionInfo, srcStoreID, dstStoreID uint64) (op *operator.Operator, typ string, err error) { if region.GetStorePeer(dstStoreID) != nil { typ = "transfer-leader" @@ -1544,6 +1834,22 @@ const ( resourceTypeLen ) +// String implements fmt.Stringer interface. +func (ty resourceType) String() string { + switch ty { + case writePeer: + return "write-peer" + case writeLeader: + return "write-leader" + case readPeer: + return "read-peer" + case readLeader: + return "read-leader" + default: + return "" + } +} + func toResourceType(rwTy statistics.RWType, opTy opType) resourceType { switch rwTy { case statistics.Write: @@ -1612,3 +1918,17 @@ func dimToString(dim int) string { func prioritiesToDim(priorities []string) (firstPriority int, secondPriority int) { return stringToDim(priorities[0]), stringToDim(priorities[1]) } + +// tooHotNeedSplit returns true if any dim of the hot region is greater than the store threshold. +func (bs *balanceSolver) tooHotNeedSplit(store *statistics.StoreLoadDetail, region *statistics.HotPeerStat, splitThresholds float64) bool { + return bs.checkByPriorityAndTolerance(store.LoadPred.Current.Loads, func(i int) bool { + return region.Loads[i] > store.LoadPred.Current.Loads[i]*splitThresholds + }) +} + +type splitStrategy int + +const ( + byLoad splitStrategy = iota + bySize +) diff --git a/pkg/schedule/schedulers/hot_region_config.go b/pkg/schedule/schedulers/hot_region_config.go index b74af17404d..61a9942e339 100644 --- a/pkg/schedule/schedulers/hot_region_config.go +++ b/pkg/schedule/schedulers/hot_region_config.go @@ -24,7 +24,7 @@ import ( "github.com/gorilla/mux" "github.com/pingcap/log" "github.com/tikv/pd/pkg/errs" - "github.com/tikv/pd/pkg/schedule" + sche "github.com/tikv/pd/pkg/schedule/core" "github.com/tikv/pd/pkg/slice" "github.com/tikv/pd/pkg/statistics" "github.com/tikv/pd/pkg/storage/endpoint" @@ -75,6 +75,7 @@ func initHotRegionScheduleConfig() *hotRegionSchedulerConfig { EnableForTiFlash: true, RankFormulaVersion: "v2", ForbidRWType: "none", + SplitThresholds: 0.2, } cfg.applyPrioritiesConfig(defaultPrioritiesConfig) return cfg @@ -102,6 +103,7 @@ func (conf *hotRegionSchedulerConfig) getValidConf() *hotRegionSchedulerConfig { EnableForTiFlash: conf.EnableForTiFlash, RankFormulaVersion: conf.getRankFormulaVersionLocked(), ForbidRWType: conf.getForbidRWTypeLocked(), + SplitThresholds: conf.SplitThresholds, } } @@ -143,12 +145,14 @@ type hotRegionSchedulerConfig struct { RankFormulaVersion string `json:"rank-formula-version"` // forbid read or write scheduler, only for test ForbidRWType string `json:"forbid-rw-type,omitempty"` + // SplitThresholds is the threshold to split hot region if the first priority flow of on hot region exceeds it. + SplitThresholds float64 `json:"split-thresholds"` } func (conf *hotRegionSchedulerConfig) EncodeConfig() ([]byte, error) { conf.RLock() defer conf.RUnlock() - return schedule.EncodeConfig(conf) + return EncodeConfig(conf) } func (conf *hotRegionSchedulerConfig) GetStoreStatZombieDuration() time.Duration { @@ -316,6 +320,12 @@ func (conf *hotRegionSchedulerConfig) IsForbidRWType(rw statistics.RWType) bool return rw.String() == conf.ForbidRWType } +func (conf *hotRegionSchedulerConfig) getSplitThresholds() float64 { + conf.RLock() + defer conf.RUnlock() + return conf.SplitThresholds +} + func (conf *hotRegionSchedulerConfig) getForbidRWTypeLocked() string { switch conf.ForbidRWType { case statistics.Read.String(), statistics.Write.String(): @@ -377,6 +387,9 @@ func (conf *hotRegionSchedulerConfig) valid() error { conf.ForbidRWType != "none" && conf.ForbidRWType != "" { return errs.ErrSchedulerConfig.FastGenByArgs("invalid forbid-rw-type") } + if conf.SplitThresholds < 0.01 || conf.SplitThresholds > 1.0 { + return errs.ErrSchedulerConfig.FastGenByArgs("invalid split-thresholds, should be in range [0.01, 1.0]") + } return nil } @@ -426,21 +439,22 @@ func (conf *hotRegionSchedulerConfig) handleSetConfig(w http.ResponseWriter, r * } func (conf *hotRegionSchedulerConfig) persistLocked() error { - data, err := schedule.EncodeConfig(conf) + data, err := EncodeConfig(conf) if err != nil { return err } return conf.storage.SaveScheduleConfig(HotRegionName, data) } -func (conf *hotRegionSchedulerConfig) checkQuerySupport(cluster schedule.Cluster) bool { - querySupport := versioninfo.IsFeatureSupported(cluster.GetOpts().GetClusterVersion(), versioninfo.HotScheduleWithQuery) +func (conf *hotRegionSchedulerConfig) checkQuerySupport(cluster sche.SchedulerCluster) bool { + version := cluster.GetSchedulerConfig().GetClusterVersion() + querySupport := versioninfo.IsFeatureSupported(version, versioninfo.HotScheduleWithQuery) conf.Lock() defer conf.Unlock() if querySupport != conf.lastQuerySupported { log.Info("query supported changed", zap.Bool("last-query-support", conf.lastQuerySupported), - zap.String("cluster-version", cluster.GetOpts().GetClusterVersion().String()), + zap.String("cluster-version", version.String()), zap.Reflect("config", conf), zap.Reflect("valid-config", conf.getValidConf())) conf.lastQuerySupported = querySupport diff --git a/pkg/schedule/schedulers/hot_region_test.go b/pkg/schedule/schedulers/hot_region_test.go index 4f024979394..680fc8f2d30 100644 --- a/pkg/schedule/schedulers/hot_region_test.go +++ b/pkg/schedule/schedulers/hot_region_test.go @@ -16,45 +16,47 @@ package schedulers import ( "encoding/hex" + "fmt" "math" "testing" "time" "github.com/docker/go-units" "github.com/pingcap/kvproto/pkg/metapb" + "github.com/pingcap/kvproto/pkg/pdpb" "github.com/stretchr/testify/require" "github.com/tikv/pd/pkg/core" "github.com/tikv/pd/pkg/mock/mockcluster" - "github.com/tikv/pd/pkg/schedule" "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/placement" "github.com/tikv/pd/pkg/statistics" + "github.com/tikv/pd/pkg/statistics/buckets" "github.com/tikv/pd/pkg/storage" "github.com/tikv/pd/pkg/storage/endpoint" - "github.com/tikv/pd/pkg/utils/testutil" + "github.com/tikv/pd/pkg/utils/operatorutil" "github.com/tikv/pd/pkg/utils/typeutil" "github.com/tikv/pd/pkg/versioninfo" ) func init() { schedulePeerPr = 1.0 - schedule.RegisterScheduler(statistics.Write.String(), func(opController *schedule.OperatorController, storage endpoint.ConfigStorage, decoder schedule.ConfigDecoder) (schedule.Scheduler, error) { + RegisterScheduler(statistics.Write.String(), func(opController *operator.Controller, storage endpoint.ConfigStorage, decoder ConfigDecoder, removeSchedulerCb ...func(string) error) (Scheduler, error) { cfg := initHotRegionScheduleConfig() return newHotWriteScheduler(opController, cfg), nil }) - schedule.RegisterScheduler(statistics.Read.String(), func(opController *schedule.OperatorController, storage endpoint.ConfigStorage, decoder schedule.ConfigDecoder) (schedule.Scheduler, error) { + RegisterScheduler(statistics.Read.String(), func(opController *operator.Controller, storage endpoint.ConfigStorage, decoder ConfigDecoder, removeSchedulerCb ...func(string) error) (Scheduler, error) { return newHotReadScheduler(opController, initHotRegionScheduleConfig()), nil }) } -func newHotReadScheduler(opController *schedule.OperatorController, conf *hotRegionSchedulerConfig) *hotScheduler { +func newHotReadScheduler(opController *operator.Controller, conf *hotRegionSchedulerConfig) *hotScheduler { ret := newHotScheduler(opController, conf) ret.name = "" ret.types = []statistics.RWType{statistics.Read} return ret } -func newHotWriteScheduler(opController *schedule.OperatorController, conf *hotRegionSchedulerConfig) *hotScheduler { +func newHotWriteScheduler(opController *operator.Controller, conf *hotRegionSchedulerConfig) *hotScheduler { ret := newHotScheduler(opController, conf) ret.name = "" ret.types = []statistics.RWType{statistics.Write} @@ -70,7 +72,7 @@ func TestUpgrade(t *testing.T) { cancel, _, _, oc := prepareSchedulersTest() defer cancel() // new - sche, err := schedule.CreateScheduler(HotRegionType, oc, storage.NewStorageWithMemoryBackend(), schedule.ConfigSliceDecoder(HotRegionType, nil)) + sche, err := CreateScheduler(HotRegionType, oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(HotRegionType, nil)) re.NoError(err) hb := sche.(*hotScheduler) re.Equal([]string{statistics.QueryPriority, statistics.BytePriority}, hb.conf.GetReadPriorities()) @@ -78,7 +80,7 @@ func TestUpgrade(t *testing.T) { re.Equal([]string{statistics.BytePriority, statistics.KeyPriority}, hb.conf.GetWritePeerPriorities()) re.Equal("v2", hb.conf.GetRankFormulaVersion()) // upgrade from json(null) - sche, err = schedule.CreateScheduler(HotRegionType, oc, storage.NewStorageWithMemoryBackend(), schedule.ConfigJSONDecoder([]byte("null"))) + sche, err = CreateScheduler(HotRegionType, oc, storage.NewStorageWithMemoryBackend(), ConfigJSONDecoder([]byte("null"))) re.NoError(err) hb = sche.(*hotScheduler) re.Equal([]string{statistics.QueryPriority, statistics.BytePriority}, hb.conf.GetReadPriorities()) @@ -87,7 +89,7 @@ func TestUpgrade(t *testing.T) { re.Equal("v2", hb.conf.GetRankFormulaVersion()) // upgrade from < 5.2 config51 := `{"min-hot-byte-rate":100,"min-hot-key-rate":10,"min-hot-query-rate":10,"max-zombie-rounds":5,"max-peer-number":1000,"byte-rate-rank-step-ratio":0.05,"key-rate-rank-step-ratio":0.05,"query-rate-rank-step-ratio":0.05,"count-rank-step-ratio":0.01,"great-dec-ratio":0.95,"minor-dec-ratio":0.99,"src-tolerance-ratio":1.05,"dst-tolerance-ratio":1.05,"strict-picking-store":"true","enable-for-tiflash":"true"}` - sche, err = schedule.CreateScheduler(HotRegionType, oc, storage.NewStorageWithMemoryBackend(), schedule.ConfigJSONDecoder([]byte(config51))) + sche, err = CreateScheduler(HotRegionType, oc, storage.NewStorageWithMemoryBackend(), ConfigJSONDecoder([]byte(config51))) re.NoError(err) hb = sche.(*hotScheduler) re.Equal([]string{statistics.BytePriority, statistics.KeyPriority}, hb.conf.GetReadPriorities()) @@ -96,7 +98,7 @@ func TestUpgrade(t *testing.T) { re.Equal("v1", hb.conf.GetRankFormulaVersion()) // upgrade from < 6.4 config54 := `{"min-hot-byte-rate":100,"min-hot-key-rate":10,"min-hot-query-rate":10,"max-zombie-rounds":5,"max-peer-number":1000,"byte-rate-rank-step-ratio":0.05,"key-rate-rank-step-ratio":0.05,"query-rate-rank-step-ratio":0.05,"count-rank-step-ratio":0.01,"great-dec-ratio":0.95,"minor-dec-ratio":0.99,"src-tolerance-ratio":1.05,"dst-tolerance-ratio":1.05,"read-priorities":["query","byte"],"write-leader-priorities":["query","byte"],"write-peer-priorities":["byte","key"],"strict-picking-store":"true","enable-for-tiflash":"true","forbid-rw-type":"none"}` - sche, err = schedule.CreateScheduler(HotRegionType, oc, storage.NewStorageWithMemoryBackend(), schedule.ConfigJSONDecoder([]byte(config54))) + sche, err = CreateScheduler(HotRegionType, oc, storage.NewStorageWithMemoryBackend(), ConfigJSONDecoder([]byte(config54))) re.NoError(err) hb = sche.(*hotScheduler) re.Equal([]string{statistics.QueryPriority, statistics.BytePriority}, hb.conf.GetReadPriorities()) @@ -122,7 +124,7 @@ func checkGCPendingOpInfos(re *require.Assertions, enablePlacementRules bool) { tc.PutStoreWithLabels(id) } - sche, err := schedule.CreateScheduler(HotRegionType, oc, storage.NewStorageWithMemoryBackend(), schedule.ConfigJSONDecoder([]byte("null"))) + sche, err := CreateScheduler(HotRegionType, oc, storage.NewStorageWithMemoryBackend(), ConfigJSONDecoder([]byte("null"))) re.NoError(err) hb := sche.(*hotScheduler) @@ -140,11 +142,11 @@ func checkGCPendingOpInfos(re *require.Assertions, enablePlacementRules bool) { op.Start() op.SetStatusReachTime(operator.CREATED, time.Now().Add(-5*statistics.StoreHeartBeatReportInterval*time.Second)) op.SetStatusReachTime(operator.STARTED, time.Now().Add((-5*statistics.StoreHeartBeatReportInterval+1)*time.Second)) - return newPendingInfluence(op, 2, 4, statistics.Influence{}, hb.conf.GetStoreStatZombieDuration()) + return newPendingInfluence(op, []uint64{2}, 4, statistics.Influence{}, hb.conf.GetStoreStatZombieDuration()) } justDoneOpInfluence := func(region *core.RegionInfo, ty opType) *pendingInfluence { infl := notDoneOpInfluence(region, ty) - infl.op.Cancel() + infl.op.Cancel(operator.AdminStop) return infl } shouldRemoveOpInfluence := func(region *core.RegionInfo, ty opType) *pendingInfluence { @@ -164,7 +166,7 @@ func checkGCPendingOpInfos(re *require.Assertions, enablePlacementRules bool) { } } - hb.summaryPendingInfluence(tc) // Calling this function will GC. + hb.summaryPendingInfluence() // Calling this function will GC. for i := range opInfluenceCreators { for j, typ := range typs { @@ -195,11 +197,200 @@ func newTestRegion(id uint64) *core.RegionInfo { func TestHotWriteRegionScheduleByteRateOnly(t *testing.T) { re := require.New(t) statistics.Denoising = false + statistics.HistorySampleDuration = 0 statisticsInterval = 0 checkHotWriteRegionScheduleByteRateOnly(re, false /* disable placement rules */) checkHotWriteRegionScheduleByteRateOnly(re, true /* enable placement rules */) } +func TestSplitIfRegionTooHot(t *testing.T) { + re := require.New(t) + statistics.Denoising = false + cancel, _, tc, oc := prepareSchedulersTest() + defer cancel() + tc.SetHotRegionCacheHitsThreshold(1) + hb, err := CreateScheduler(statistics.Read.String(), oc, storage.NewStorageWithMemoryBackend(), nil) + re.NoError(err) + b := &metapb.Buckets{ + RegionId: 1, + PeriodInMs: 1000, + Keys: [][]byte{ + []byte(fmt.Sprintf("%21d", 11)), + []byte(fmt.Sprintf("%21d", 12)), + []byte(fmt.Sprintf("%21d", 13)), + []byte(fmt.Sprintf("%21d", 14)), + []byte(fmt.Sprintf("%21d", 15)), + }, + Stats: &metapb.BucketStats{ + ReadBytes: []uint64{10 * units.KiB, 11 * units.KiB, 11 * units.KiB, 10 * units.KiB}, + ReadKeys: []uint64{256, 256, 156, 256}, + ReadQps: []uint64{0, 0, 0, 0}, + WriteBytes: []uint64{100 * units.KiB, 10 * units.KiB, 10 * units.KiB, 10 * units.KiB}, + WriteQps: []uint64{256, 256, 156, 256}, + WriteKeys: []uint64{0, 0, 0, 0}, + }, + } + + task := buckets.NewCheckPeerTask(b) + re.True(tc.HotBucketCache.CheckAsync(task)) + time.Sleep(time.Millisecond * 10) + + tc.AddRegionStore(1, 3) + tc.AddRegionStore(2, 2) + tc.AddRegionStore(3, 2) + + tc.UpdateStorageReadBytes(1, 6*units.MiB*statistics.StoreHeartBeatReportInterval) + tc.UpdateStorageReadBytes(2, 1*units.MiB*statistics.StoreHeartBeatReportInterval) + tc.UpdateStorageReadBytes(3, 1*units.MiB*statistics.StoreHeartBeatReportInterval) + // Region 1, 2 and 3 are hot regions. + addRegionInfo(tc, statistics.Read, []testRegionInfo{ + {1, []uint64{1, 2, 3}, 4 * units.MiB, 0, 0}, + }) + tc.GetStoreConfig().SetRegionBucketEnabled(true) + ops, _ := hb.Schedule(tc, false) + re.Len(ops, 1) + expectOp, _ := operator.CreateSplitRegionOperator(splitHotReadBuckets, tc.GetRegion(1), operator.OpSplit, + pdpb.CheckPolicy_USEKEY, [][]byte{[]byte(fmt.Sprintf("%21d", 13))}) + re.Equal(expectOp.Brief(), ops[0].Brief()) + re.Equal(expectOp.Kind(), ops[0].Kind()) + + ops, _ = hb.Schedule(tc, false) + re.Len(ops, 0) + + tc.UpdateStorageWrittenBytes(1, 6*units.MiB*statistics.StoreHeartBeatReportInterval) + tc.UpdateStorageWrittenBytes(2, 1*units.MiB*statistics.StoreHeartBeatReportInterval) + tc.UpdateStorageWrittenBytes(3, 1*units.MiB*statistics.StoreHeartBeatReportInterval) + // Region 1, 2 and 3 are hot regions. + addRegionInfo(tc, statistics.Write, []testRegionInfo{ + {1, []uint64{1, 2, 3}, 4 * units.MiB, 0, 0}, + }) + hb, _ = CreateScheduler(statistics.Write.String(), oc, storage.NewStorageWithMemoryBackend(), nil) + ops, _ = hb.Schedule(tc, false) + re.Len(ops, 1) + expectOp, _ = operator.CreateSplitRegionOperator(splitHotReadBuckets, tc.GetRegion(1), operator.OpSplit, + pdpb.CheckPolicy_USEKEY, [][]byte{[]byte(fmt.Sprintf("%21d", 12))}) + re.Equal(expectOp.Brief(), ops[0].Brief()) + re.Equal(expectOp.Kind(), ops[0].Kind()) + re.Equal(operator.OpSplit, ops[0].Kind()) + + ops, _ = hb.Schedule(tc, false) + re.Len(ops, 0) +} + +func TestSplitBucketsBySize(t *testing.T) { + re := require.New(t) + statistics.Denoising = false + cancel, _, tc, oc := prepareSchedulersTest() + tc.SetHotRegionCacheHitsThreshold(1) + tc.GetStoreConfig().SetRegionBucketEnabled(true) + defer cancel() + hb, err := CreateScheduler(statistics.Read.String(), oc, storage.NewStorageWithMemoryBackend(), nil) + re.NoError(err) + solve := newBalanceSolver(hb.(*hotScheduler), tc, statistics.Read, transferLeader) + solve.cur = &solution{} + region := core.NewTestRegionInfo(1, 1, []byte("a"), []byte("f")) + + testdata := []struct { + hotBuckets [][]byte + splitKeys [][]byte + }{ + { + [][]byte{[]byte("a"), []byte("b"), []byte("f")}, + [][]byte{[]byte("b")}, + }, + { + [][]byte{[]byte(""), []byte("a"), []byte("")}, + nil, + }, + { + [][]byte{}, + nil, + }, + } + + for _, data := range testdata { + b := &metapb.Buckets{ + RegionId: 1, + PeriodInMs: 1000, + Keys: data.hotBuckets, + } + region.UpdateBuckets(b, region.GetBuckets()) + ops := solve.createSplitOperator([]*core.RegionInfo{region}, bySize) + if data.splitKeys == nil { + re.Equal(0, len(ops)) + continue + } + re.Equal(1, len(ops)) + op := ops[0] + re.Equal(splitHotReadBuckets, op.Desc()) + + expectOp, err := operator.CreateSplitRegionOperator(splitHotReadBuckets, region, operator.OpSplit, pdpb.CheckPolicy_USEKEY, data.splitKeys) + re.NoError(err) + re.Equal(expectOp.Brief(), op.Brief()) + } +} + +func TestSplitBucketsByLoad(t *testing.T) { + re := require.New(t) + statistics.Denoising = false + cancel, _, tc, oc := prepareSchedulersTest() + tc.SetHotRegionCacheHitsThreshold(1) + tc.GetStoreConfig().SetRegionBucketEnabled(true) + defer cancel() + hb, err := CreateScheduler(statistics.Read.String(), oc, storage.NewStorageWithMemoryBackend(), nil) + re.NoError(err) + solve := newBalanceSolver(hb.(*hotScheduler), tc, statistics.Read, transferLeader) + solve.cur = &solution{} + region := core.NewTestRegionInfo(1, 1, []byte("a"), []byte("f")) + testdata := []struct { + hotBuckets [][]byte + splitKeys [][]byte + }{ + { + [][]byte{[]byte(""), []byte("b"), []byte("")}, + [][]byte{[]byte("b")}, + }, + { + [][]byte{[]byte(""), []byte("a"), []byte("")}, + nil, + }, + { + [][]byte{[]byte("b"), []byte("c"), []byte("")}, + [][]byte{[]byte("c")}, + }, + } + for _, data := range testdata { + b := &metapb.Buckets{ + RegionId: 1, + PeriodInMs: 1000, + Keys: data.hotBuckets, + Stats: &metapb.BucketStats{ + ReadBytes: []uint64{10 * units.KiB, 10 * units.MiB}, + ReadKeys: []uint64{256, 256}, + ReadQps: []uint64{0, 0}, + WriteBytes: []uint64{0, 0}, + WriteQps: []uint64{0, 0}, + WriteKeys: []uint64{0, 0}, + }, + } + task := buckets.NewCheckPeerTask(b) + re.True(tc.HotBucketCache.CheckAsync(task)) + time.Sleep(time.Millisecond * 10) + ops := solve.createSplitOperator([]*core.RegionInfo{region}, byLoad) + if data.splitKeys == nil { + re.Equal(0, len(ops)) + continue + } + re.Equal(1, len(ops)) + op := ops[0] + re.Equal(splitHotReadBuckets, op.Desc()) + + expectOp, err := operator.CreateSplitRegionOperator(splitHotReadBuckets, region, operator.OpSplit, pdpb.CheckPolicy_USEKEY, data.splitKeys) + re.NoError(err) + re.Equal(expectOp.Brief(), op.Brief()) + } +} + func checkHotWriteRegionScheduleByteRateOnly(re *require.Assertions, enablePlacementRules bool) { cancel, opt, tc, oc := prepareSchedulersTest() defer cancel() @@ -207,7 +398,7 @@ func checkHotWriteRegionScheduleByteRateOnly(re *require.Assertions, enablePlace tc.SetEnablePlacementRules(enablePlacementRules) labels := []string{"zone", "host"} tc.SetMaxReplicasWithLabel(enablePlacementRules, 3, labels...) - hb, err := schedule.CreateScheduler(statistics.Write.String(), oc, storage.NewStorageWithMemoryBackend(), nil) + hb, err := CreateScheduler(statistics.Write.String(), oc, storage.NewStorageWithMemoryBackend(), nil) re.NoError(err) tc.SetHotRegionCacheHitsThreshold(0) @@ -260,15 +451,15 @@ func checkHotWriteRegionScheduleByteRateOnly(re *require.Assertions, enablePlace switch op.Len() { case 1: // balance by leader selected - testutil.CheckTransferLeaderFrom(re, op, operator.OpHotRegion, 1) + operatorutil.CheckTransferLeaderFrom(re, op, operator.OpHotRegion, 1) case 4: // balance by peer selected if op.RegionID() == 2 { // peer in store 1 of the region 2 can transfer to store 5 or store 6 because of the label - testutil.CheckTransferPeerWithLeaderTransferFrom(re, op, operator.OpHotRegion, 1) + operatorutil.CheckTransferPeerWithLeaderTransferFrom(re, op, operator.OpHotRegion, 1) } else { // peer in store 1 of the region 1,3 can only transfer to store 6 - testutil.CheckTransferPeerWithLeaderTransfer(re, op, operator.OpHotRegion, 1, 6) + operatorutil.CheckTransferPeerWithLeaderTransfer(re, op, operator.OpHotRegion, 1, 6) } default: re.FailNow("wrong op: " + op.String()) @@ -288,10 +479,10 @@ func checkHotWriteRegionScheduleByteRateOnly(re *require.Assertions, enablePlace re.Equal(4, op.Len()) if op.RegionID() == 2 { // peer in store 1 of the region 2 can transfer to store 5 or store 6 because of the label - testutil.CheckTransferPeerWithLeaderTransferFrom(re, op, operator.OpHotRegion, 1) + operatorutil.CheckTransferPeerWithLeaderTransferFrom(re, op, operator.OpHotRegion, 1) } else { // peer in store 1 of the region 1,3 can only transfer to store 6 - testutil.CheckTransferPeerWithLeaderTransfer(re, op, operator.OpHotRegion, 1, 6) + operatorutil.CheckTransferPeerWithLeaderTransfer(re, op, operator.OpHotRegion, 1, 6) } } @@ -354,16 +545,16 @@ func checkHotWriteRegionScheduleByteRateOnly(re *require.Assertions, enablePlace switch op.RegionID() { case 1, 2: if op.Len() == 3 { - testutil.CheckTransferPeer(re, op, operator.OpHotRegion, 3, 6) + operatorutil.CheckTransferPeer(re, op, operator.OpHotRegion, 3, 6) } else if op.Len() == 4 { - testutil.CheckTransferPeerWithLeaderTransfer(re, op, operator.OpHotRegion, 1, 6) + operatorutil.CheckTransferPeerWithLeaderTransfer(re, op, operator.OpHotRegion, 1, 6) } else { re.FailNow("wrong operator: " + op.String()) } case 3: - testutil.CheckTransferPeer(re, op, operator.OpHotRegion, 1, 5) + operatorutil.CheckTransferPeer(re, op, operator.OpHotRegion, 1, 5) case 5: - testutil.CheckTransferPeerWithLeaderTransfer(re, op, operator.OpHotRegion, 3, 6) + operatorutil.CheckTransferPeerWithLeaderTransfer(re, op, operator.OpHotRegion, 3, 6) default: re.FailNow("wrong operator: " + op.String()) } @@ -410,7 +601,7 @@ func TestHotWriteRegionScheduleByteRateOnlyWithTiFlash(t *testing.T) { }, }, })) - sche, err := schedule.CreateScheduler(statistics.Write.String(), oc, storage.NewStorageWithMemoryBackend(), nil) + sche, err := CreateScheduler(statistics.Write.String(), oc, storage.NewStorageWithMemoryBackend(), nil) re.NoError(err) hb := sche.(*hotScheduler) @@ -478,10 +669,10 @@ func TestHotWriteRegionScheduleByteRateOnlyWithTiFlash(t *testing.T) { switch op.Len() { case 1: // balance by leader selected - testutil.CheckTransferLeaderFrom(re, op, operator.OpHotRegion, 1) + operatorutil.CheckTransferLeaderFrom(re, op, operator.OpHotRegion, 1) case 2: // balance by peer selected - testutil.CheckTransferLearner(re, op, operator.OpHotRegion, 8, 10) + operatorutil.CheckTransferLearner(re, op, operator.OpHotRegion, 8, 10) default: re.FailNow("wrong op: " + op.String()) } @@ -492,7 +683,7 @@ func TestHotWriteRegionScheduleByteRateOnlyWithTiFlash(t *testing.T) { clearPendingInfluence(hb) ops, _ := hb.Schedule(tc, false) op := ops[0] - testutil.CheckTransferLeaderFrom(re, op, operator.OpHotRegion, 1) + operatorutil.CheckTransferLeaderFrom(re, op, operator.OpHotRegion, 1) } // | store_id | write_bytes_rate | // |----------|------------------| @@ -520,11 +711,13 @@ func TestHotWriteRegionScheduleByteRateOnlyWithTiFlash(t *testing.T) { tikvKeysSum += float64(storesBytes[i]/100) / 10 tikvQuerySum += float64(storesBytes[i]/100) / 10 } + for i := uint64(1); i <= storeCount; i++ { if i != downStoreID { tc.UpdateStorageWrittenBytes(i, storesBytes[i]) } } + { // Check the load expect aliveTiKVCount := float64(aliveTiKVLastID - aliveTiKVStartID + 1) allowLeaderTiKVCount := aliveTiKVCount - 1 // store 5 with evict leader @@ -569,15 +762,15 @@ func TestHotWriteRegionScheduleByteRateOnlyWithTiFlash(t *testing.T) { switch op.Len() { case 1: // balance by leader selected - testutil.CheckTransferLeaderFrom(re, op, operator.OpHotRegion, 1) + operatorutil.CheckTransferLeaderFrom(re, op, operator.OpHotRegion, 1) case 4: // balance by peer selected if op.RegionID() == 2 { // peer in store 1 of the region 2 can transfer to store 5 or store 6 because of the label - testutil.CheckTransferPeerWithLeaderTransferFrom(re, op, operator.OpHotRegion, 1) + operatorutil.CheckTransferPeerWithLeaderTransferFrom(re, op, operator.OpHotRegion, 1) } else { // peer in store 1 of the region 1,3 can only transfer to store 6 - testutil.CheckTransferPeerWithLeaderTransfer(re, op, operator.OpHotRegion, 1, 6) + operatorutil.CheckTransferPeerWithLeaderTransfer(re, op, operator.OpHotRegion, 1, 6) } default: re.FailNow("wrong op: " + op.String()) @@ -597,7 +790,7 @@ func TestHotWriteRegionScheduleWithQuery(t *testing.T) { statistics.Denoising = false statisticsInterval = 0 - hb, err := schedule.CreateScheduler(statistics.Write.String(), oc, storage.NewStorageWithMemoryBackend(), nil) + hb, err := CreateScheduler(statistics.Write.String(), oc, storage.NewStorageWithMemoryBackend(), nil) re.NoError(err) hb.(*hotScheduler).conf.SetSrcToleranceRatio(1) hb.(*hotScheduler).conf.SetDstToleranceRatio(1) @@ -622,7 +815,7 @@ func TestHotWriteRegionScheduleWithQuery(t *testing.T) { clearPendingInfluence(hb.(*hotScheduler)) ops, _ := hb.Schedule(tc, false) op := ops[0] - testutil.CheckTransferLeader(re, op, operator.OpHotRegion, 1, 3) + operatorutil.CheckTransferLeader(re, op, operator.OpHotRegion, 1, 3) } } @@ -633,7 +826,7 @@ func TestHotWriteRegionScheduleWithKeyRate(t *testing.T) { cancel, _, tc, oc := prepareSchedulersTest() defer cancel() - hb, err := schedule.CreateScheduler(statistics.Write.String(), oc, storage.NewStorageWithMemoryBackend(), nil) + hb, err := CreateScheduler(statistics.Write.String(), oc, storage.NewStorageWithMemoryBackend(), nil) re.NoError(err) hb.(*hotScheduler).conf.SetDstToleranceRatio(1) hb.(*hotScheduler).conf.SetSrcToleranceRatio(1) @@ -665,21 +858,21 @@ func TestHotWriteRegionScheduleWithKeyRate(t *testing.T) { ops, _ := hb.Schedule(tc, false) op := ops[0] // byteDecRatio <= 0.95 && keyDecRatio <= 0.95 - testutil.CheckTransferPeer(re, op, operator.OpHotRegion, 1, 4) + operatorutil.CheckTransferPeer(re, op, operator.OpHotRegion, 1, 4) // store byte rate (min, max): (10, 10.5) | 9.5 | 9.5 | (9, 9.5) | 8.9 // store key rate (min, max): (10, 10.5) | 9.5 | 9.8 | (9, 9.5) | 9.2 ops, _ = hb.Schedule(tc, false) op = ops[0] // byteDecRatio <= 0.99 && keyDecRatio <= 0.95 - testutil.CheckTransferPeer(re, op, operator.OpHotRegion, 3, 5) + operatorutil.CheckTransferPeer(re, op, operator.OpHotRegion, 3, 5) // store byte rate (min, max): (10, 10.5) | 9.5 | (9.45, 9.5) | (9, 9.5) | (8.9, 8.95) // store key rate (min, max): (10, 10.5) | 9.5 | (9.7, 9.8) | (9, 9.5) | (9.2, 9.3) // byteDecRatio <= 0.95 // op = hb.Schedule(tc, false)[0] // FIXME: cover this case - // testutil.CheckTransferPeerWithLeaderTransfer(re, op, operator.OpHotRegion, 1, 5) + // operatorutil.CheckTransferPeerWithLeaderTransfer(re, op, operator.OpHotRegion, 1, 5) // store byte rate (min, max): (9.5, 10.5) | 9.5 | (9.45, 9.5) | (9, 9.5) | (8.9, 9.45) // store key rate (min, max): (9.2, 10.2) | 9.5 | (9.7, 9.8) | (9, 9.5) | (9.2, 9.8) } @@ -692,7 +885,7 @@ func TestHotWriteRegionScheduleUnhealthyStore(t *testing.T) { cancel, _, tc, oc := prepareSchedulersTest() defer cancel() - hb, err := schedule.CreateScheduler(statistics.Write.String(), oc, storage.NewStorageWithMemoryBackend(), nil) + hb, err := CreateScheduler(statistics.Write.String(), oc, storage.NewStorageWithMemoryBackend(), nil) re.NoError(err) hb.(*hotScheduler).conf.SetDstToleranceRatio(1) hb.(*hotScheduler).conf.SetSrcToleranceRatio(1) @@ -740,7 +933,7 @@ func TestHotWriteRegionScheduleCheckHot(t *testing.T) { cancel, _, tc, oc := prepareSchedulersTest() defer cancel() - hb, err := schedule.CreateScheduler(statistics.Write.String(), oc, storage.NewStorageWithMemoryBackend(), nil) + hb, err := CreateScheduler(statistics.Write.String(), oc, storage.NewStorageWithMemoryBackend(), nil) re.NoError(err) hb.(*hotScheduler).conf.SetDstToleranceRatio(1) hb.(*hotScheduler).conf.SetSrcToleranceRatio(1) @@ -775,7 +968,7 @@ func TestHotWriteRegionScheduleWithLeader(t *testing.T) { cancel, _, tc, oc := prepareSchedulersTest() defer cancel() - hb, err := schedule.CreateScheduler(statistics.Write.String(), oc, storage.NewStorageWithMemoryBackend(), nil) + hb, err := CreateScheduler(statistics.Write.String(), oc, storage.NewStorageWithMemoryBackend(), nil) hb.(*hotScheduler).conf.WriteLeaderPriorities = []string{statistics.KeyPriority, statistics.BytePriority} re.NoError(err) @@ -824,7 +1017,7 @@ func TestHotWriteRegionScheduleWithLeader(t *testing.T) { clearPendingInfluence(hb.(*hotScheduler)) ops, _ := hb.Schedule(tc, false) op := ops[0] - testutil.CheckTransferLeaderFrom(re, op, operator.OpHotRegion, 2) + operatorutil.CheckTransferLeaderFrom(re, op, operator.OpHotRegion, 2) ops, _ = hb.Schedule(tc, false) re.Empty(ops) } @@ -841,7 +1034,7 @@ func TestHotWriteRegionScheduleWithPendingInfluence(t *testing.T) { func checkHotWriteRegionScheduleWithPendingInfluence(re *require.Assertions, dim int) { cancel, _, tc, oc := prepareSchedulersTest() defer cancel() - hb, err := schedule.CreateScheduler(statistics.Write.String(), oc, storage.NewStorageWithMemoryBackend(), nil) + hb, err := CreateScheduler(statistics.Write.String(), oc, storage.NewStorageWithMemoryBackend(), nil) re.NoError(err) hb.(*hotScheduler).conf.WriteLeaderPriorities = []string{statistics.KeyPriority, statistics.BytePriority} hb.(*hotScheduler).conf.RankFormulaVersion = "v1" @@ -906,13 +1099,13 @@ func checkHotWriteRegionScheduleWithPendingInfluence(re *require.Assertions, dim switch op.Len() { case 1: // balance by leader selected - testutil.CheckTransferLeaderFrom(re, op, operator.OpHotRegion, 1) + operatorutil.CheckTransferLeaderFrom(re, op, operator.OpHotRegion, 1) case 4: // balance by peer selected - testutil.CheckTransferPeerWithLeaderTransfer(re, op, operator.OpHotRegion, 1, 4) + operatorutil.CheckTransferPeerWithLeaderTransfer(re, op, operator.OpHotRegion, 1, 4) cnt++ if cnt == 3 { - re.True(op.Cancel()) + re.True(op.Cancel(operator.AdminStop)) } default: re.FailNow("wrong op: " + op.String()) @@ -929,7 +1122,7 @@ func TestHotWriteRegionScheduleWithRuleEnabled(t *testing.T) { cancel, _, tc, oc := prepareSchedulersTest() defer cancel() tc.SetEnablePlacementRules(true) - hb, err := schedule.CreateScheduler(statistics.Write.String(), oc, storage.NewStorageWithMemoryBackend(), nil) + hb, err := CreateScheduler(statistics.Write.String(), oc, storage.NewStorageWithMemoryBackend(), nil) re.NoError(err) hb.(*hotScheduler).conf.WriteLeaderPriorities = []string{statistics.KeyPriority, statistics.BytePriority} @@ -1000,7 +1193,7 @@ func TestHotWriteRegionScheduleWithRuleEnabled(t *testing.T) { ops, _ := hb.Schedule(tc, false) op := ops[0] // The targetID should always be 1 as leader is only allowed to be placed in store1 or store2 by placement rule - testutil.CheckTransferLeader(re, op, operator.OpHotRegion, 2, 1) + operatorutil.CheckTransferLeader(re, op, operator.OpHotRegion, 2, 1) ops, _ = hb.Schedule(tc, false) re.Empty(ops) } @@ -1011,7 +1204,7 @@ func TestHotReadRegionScheduleByteRateOnly(t *testing.T) { cancel, _, tc, oc := prepareSchedulersTest() defer cancel() tc.SetClusterVersion(versioninfo.MinSupportedVersion(versioninfo.Version4_0)) - scheduler, err := schedule.CreateScheduler(statistics.Read.String(), oc, storage.NewStorageWithMemoryBackend(), nil) + scheduler, err := CreateScheduler(statistics.Read.String(), oc, storage.NewStorageWithMemoryBackend(), nil) re.NoError(err) hb := scheduler.(*hotScheduler) hb.conf.ReadPriorities = []string{statistics.BytePriority, statistics.KeyPriority} @@ -1070,7 +1263,7 @@ func TestHotReadRegionScheduleByteRateOnly(t *testing.T) { // move leader from store 1 to store 5 // it is better than transfer leader from store 1 to store 3 - testutil.CheckTransferPeerWithLeaderTransfer(re, op, operator.OpHotRegion, 1, 5) + operatorutil.CheckTransferPeerWithLeaderTransfer(re, op, operator.OpHotRegion, 1, 5) re.Contains(hb.regionPendings, uint64(1)) re.True(typeutil.Float64Equal(512.0*units.KiB, hb.regionPendings[1].origin.Loads[statistics.RegionReadBytes])) clearPendingInfluence(hb) @@ -1109,7 +1302,7 @@ func TestHotReadRegionScheduleByteRateOnly(t *testing.T) { // We will move leader peer of region 1 from 1 to 5 ops, _ = hb.Schedule(tc, false) op = ops[0] - testutil.CheckTransferPeerWithLeaderTransfer(re, op, operator.OpHotRegion|operator.OpLeader, 1, 5) + operatorutil.CheckTransferPeerWithLeaderTransfer(re, op, operator.OpHotRegion|operator.OpLeader, 1, 5) re.Contains(hb.regionPendings, uint64(1)) re.True(typeutil.Float64Equal(512.0*units.KiB, hb.regionPendings[1].origin.Loads[statistics.RegionReadBytes])) clearPendingInfluence(hb) @@ -1120,6 +1313,7 @@ func TestHotReadRegionScheduleByteRateOnly(t *testing.T) { tc.RemoveRegion(r) tc.RemoveRegionFromSubTree(r) } + hb.updateReadTime = time.Now().Add(-time.Second) hb.Schedule(tc, false) re.Contains(hb.regionPendings, uint64(4)) re.True(typeutil.Float64Equal(509.0*units.KiB, hb.regionPendings[4].origin.Loads[statistics.RegionReadBytes])) @@ -1133,7 +1327,7 @@ func TestHotReadRegionScheduleWithQuery(t *testing.T) { cancel, _, tc, oc := prepareSchedulersTest() defer cancel() - hb, err := schedule.CreateScheduler(statistics.Read.String(), oc, storage.NewStorageWithMemoryBackend(), nil) + hb, err := CreateScheduler(statistics.Read.String(), oc, storage.NewStorageWithMemoryBackend(), nil) re.NoError(err) hb.(*hotScheduler).conf.SetSrcToleranceRatio(1) hb.(*hotScheduler).conf.SetDstToleranceRatio(1) @@ -1157,7 +1351,7 @@ func TestHotReadRegionScheduleWithQuery(t *testing.T) { clearPendingInfluence(hb.(*hotScheduler)) ops, _ := hb.Schedule(tc, false) op := ops[0] - testutil.CheckTransferLeader(re, op, operator.OpHotRegion, 1, 3) + operatorutil.CheckTransferLeader(re, op, operator.OpHotRegion, 1, 3) } } @@ -1168,7 +1362,7 @@ func TestHotReadRegionScheduleWithKeyRate(t *testing.T) { cancel, _, tc, oc := prepareSchedulersTest() defer cancel() - hb, err := schedule.CreateScheduler(statistics.Read.String(), oc, storage.NewStorageWithMemoryBackend(), nil) + hb, err := CreateScheduler(statistics.Read.String(), oc, storage.NewStorageWithMemoryBackend(), nil) re.NoError(err) hb.(*hotScheduler).conf.RankFormulaVersion = "v1" hb.(*hotScheduler).conf.SetSrcToleranceRatio(1) @@ -1199,21 +1393,21 @@ func TestHotReadRegionScheduleWithKeyRate(t *testing.T) { ops, _ := hb.Schedule(tc, false) op := ops[0] // byteDecRatio <= 0.95 && keyDecRatio <= 0.95 - testutil.CheckTransferLeader(re, op, operator.OpHotRegion, 1, 4) + operatorutil.CheckTransferLeader(re, op, operator.OpHotRegion, 1, 4) // store byte rate (min, max): (10, 10.5) | 9.5 | 9.5 | (9, 9.5) | 8.9 // store key rate (min, max): (10, 10.5) | 9.5 | 9.8 | (9, 9.5) | 9.2 ops, _ = hb.Schedule(tc, false) op = ops[0] // byteDecRatio <= 0.99 && keyDecRatio <= 0.95 - testutil.CheckTransferLeader(re, op, operator.OpHotRegion, 3, 5) + operatorutil.CheckTransferLeader(re, op, operator.OpHotRegion, 3, 5) // store byte rate (min, max): (10, 10.5) | 9.5 | (9.45, 9.5) | (9, 9.5) | (8.9, 8.95) // store key rate (min, max): (10, 10.5) | 9.5 | (9.7, 9.8) | (9, 9.5) | (9.2, 9.3) // byteDecRatio <= 0.95 // FIXME: cover this case // op = hb.Schedule(tc, false)[0] - // testutil.CheckTransferPeerWithLeaderTransfer(re, op, operator.OpHotRegion, 1, 5) + // operatorutil.CheckTransferPeerWithLeaderTransfer(re, op, operator.OpHotRegion, 1, 5) // store byte rate (min, max): (9.5, 10.5) | 9.5 | (9.45, 9.5) | (9, 9.5) | (8.9, 9.45) // store key rate (min, max): (9.2, 10.2) | 9.5 | (9.7, 9.8) | (9, 9.5) | (9.2, 9.8) } @@ -1230,7 +1424,7 @@ func TestHotReadRegionScheduleWithPendingInfluence(t *testing.T) { func checkHotReadRegionScheduleWithPendingInfluence(re *require.Assertions, dim int) { cancel, _, tc, oc := prepareSchedulersTest() defer cancel() - hb, err := schedule.CreateScheduler(statistics.Read.String(), oc, storage.NewStorageWithMemoryBackend(), nil) + hb, err := CreateScheduler(statistics.Read.String(), oc, storage.NewStorageWithMemoryBackend(), nil) re.NoError(err) // For test hb.(*hotScheduler).conf.RankFormulaVersion = "v1" @@ -1291,7 +1485,7 @@ func checkHotReadRegionScheduleWithPendingInfluence(re *require.Assertions, dim ops, _ := hb.Schedule(tc, false) op1 := ops[0] - testutil.CheckTransferPeer(re, op1, operator.OpHotRegion, 1, 4) + operatorutil.CheckTransferPeer(re, op1, operator.OpHotRegion, 1, 4) // After move-peer, store byte/key rate (min, max): (6.6, 7.1) | 6.1 | 6 | (5, 5.5) pendingAmpFactor = old @@ -1301,7 +1495,7 @@ func checkHotReadRegionScheduleWithPendingInfluence(re *require.Assertions, dim ops, _ = hb.Schedule(tc, false) op2 := ops[0] - testutil.CheckTransferPeer(re, op2, operator.OpHotRegion, 1, 4) + operatorutil.CheckTransferPeer(re, op2, operator.OpHotRegion, 1, 4) // After move-peer, store byte/key rate (min, max): (6.1, 7.1) | 6.1 | 6 | (5, 6) ops, _ = hb.Schedule(tc, false) @@ -1314,26 +1508,26 @@ func checkHotReadRegionScheduleWithPendingInfluence(re *require.Assertions, dim ops, _ := hb.Schedule(tc, false) op1 := ops[0] - testutil.CheckTransferPeer(re, op1, operator.OpHotRegion, 1, 4) + operatorutil.CheckTransferPeer(re, op1, operator.OpHotRegion, 1, 4) // After move-peer, store byte/key rate (min, max): (6.6, 7.1) | 6.1 | 6 | (5, 5.5) ops, _ = hb.Schedule(tc, false) op2 := ops[0] - testutil.CheckTransferPeer(re, op2, operator.OpHotRegion, 1, 4) + operatorutil.CheckTransferPeer(re, op2, operator.OpHotRegion, 1, 4) // After move-peer, store byte/key rate (min, max): (6.1, 7.1) | 6.1 | 6 | (5, 6) - re.True(op2.Cancel()) + re.True(op2.Cancel(operator.AdminStop)) ops, _ = hb.Schedule(tc, false) op2 = ops[0] - testutil.CheckTransferPeer(re, op2, operator.OpHotRegion, 1, 4) + operatorutil.CheckTransferPeer(re, op2, operator.OpHotRegion, 1, 4) // After move-peer, store byte/key rate (min, max): (6.1, 7.1) | 6.1 | (6, 6.5) | (5, 5.5) - re.True(op1.Cancel()) + re.True(op1.Cancel(operator.AdminStop)) // store byte/key rate (min, max): (6.6, 7.1) | 6.1 | 6 | (5, 5.5) ops, _ = hb.Schedule(tc, false) op3 := ops[0] - testutil.CheckTransferPeer(re, op3, operator.OpHotRegion, 1, 4) + operatorutil.CheckTransferPeer(re, op3, operator.OpHotRegion, 1, 4) // store byte/key rate (min, max): (6.1, 7.1) | 6.1 | 6 | (5, 6) ops, _ = hb.Schedule(tc, false) @@ -1348,7 +1542,7 @@ func TestHotReadWithEvictLeaderScheduler(t *testing.T) { cancel, _, tc, oc := prepareSchedulersTest() defer cancel() - hb, err := schedule.CreateScheduler(statistics.Read.String(), oc, storage.NewStorageWithMemoryBackend(), nil) + hb, err := CreateScheduler(statistics.Read.String(), oc, storage.NewStorageWithMemoryBackend(), nil) re.NoError(err) hb.(*hotScheduler).conf.SetSrcToleranceRatio(1) hb.(*hotScheduler).conf.SetDstToleranceRatio(1) @@ -1374,7 +1568,7 @@ func TestHotReadWithEvictLeaderScheduler(t *testing.T) { ops, _ := hb.Schedule(tc, false) re.Len(ops, 1) clearPendingInfluence(hb.(*hotScheduler)) - testutil.CheckTransferPeerWithLeaderTransfer(re, ops[0], operator.OpHotRegion|operator.OpLeader, 1, 4) + operatorutil.CheckTransferPeerWithLeaderTransfer(re, ops[0], operator.OpHotRegion|operator.OpLeader, 1, 4) // two dim are both enough uniform among three stores tc.SetStoreEvictLeader(4, true) ops, _ = hb.Schedule(tc, false) @@ -1656,7 +1850,7 @@ func checkHotCacheCheckRegionFlow(re *require.Assertions, testCase testHotCacheC tc.SetEnablePlacementRules(enablePlacementRules) labels := []string{"zone", "host"} tc.SetMaxReplicasWithLabel(enablePlacementRules, 3, labels...) - sche, err := schedule.CreateScheduler(HotRegionType, oc, storage.NewStorageWithMemoryBackend(), schedule.ConfigJSONDecoder([]byte("null"))) + sche, err := CreateScheduler(HotRegionType, oc, storage.NewStorageWithMemoryBackend(), ConfigJSONDecoder([]byte("null"))) re.NoError(err) hb := sche.(*hotScheduler) heartbeat := tc.AddLeaderRegionWithWriteInfo @@ -1763,7 +1957,7 @@ func TestHotCacheSortHotPeer(t *testing.T) { re := require.New(t) cancel, _, tc, oc := prepareSchedulersTest() defer cancel() - sche, err := schedule.CreateScheduler(HotRegionType, oc, storage.NewStorageWithMemoryBackend(), schedule.ConfigJSONDecoder([]byte("null"))) + sche, err := CreateScheduler(HotRegionType, oc, storage.NewStorageWithMemoryBackend(), ConfigJSONDecoder([]byte("null"))) re.NoError(err) hb := sche.(*hotScheduler) leaderSolver := newBalanceSolver(hb, tc, statistics.Read, transferLeader) @@ -1787,20 +1981,23 @@ func TestHotCacheSortHotPeer(t *testing.T) { }, }} + st := &statistics.StoreLoadDetail{ + HotPeers: hotPeers, + } leaderSolver.maxPeerNum = 1 - u := leaderSolver.sortHotPeers(hotPeers) + u := leaderSolver.filterHotPeers(st) checkSortResult(re, []uint64{1}, u) leaderSolver.maxPeerNum = 2 - u = leaderSolver.sortHotPeers(hotPeers) + u = leaderSolver.filterHotPeers(st) checkSortResult(re, []uint64{1, 2}, u) } -func checkSortResult(re *require.Assertions, regions []uint64, hotPeers map[*statistics.HotPeerStat]struct{}) { +func checkSortResult(re *require.Assertions, regions []uint64, hotPeers []*statistics.HotPeerStat) { re.Equal(len(hotPeers), len(regions)) for _, region := range regions { in := false - for hotPeer := range hotPeers { + for _, hotPeer := range hotPeers { if hotPeer.RegionID == region { in = true break @@ -1812,6 +2009,7 @@ func checkSortResult(re *require.Assertions, regions []uint64, hotPeers map[*sta func TestInfluenceByRWType(t *testing.T) { re := require.New(t) + statistics.HistorySampleDuration = 0 originValue := schedulePeerPr defer func() { schedulePeerPr = originValue @@ -1821,7 +2019,7 @@ func TestInfluenceByRWType(t *testing.T) { cancel, _, tc, oc := prepareSchedulersTest() defer cancel() - hb, err := schedule.CreateScheduler(statistics.Write.String(), oc, storage.NewStorageWithMemoryBackend(), nil) + hb, err := CreateScheduler(statistics.Write.String(), oc, storage.NewStorageWithMemoryBackend(), nil) re.NoError(err) hb.(*hotScheduler).conf.SetDstToleranceRatio(1) hb.(*hotScheduler).conf.SetSrcToleranceRatio(1) @@ -1848,7 +2046,7 @@ func TestInfluenceByRWType(t *testing.T) { op := ops[0] re.NotNil(op) - hb.(*hotScheduler).summaryPendingInfluence(tc) + hb.(*hotScheduler).summaryPendingInfluence() stInfos := hb.(*hotScheduler).stInfos re.True(nearlyAbout(stInfos[1].PendingSum.Loads[statistics.RegionWriteKeys], -0.5*units.MiB)) re.True(nearlyAbout(stInfos[1].PendingSum.Loads[statistics.RegionWriteBytes], -0.5*units.MiB)) @@ -1873,7 +2071,7 @@ func TestInfluenceByRWType(t *testing.T) { op = ops[0] re.NotNil(op) - hb.(*hotScheduler).summaryPendingInfluence(tc) + hb.(*hotScheduler).summaryPendingInfluence() stInfos = hb.(*hotScheduler).stInfos // assert read/write influence is the sum of write peer and write leader re.True(nearlyAbout(stInfos[1].PendingSum.Loads[statistics.RegionWriteKeys], -1.2*units.MiB)) @@ -1922,7 +2120,7 @@ func checkHotReadPeerSchedule(re *require.Assertions, enablePlacementRules bool) tc.PutStoreWithLabels(id) } - sche, err := schedule.CreateScheduler(statistics.Read.String(), oc, storage.NewStorageWithMemoryBackend(), schedule.ConfigJSONDecoder([]byte("null"))) + sche, err := CreateScheduler(statistics.Read.String(), oc, storage.NewStorageWithMemoryBackend(), ConfigJSONDecoder([]byte("null"))) re.NoError(err) hb := sche.(*hotScheduler) hb.conf.ReadPriorities = []string{statistics.BytePriority, statistics.KeyPriority} @@ -1934,7 +2132,7 @@ func checkHotReadPeerSchedule(re *require.Assertions, enablePlacementRules bool) tc.AddRegionWithPeerReadInfo(1, 3, 1, uint64(0.9*units.KiB*float64(10)), uint64(0.9*units.KiB*float64(10)), 10, []uint64{1, 2}, 3) ops, _ := hb.Schedule(tc, false) op := ops[0] - testutil.CheckTransferPeer(re, op, operator.OpHotRegion, 1, 4) + operatorutil.CheckTransferPeer(re, op, operator.OpHotRegion, 1, 4) } func TestHotScheduleWithPriority(t *testing.T) { @@ -1944,7 +2142,7 @@ func TestHotScheduleWithPriority(t *testing.T) { cancel, _, tc, oc := prepareSchedulersTest() defer cancel() - hb, err := schedule.CreateScheduler(statistics.Write.String(), oc, storage.NewStorageWithMemoryBackend(), nil) + hb, err := CreateScheduler(statistics.Write.String(), oc, storage.NewStorageWithMemoryBackend(), nil) re.NoError(err) hb.(*hotScheduler).conf.SetDstToleranceRatio(1.05) hb.(*hotScheduler).conf.SetSrcToleranceRatio(1.05) @@ -1977,16 +2175,16 @@ func TestHotScheduleWithPriority(t *testing.T) { hb.(*hotScheduler).conf.WritePeerPriorities = []string{statistics.BytePriority, statistics.KeyPriority} ops, _ := hb.Schedule(tc, false) re.Len(ops, 1) - testutil.CheckTransferPeer(re, ops[0], operator.OpHotRegion, 1, 5) + operatorutil.CheckTransferPeer(re, ops[0], operator.OpHotRegion, 1, 5) clearPendingInfluence(hb.(*hotScheduler)) hb.(*hotScheduler).conf.WritePeerPriorities = []string{statistics.KeyPriority, statistics.BytePriority} ops, _ = hb.Schedule(tc, false) re.Len(ops, 1) - testutil.CheckTransferPeer(re, ops[0], operator.OpHotRegion, 4, 5) + operatorutil.CheckTransferPeer(re, ops[0], operator.OpHotRegion, 4, 5) clearPendingInfluence(hb.(*hotScheduler)) // assert read priority schedule - hb, err = schedule.CreateScheduler(statistics.Read.String(), oc, storage.NewStorageWithMemoryBackend(), nil) + hb, err = CreateScheduler(statistics.Read.String(), oc, storage.NewStorageWithMemoryBackend(), nil) re.NoError(err) tc.UpdateStorageReadStats(5, 10*units.MiB*statistics.StoreHeartBeatReportInterval, 10*units.MiB*statistics.StoreHeartBeatReportInterval) tc.UpdateStorageReadStats(4, 10*units.MiB*statistics.StoreHeartBeatReportInterval, 10*units.MiB*statistics.StoreHeartBeatReportInterval) @@ -1999,14 +2197,14 @@ func TestHotScheduleWithPriority(t *testing.T) { hb.(*hotScheduler).conf.ReadPriorities = []string{statistics.BytePriority, statistics.KeyPriority} ops, _ = hb.Schedule(tc, false) re.Len(ops, 1) - testutil.CheckTransferLeader(re, ops[0], operator.OpHotRegion, 1, 2) + operatorutil.CheckTransferLeader(re, ops[0], operator.OpHotRegion, 1, 2) clearPendingInfluence(hb.(*hotScheduler)) hb.(*hotScheduler).conf.ReadPriorities = []string{statistics.KeyPriority, statistics.BytePriority} ops, _ = hb.Schedule(tc, false) re.Len(ops, 1) - testutil.CheckTransferLeader(re, ops[0], operator.OpHotRegion, 1, 3) + operatorutil.CheckTransferLeader(re, ops[0], operator.OpHotRegion, 1, 3) - hb, err = schedule.CreateScheduler(statistics.Write.String(), oc, storage.NewStorageWithMemoryBackend(), nil) + hb, err = CreateScheduler(statistics.Write.String(), oc, storage.NewStorageWithMemoryBackend(), nil) hb.(*hotScheduler).conf.WriteLeaderPriorities = []string{statistics.KeyPriority, statistics.BytePriority} hb.(*hotScheduler).conf.RankFormulaVersion = "v1" re.NoError(err) @@ -2024,7 +2222,7 @@ func TestHotScheduleWithPriority(t *testing.T) { hb.(*hotScheduler).conf.StrictPickingStore = false ops, _ = hb.Schedule(tc, false) re.Len(ops, 1) - testutil.CheckTransferPeer(re, ops[0], operator.OpHotRegion, 1, 5) + operatorutil.CheckTransferPeer(re, ops[0], operator.OpHotRegion, 1, 5) clearPendingInfluence(hb.(*hotScheduler)) tc.UpdateStorageWrittenStats(1, 6*units.MiB*statistics.StoreHeartBeatReportInterval, 6*units.MiB*statistics.StoreHeartBeatReportInterval) @@ -2039,7 +2237,7 @@ func TestHotScheduleWithPriority(t *testing.T) { hb.(*hotScheduler).conf.StrictPickingStore = false ops, _ = hb.Schedule(tc, false) re.Len(ops, 1) - testutil.CheckTransferPeer(re, ops[0], operator.OpHotRegion, 4, 5) + operatorutil.CheckTransferPeer(re, ops[0], operator.OpHotRegion, 4, 5) clearPendingInfluence(hb.(*hotScheduler)) } @@ -2050,7 +2248,7 @@ func TestHotScheduleWithStddev(t *testing.T) { cancel, _, tc, oc := prepareSchedulersTest() defer cancel() - hb, err := schedule.CreateScheduler(statistics.Write.String(), oc, storage.NewStorageWithMemoryBackend(), nil) + hb, err := CreateScheduler(statistics.Write.String(), oc, storage.NewStorageWithMemoryBackend(), nil) re.NoError(err) hb.(*hotScheduler).conf.SetDstToleranceRatio(1.0) hb.(*hotScheduler).conf.SetSrcToleranceRatio(1.0) @@ -2080,7 +2278,7 @@ func TestHotScheduleWithStddev(t *testing.T) { stddevThreshold = -1.0 ops, _ = hb.Schedule(tc, false) re.Len(ops, 1) - testutil.CheckTransferPeer(re, ops[0], operator.OpHotRegion, 2, 5) + operatorutil.CheckTransferPeer(re, ops[0], operator.OpHotRegion, 2, 5) clearPendingInfluence(hb.(*hotScheduler)) // skip -1 case (uniform cluster) @@ -2099,7 +2297,7 @@ func TestHotScheduleWithStddev(t *testing.T) { stddevThreshold = -1.0 ops, _ = hb.Schedule(tc, false) re.Len(ops, 1) - testutil.CheckTransferPeer(re, ops[0], operator.OpHotRegion, 2, 5) + operatorutil.CheckTransferPeer(re, ops[0], operator.OpHotRegion, 2, 5) clearPendingInfluence(hb.(*hotScheduler)) } @@ -2110,7 +2308,7 @@ func TestHotWriteLeaderScheduleWithPriority(t *testing.T) { cancel, _, tc, oc := prepareSchedulersTest() defer cancel() - hb, err := schedule.CreateScheduler(statistics.Write.String(), oc, storage.NewStorageWithMemoryBackend(), nil) + hb, err := CreateScheduler(statistics.Write.String(), oc, storage.NewStorageWithMemoryBackend(), nil) re.NoError(err) hb.(*hotScheduler).conf.SetDstToleranceRatio(1) hb.(*hotScheduler).conf.SetSrcToleranceRatio(1) @@ -2139,11 +2337,11 @@ func TestHotWriteLeaderScheduleWithPriority(t *testing.T) { hb.(*hotScheduler).conf.WriteLeaderPriorities = []string{statistics.KeyPriority, statistics.BytePriority} ops, _ := hb.Schedule(tc, false) re.Len(ops, 1) - testutil.CheckTransferLeader(re, ops[0], operator.OpHotRegion, 1, 2) + operatorutil.CheckTransferLeader(re, ops[0], operator.OpHotRegion, 1, 2) hb.(*hotScheduler).conf.WriteLeaderPriorities = []string{statistics.BytePriority, statistics.KeyPriority} ops, _ = hb.Schedule(tc, false) re.Len(ops, 1) - testutil.CheckTransferLeader(re, ops[0], operator.OpHotRegion, 1, 3) + operatorutil.CheckTransferLeader(re, ops[0], operator.OpHotRegion, 1, 3) } func TestCompatibility(t *testing.T) { @@ -2153,7 +2351,7 @@ func TestCompatibility(t *testing.T) { cancel, _, tc, oc := prepareSchedulersTest() defer cancel() - hb, err := schedule.CreateScheduler(statistics.Write.String(), oc, storage.NewStorageWithMemoryBackend(), nil) + hb, err := CreateScheduler(statistics.Write.String(), oc, storage.NewStorageWithMemoryBackend(), nil) re.NoError(err) // default checkPriority(re, hb.(*hotScheduler), tc, [3][2]int{ @@ -2221,7 +2419,7 @@ func TestCompatibilityConfig(t *testing.T) { defer cancel() // From new or 3.x cluster, it will use new config - hb, err := schedule.CreateScheduler(HotRegionType, oc, storage.NewStorageWithMemoryBackend(), schedule.ConfigSliceDecoder("hot-region", nil)) + hb, err := CreateScheduler(HotRegionType, oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder("hot-region", nil)) re.NoError(err) checkPriority(re, hb.(*hotScheduler), tc, [3][2]int{ {statistics.QueryDim, statistics.ByteDim}, @@ -2230,8 +2428,8 @@ func TestCompatibilityConfig(t *testing.T) { }) // Config file is not currently supported - hb, err = schedule.CreateScheduler(HotRegionType, oc, storage.NewStorageWithMemoryBackend(), - schedule.ConfigSliceDecoder("hot-region", []string{"read-priorities=byte,query"})) + hb, err = CreateScheduler(HotRegionType, oc, storage.NewStorageWithMemoryBackend(), + ConfigSliceDecoder("hot-region", []string{"read-priorities=byte,query"})) re.NoError(err) checkPriority(re, hb.(*hotScheduler), tc, [3][2]int{ {statistics.QueryDim, statistics.ByteDim}, @@ -2242,7 +2440,7 @@ func TestCompatibilityConfig(t *testing.T) { // from 4.0 or 5.0 or 5.1 cluster var data []byte storage := storage.NewStorageWithMemoryBackend() - data, err = schedule.EncodeConfig(map[string]interface{}{ + data, err = EncodeConfig(map[string]interface{}{ "min-hot-byte-rate": 100, "min-hot-key-rate": 10, "max-zombie-rounds": 3, @@ -2258,7 +2456,7 @@ func TestCompatibilityConfig(t *testing.T) { re.NoError(err) err = storage.SaveScheduleConfig(HotRegionName, data) re.NoError(err) - hb, err = schedule.CreateScheduler(HotRegionType, oc, storage, schedule.ConfigJSONDecoder(data)) + hb, err = CreateScheduler(HotRegionType, oc, storage, ConfigJSONDecoder(data)) re.NoError(err) checkPriority(re, hb.(*hotScheduler), tc, [3][2]int{ {statistics.ByteDim, statistics.KeyDim}, @@ -2270,11 +2468,11 @@ func TestCompatibilityConfig(t *testing.T) { cfg := initHotRegionScheduleConfig() cfg.ReadPriorities = []string{"key", "query"} cfg.WriteLeaderPriorities = []string{"query", "key"} - data, err = schedule.EncodeConfig(cfg) + data, err = EncodeConfig(cfg) re.NoError(err) err = storage.SaveScheduleConfig(HotRegionName, data) re.NoError(err) - hb, err = schedule.CreateScheduler(HotRegionType, oc, storage, schedule.ConfigJSONDecoder(data)) + hb, err = CreateScheduler(HotRegionType, oc, storage, ConfigJSONDecoder(data)) re.NoError(err) checkPriority(re, hb.(*hotScheduler), tc, [3][2]int{ {statistics.KeyDim, statistics.QueryDim}, @@ -2367,6 +2565,14 @@ func TestConfigValidation(t *testing.T) { hc.ForbidRWType = "test" err = hc.valid() re.Error(err) + + hc.SplitThresholds = 0 + err = hc.valid() + re.Error(err) + + hc.SplitThresholds = 1.1 + err = hc.valid() + re.Error(err) } type maxZombieDurTestCase struct { @@ -2380,7 +2586,7 @@ func TestMaxZombieDuration(t *testing.T) { re := require.New(t) cancel, _, _, oc := prepareSchedulersTest() defer cancel() - hb, err := schedule.CreateScheduler(HotRegionType, oc, storage.NewStorageWithMemoryBackend(), schedule.ConfigSliceDecoder("hot-region", nil)) + hb, err := CreateScheduler(HotRegionType, oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder("hot-region", nil)) re.NoError(err) maxZombieDur := hb.(*hotScheduler).conf.getValidConf().MaxZombieRounds testCases := []maxZombieDurTestCase{ @@ -2433,7 +2639,7 @@ func TestExpect(t *testing.T) { re := require.New(t) cancel, _, _, oc := prepareSchedulersTest() defer cancel() - hb, err := schedule.CreateScheduler(HotRegionType, oc, storage.NewStorageWithMemoryBackend(), schedule.ConfigSliceDecoder("hot-region", nil)) + hb, err := CreateScheduler(HotRegionType, oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder("hot-region", nil)) re.NoError(err) testCases := []struct { initFunc func(*balanceSolver) @@ -2450,10 +2656,12 @@ func TestExpect(t *testing.T) { initFunc: (*balanceSolver).pickCheckPolicyV1, strict: true, // all of load: &statistics.StoreLoad{ // all dims are higher than expect, allow schedule - Loads: []float64{2.0, 2.0, 2.0}, + Loads: []float64{2.0, 2.0, 2.0}, + HistoryLoads: [][]float64{{2.0, 2.0}, {2.0, 2.0}, {2.0, 2.0}}, }, expect: &statistics.StoreLoad{ - Loads: []float64{1.0, 1.0, 1.0}, + Loads: []float64{1.0, 1.0, 1.0}, + HistoryLoads: [][]float64{{1.0, 1.0}, {1.0, 1.0}, {1.0, 1.0}}, }, isSrc: true, allow: true, @@ -2462,10 +2670,12 @@ func TestExpect(t *testing.T) { initFunc: (*balanceSolver).pickCheckPolicyV1, strict: true, // all of load: &statistics.StoreLoad{ // all dims are higher than expect, but lower than expect*toleranceRatio, not allow schedule - Loads: []float64{2.0, 2.0, 2.0}, + Loads: []float64{2.0, 2.0, 2.0}, + HistoryLoads: [][]float64{{2.0, 2.0}, {2.0, 2.0}, {2.0, 2.0}}, }, expect: &statistics.StoreLoad{ - Loads: []float64{1.0, 1.0, 1.0}, + Loads: []float64{1.0, 1.0, 1.0}, + HistoryLoads: [][]float64{{1.0, 1.0}, {1.0, 1.0}, {1.0, 1.0}}, }, isSrc: true, toleranceRatio: 2.2, @@ -2475,10 +2685,12 @@ func TestExpect(t *testing.T) { initFunc: (*balanceSolver).pickCheckPolicyV1, strict: true, // all of load: &statistics.StoreLoad{ // only queryDim is lower, but the dim is no selected, allow schedule - Loads: []float64{2.0, 2.0, 1.0}, + Loads: []float64{2.0, 2.0, 1.0}, + HistoryLoads: [][]float64{{2.0, 2.0}, {2.0, 2.0}, {1.0, 1.0}}, }, expect: &statistics.StoreLoad{ - Loads: []float64{1.0, 1.0, 1.0}, + Loads: []float64{1.0, 1.0, 1.0}, + HistoryLoads: [][]float64{{1.0, 1.0}, {1.0, 1.0}, {1.0, 1.0}}, }, isSrc: true, allow: true, @@ -2487,10 +2699,12 @@ func TestExpect(t *testing.T) { initFunc: (*balanceSolver).pickCheckPolicyV1, strict: true, // all of load: &statistics.StoreLoad{ // only keyDim is lower, and the dim is selected, not allow schedule - Loads: []float64{2.0, 1.0, 2.0}, + Loads: []float64{2.0, 1.0, 2.0}, + HistoryLoads: [][]float64{{2.0, 2.0}, {1.0, 1.0}, {1.0, 1.0}}, }, expect: &statistics.StoreLoad{ - Loads: []float64{1.0, 1.0, 1.0}, + Loads: []float64{1.0, 1.0, 1.0}, + HistoryLoads: [][]float64{{1.0, 1.0}, {1.0, 1.0}, {1.0, 1.0}}, }, isSrc: true, allow: false, @@ -2499,10 +2713,12 @@ func TestExpect(t *testing.T) { initFunc: (*balanceSolver).pickCheckPolicyV1, strict: false, // first only load: &statistics.StoreLoad{ // keyDim is higher, and the dim is selected, allow schedule - Loads: []float64{1.0, 2.0, 1.0}, + Loads: []float64{1.0, 2.0, 1.0}, + HistoryLoads: [][]float64{{1.0, 1.0}, {2.0, 2.0}, {1.0, 1.0}}, }, expect: &statistics.StoreLoad{ - Loads: []float64{1.0, 1.0, 1.0}, + Loads: []float64{1.0, 1.0, 1.0}, + HistoryLoads: [][]float64{{1.0, 1.0}, {1.0, 1.0}, {1.0, 1.0}}, }, isSrc: true, allow: true, @@ -2511,10 +2727,12 @@ func TestExpect(t *testing.T) { initFunc: (*balanceSolver).pickCheckPolicyV1, strict: false, // first only load: &statistics.StoreLoad{ // although byteDim is higher, the dim is not first, not allow schedule - Loads: []float64{2.0, 1.0, 1.0}, + Loads: []float64{2.0, 1.0, 1.0}, + HistoryLoads: [][]float64{{2.0, 1.0}, {1.0, 1.0}, {1.0, 1.0}}, }, expect: &statistics.StoreLoad{ - Loads: []float64{1.0, 1.0, 1.0}, + Loads: []float64{1.0, 1.0, 1.0}, + HistoryLoads: [][]float64{{1.0, 1.0}, {1.0, 1.0}, {1.0, 1.0}}, }, isSrc: true, allow: false, @@ -2523,10 +2741,12 @@ func TestExpect(t *testing.T) { initFunc: (*balanceSolver).pickCheckPolicyV1, strict: false, // first only load: &statistics.StoreLoad{ // although queryDim is higher, the dim is no selected, not allow schedule - Loads: []float64{1.0, 1.0, 2.0}, + Loads: []float64{1.0, 1.0, 2.0}, + HistoryLoads: [][]float64{{1.0, 1.0}, {1.0, 1.0}, {2.0, 2.0}}, }, expect: &statistics.StoreLoad{ - Loads: []float64{1.0, 1.0, 1.0}, + Loads: []float64{1.0, 1.0, 1.0}, + HistoryLoads: [][]float64{{1.0, 1.0}, {1.0, 1.0}, {1.0, 1.0}}, }, isSrc: true, allow: false, @@ -2535,10 +2755,12 @@ func TestExpect(t *testing.T) { initFunc: (*balanceSolver).pickCheckPolicyV1, strict: false, // first only load: &statistics.StoreLoad{ // all dims are lower than expect, not allow schedule - Loads: []float64{1.0, 1.0, 1.0}, + Loads: []float64{1.0, 1.0, 1.0}, + HistoryLoads: [][]float64{{1.0, 1.0}, {1.0, 1.0}, {1.0, 1.0}}, }, expect: &statistics.StoreLoad{ - Loads: []float64{2.0, 2.0, 2.0}, + Loads: []float64{2.0, 2.0, 2.0}, + HistoryLoads: [][]float64{{2.0, 2.0}, {2.0, 2.0}, {2.0, 2.0}}, }, isSrc: true, allow: false, @@ -2548,10 +2770,12 @@ func TestExpect(t *testing.T) { strict: true, rs: writeLeader, load: &statistics.StoreLoad{ // only keyDim is higher, but write leader only consider the first priority - Loads: []float64{1.0, 2.0, 1.0}, + Loads: []float64{1.0, 2.0, 1.0}, + HistoryLoads: [][]float64{{1.0, 1.0}, {2.0, 2.0}, {2.0, 2.0}}, }, expect: &statistics.StoreLoad{ - Loads: []float64{1.0, 1.0, 1.0}, + Loads: []float64{1.0, 1.0, 1.0}, + HistoryLoads: [][]float64{{1.0, 1.0}, {1.0, 1.0}, {1.0, 1.0}}, }, isSrc: true, allow: true, @@ -2561,10 +2785,27 @@ func TestExpect(t *testing.T) { strict: true, rs: writeLeader, load: &statistics.StoreLoad{ // although byteDim is higher, the dim is not first, not allow schedule - Loads: []float64{2.0, 1.0, 1.0}, + Loads: []float64{2.0, 1.0, 1.0}, + HistoryLoads: [][]float64{{2.0, 2.0}, {1.0, 1.0}, {2.0, 2.0}}, + }, + expect: &statistics.StoreLoad{ + Loads: []float64{1.0, 1.0, 1.0}, + HistoryLoads: [][]float64{{1.0, 1.0}, {1.0, 1.0}, {1.0, 1.0}}, + }, + isSrc: true, + allow: false, + }, + { + initFunc: (*balanceSolver).pickCheckPolicyV1, + strict: true, + rs: writeLeader, + load: &statistics.StoreLoad{ // history loads is not higher than the expected. + Loads: []float64{2.0, 1.0, 1.0}, + HistoryLoads: [][]float64{{2.0, 2.0}, {1.0, 2.0}, {1.0, 2.0}}, }, expect: &statistics.StoreLoad{ - Loads: []float64{1.0, 1.0, 1.0}, + Loads: []float64{1.0, 1.0, 1.0}, + HistoryLoads: [][]float64{{1.0, 2.0}, {1.0, 2.0}, {1.0, 2.0}}, }, isSrc: true, allow: false, @@ -2574,10 +2815,12 @@ func TestExpect(t *testing.T) { initFunc: (*balanceSolver).pickCheckPolicyV2, strict: false, // any of load: &statistics.StoreLoad{ // keyDim is higher, and the dim is selected, allow schedule - Loads: []float64{1.0, 2.0, 1.0}, + Loads: []float64{1.0, 2.0, 1.0}, + HistoryLoads: [][]float64{{1.0, 1.0}, {2.0, 2.0}, {1.0, 1.0}}, }, expect: &statistics.StoreLoad{ - Loads: []float64{1.0, 1.0, 1.0}, + Loads: []float64{1.0, 1.0, 1.0}, + HistoryLoads: [][]float64{{1.0, 1.0}, {1.0, 1.0}, {1.0, 1.0}}, }, isSrc: true, allow: true, @@ -2586,10 +2829,12 @@ func TestExpect(t *testing.T) { initFunc: (*balanceSolver).pickCheckPolicyV2, strict: false, // any of load: &statistics.StoreLoad{ // byteDim is higher, and the dim is selected, allow schedule - Loads: []float64{2.0, 1.0, 1.0}, + Loads: []float64{2.0, 1.0, 1.0}, + HistoryLoads: [][]float64{{2.0, 2.0}, {1.0, 1.0}, {1.0, 1.0}}, }, expect: &statistics.StoreLoad{ - Loads: []float64{1.0, 1.0, 1.0}, + Loads: []float64{1.0, 1.0, 1.0}, + HistoryLoads: [][]float64{{1.0, 1.0}, {1.0, 1.0}, {1.0, 1.0}}, }, isSrc: true, allow: true, @@ -2598,10 +2843,12 @@ func TestExpect(t *testing.T) { initFunc: (*balanceSolver).pickCheckPolicyV2, strict: false, // any of load: &statistics.StoreLoad{ // although queryDim is higher, the dim is no selected, not allow schedule - Loads: []float64{1.0, 1.0, 2.0}, + Loads: []float64{1.0, 1.0, 2.0}, + HistoryLoads: [][]float64{{1.0, 1.0}, {1.0, 1.0}, {2.0, 2.0}}, }, expect: &statistics.StoreLoad{ - Loads: []float64{1.0, 1.0, 1.0}, + Loads: []float64{1.0, 1.0, 1.0}, + HistoryLoads: [][]float64{{1.0, 1.0}, {1.0, 1.0}, {1.0, 1.0}}, }, isSrc: true, allow: false, @@ -2610,10 +2857,12 @@ func TestExpect(t *testing.T) { initFunc: (*balanceSolver).pickCheckPolicyV2, strict: false, // any of load: &statistics.StoreLoad{ // all dims are lower than expect, not allow schedule - Loads: []float64{1.0, 1.0, 1.0}, + Loads: []float64{1.0, 1.0, 1.0}, + HistoryLoads: [][]float64{{1.0, 1.0}, {1.0, 1.0}, {1.0, 1.0}}, }, expect: &statistics.StoreLoad{ - Loads: []float64{2.0, 2.0, 2.0}, + Loads: []float64{2.0, 2.0, 2.0}, + HistoryLoads: [][]float64{{2.0, 2.0}, {2.0, 2.0}, {2.0, 2.0}}, }, isSrc: true, allow: false, @@ -2623,10 +2872,12 @@ func TestExpect(t *testing.T) { strict: true, rs: writeLeader, load: &statistics.StoreLoad{ // only keyDim is higher, but write leader only consider the first priority - Loads: []float64{1.0, 2.0, 1.0}, + Loads: []float64{1.0, 2.0, 1.0}, + HistoryLoads: [][]float64{{1.0, 1.0}, {2.0, 2.0}, {1.0, 1.0}}, }, expect: &statistics.StoreLoad{ - Loads: []float64{1.0, 1.0, 1.0}, + Loads: []float64{1.0, 1.0, 1.0}, + HistoryLoads: [][]float64{{1.0, 1.0}, {1.0, 1.0}, {1.0, 1.0}}, }, isSrc: true, allow: true, @@ -2636,10 +2887,12 @@ func TestExpect(t *testing.T) { strict: true, rs: writeLeader, load: &statistics.StoreLoad{ // although byteDim is higher, the dim is not first, not allow schedule - Loads: []float64{2.0, 1.0, 1.0}, + Loads: []float64{2.0, 1.0, 1.0}, + HistoryLoads: [][]float64{{2.0, 2.0}, {1.0, 1.0}, {1.0, 1.0}}, }, expect: &statistics.StoreLoad{ - Loads: []float64{1.0, 1.0, 1.0}, + Loads: []float64{1.0, 1.0, 1.0}, + HistoryLoads: [][]float64{{1.0, 1.0}, {1.0, 1.0}, {1.0, 1.0}}, }, isSrc: true, allow: false, @@ -2651,8 +2904,16 @@ func TestExpect(t *testing.T) { for i, v := range src.Loads { dst[i] = 3.0 - v } + historyLoads := make([][]float64, len(src.HistoryLoads)) + for i, dim := range src.HistoryLoads { + historyLoads[i] = make([]float64, len(dim)) + for j, load := range dim { + historyLoads[i][j] = 3.0 - load + } + } return &statistics.StoreLoad{ - Loads: dst, + Loads: dst, + HistoryLoads: historyLoads, } } @@ -2671,6 +2932,8 @@ func TestExpect(t *testing.T) { testCase.initFunc(bs) re.Equal(testCase.allow, bs.checkSrcByPriorityAndTolerance(testCase.load, testCase.expect, toleranceRatio)) re.Equal(testCase.allow, bs.checkDstByPriorityAndTolerance(srcToDst(testCase.load), srcToDst(testCase.expect), toleranceRatio)) + re.Equal(testCase.allow, bs.checkSrcHistoryLoadsByPriorityAndTolerance(testCase.load, testCase.expect, toleranceRatio)) + re.Equal(testCase.allow, bs.checkDstHistoryLoadsByPriorityAndTolerance(srcToDst(testCase.load), srcToDst(testCase.expect), toleranceRatio)) } } @@ -2679,9 +2942,52 @@ func TestEncodeConfig(t *testing.T) { re := require.New(t) cancel, _, _, oc := prepareSchedulersTest() defer cancel() - sche, err := schedule.CreateScheduler(HotRegionType, oc, storage.NewStorageWithMemoryBackend(), schedule.ConfigJSONDecoder([]byte("null"))) + sche, err := CreateScheduler(HotRegionType, oc, storage.NewStorageWithMemoryBackend(), ConfigJSONDecoder([]byte("null"))) re.NoError(err) data, err := sche.EncodeConfig() re.NoError(err) re.NotEqual("null", string(data)) } + +func TestBucketFirstStat(t *testing.T) { + re := require.New(t) + testdata := []struct { + firstPriority int + secondPriority int + rwTy statistics.RWType + expect statistics.RegionStatKind + }{ + { + firstPriority: statistics.KeyDim, + secondPriority: statistics.ByteDim, + rwTy: statistics.Write, + expect: statistics.RegionWriteKeys, + }, + { + firstPriority: statistics.QueryDim, + secondPriority: statistics.ByteDim, + rwTy: statistics.Write, + expect: statistics.RegionWriteBytes, + }, + { + firstPriority: statistics.KeyDim, + secondPriority: statistics.ByteDim, + rwTy: statistics.Read, + expect: statistics.RegionReadKeys, + }, + { + firstPriority: statistics.QueryDim, + secondPriority: statistics.ByteDim, + rwTy: statistics.Read, + expect: statistics.RegionReadBytes, + }, + } + for _, data := range testdata { + bs := &balanceSolver{ + firstPriority: data.firstPriority, + secondPriority: data.secondPriority, + rwTy: data.rwTy, + } + re.Equal(data.expect, bs.bucketFirstStat()) + } +} diff --git a/pkg/schedule/schedulers/hot_region_v2.go b/pkg/schedule/schedulers/hot_region_v2.go index 48ede517c8b..c81225d959e 100644 --- a/pkg/schedule/schedulers/hot_region_v2.go +++ b/pkg/schedule/schedulers/hot_region_v2.go @@ -86,8 +86,10 @@ func (bs *balanceSolver) pickCheckPolicyV2() { switch { case bs.resourceTy == writeLeader: bs.checkByPriorityAndTolerance = bs.checkByPriorityAndToleranceFirstOnly + bs.checkHistoryLoadsByPriority = bs.checkHistoryLoadsByPriorityAndToleranceFirstOnly default: bs.checkByPriorityAndTolerance = bs.checkByPriorityAndToleranceAnyOf + bs.checkHistoryLoadsByPriority = bs.checkHistoryByPriorityAndToleranceAnyOf } } @@ -204,11 +206,17 @@ func (bs *balanceSolver) getScoreByPriorities(dim int, rs *rankV2Ratios) int { srcPendingRate, dstPendingRate := bs.cur.getPendingLoad(dim) peersRate := bs.cur.getPeersRateFromCache(dim) highRate, lowRate := srcRate, dstRate + topnHotPeer := bs.nthHotPeer[bs.cur.srcStore.GetID()][dim] reverse := false if srcRate < dstRate { highRate, lowRate = dstRate, srcRate peersRate = -peersRate reverse = true + topnHotPeer = bs.nthHotPeer[bs.cur.dstStore.GetID()][dim] + } + topnRate := math.MaxFloat64 + if topnHotPeer != nil { + topnRate = topnHotPeer.GetLoad(dim) } if highRate*rs.balancedCheckRatio <= lowRate { @@ -260,6 +268,7 @@ func (bs *balanceSolver) getScoreByPriorities(dim int, rs *rankV2Ratios) int { // maxBetterRate may be less than minBetterRate, in which case a positive fraction cannot be produced. minNotWorsenedRate = -bs.getMinRate(dim) minBetterRate = math.Min(minBalancedRate*rs.perceivedRatio, lowRate*rs.minHotRatio) + minBetterRate = math.Min(minBetterRate, topnRate) maxBetterRate = maxBalancedRate + (highRate-lowRate-minBetterRate-maxBalancedRate)*rs.perceivedRatio maxNotWorsenedRate = maxBalancedRate + (highRate-lowRate-minNotWorsenedRate-maxBalancedRate)*rs.perceivedRatio } @@ -290,7 +299,7 @@ func (bs *balanceSolver) getScoreByPriorities(dim int, rs *rankV2Ratios) int { // betterThan checks if `bs.cur` is a better solution than `old`. func (bs *balanceSolver) betterThanV2(old *solution) bool { - if old == nil { + if old == nil || bs.cur.progressiveRank <= splitProgressiveRank { return true } if bs.cur.progressiveRank != old.progressiveRank { diff --git a/pkg/schedule/schedulers/hot_region_v2_test.go b/pkg/schedule/schedulers/hot_region_v2_test.go index 681de0a139b..87d4950b611 100644 --- a/pkg/schedule/schedulers/hot_region_v2_test.go +++ b/pkg/schedule/schedulers/hot_region_v2_test.go @@ -19,11 +19,11 @@ import ( "github.com/docker/go-units" "github.com/stretchr/testify/require" - "github.com/tikv/pd/pkg/schedule" + "github.com/tikv/pd/pkg/mock/mockcluster" "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/statistics" "github.com/tikv/pd/pkg/storage" - "github.com/tikv/pd/pkg/utils/testutil" + "github.com/tikv/pd/pkg/utils/operatorutil" "github.com/tikv/pd/pkg/versioninfo" ) @@ -33,8 +33,9 @@ func TestHotWriteRegionScheduleWithRevertRegionsDimSecond(t *testing.T) { cancel, _, tc, oc := prepareSchedulersTest() defer cancel() statistics.Denoising = false + statisticsInterval = 0 - sche, err := schedule.CreateScheduler(statistics.Write.String(), oc, storage.NewStorageWithMemoryBackend(), nil) + sche, err := CreateScheduler(statistics.Write.String(), oc, storage.NewStorageWithMemoryBackend(), nil, nil) re.NoError(err) hb := sche.(*hotScheduler) hb.conf.SetDstToleranceRatio(0.0) @@ -72,8 +73,8 @@ func TestHotWriteRegionScheduleWithRevertRegionsDimSecond(t *testing.T) { ops, _ = hb.Schedule(tc, false) /* The revert region is currently disabled for the -1 case. re.Len(ops, 2) - testutil.CheckTransferPeer(re, ops[0], operator.OpHotRegion, 2, 5) - testutil.CheckTransferPeer(re, ops[1], operator.OpHotRegion, 5, 2) + operatorutil.CheckTransferPeer(re, ops[0], operator.OpHotRegion, 2, 5) + operatorutil.CheckTransferPeer(re, ops[1], operator.OpHotRegion, 5, 2) */ re.Empty(ops) re.True(hb.searchRevertRegions[writePeer]) @@ -84,7 +85,7 @@ func TestHotWriteRegionScheduleWithRevertRegionsDimSecond(t *testing.T) { }) ops, _ = hb.Schedule(tc, false) re.Len(ops, 1) - testutil.CheckTransferPeer(re, ops[0], operator.OpHotRegion, 2, 5) + operatorutil.CheckTransferPeer(re, ops[0], operator.OpHotRegion, 2, 5) re.False(hb.searchRevertRegions[writePeer]) clearPendingInfluence(hb) } @@ -96,7 +97,7 @@ func TestHotWriteRegionScheduleWithRevertRegionsDimFirst(t *testing.T) { defer cancel() statistics.Denoising = false - sche, err := schedule.CreateScheduler(statistics.Write.String(), oc, storage.NewStorageWithMemoryBackend(), nil) + sche, err := CreateScheduler(statistics.Write.String(), oc, storage.NewStorageWithMemoryBackend(), nil, nil) re.NoError(err) hb := sche.(*hotScheduler) hb.conf.SetDstToleranceRatio(0.0) @@ -123,7 +124,7 @@ func TestHotWriteRegionScheduleWithRevertRegionsDimFirst(t *testing.T) { // One operator can be generated when RankFormulaVersion == "v1". ops, _ := hb.Schedule(tc, false) re.Len(ops, 1) - testutil.CheckTransferPeer(re, ops[0], operator.OpHotRegion, 2, 5) + operatorutil.CheckTransferPeer(re, ops[0], operator.OpHotRegion, 2, 5) re.False(hb.searchRevertRegions[writePeer]) clearPendingInfluence(hb) @@ -136,8 +137,8 @@ func TestHotWriteRegionScheduleWithRevertRegionsDimFirst(t *testing.T) { // Two operators can be generated when RankFormulaVersion == "v2". ops, _ = hb.Schedule(tc, false) re.Len(ops, 2) - testutil.CheckTransferPeer(re, ops[0], operator.OpHotRegion, 2, 5) - testutil.CheckTransferPeer(re, ops[1], operator.OpHotRegion, 5, 2) + operatorutil.CheckTransferPeer(re, ops[0], operator.OpHotRegion, 2, 5) + operatorutil.CheckTransferPeer(re, ops[1], operator.OpHotRegion, 5, 2) re.True(hb.searchRevertRegions[writePeer]) clearPendingInfluence(hb) } @@ -146,10 +147,11 @@ func TestHotWriteRegionScheduleWithRevertRegionsDimFirstOnly(t *testing.T) { // This is a test that searchRevertRegions finds a solution of rank -2. re := require.New(t) statistics.Denoising = false + statisticsInterval = 0 cancel, _, tc, oc := prepareSchedulersTest() defer cancel() - sche, err := schedule.CreateScheduler(statistics.Write.String(), oc, storage.NewStorageWithMemoryBackend(), nil) + sche, err := CreateScheduler(statistics.Write.String(), oc, storage.NewStorageWithMemoryBackend(), nil, nil) re.NoError(err) hb := sche.(*hotScheduler) hb.conf.SetDstToleranceRatio(0.0) @@ -176,7 +178,7 @@ func TestHotWriteRegionScheduleWithRevertRegionsDimFirstOnly(t *testing.T) { // One operator can be generated when RankFormulaVersion == "v1". ops, _ := hb.Schedule(tc, false) re.Len(ops, 1) - testutil.CheckTransferPeer(re, ops[0], operator.OpHotRegion, 2, 5) + operatorutil.CheckTransferPeer(re, ops[0], operator.OpHotRegion, 2, 5) re.False(hb.searchRevertRegions[writePeer]) clearPendingInfluence(hb) @@ -189,7 +191,7 @@ func TestHotWriteRegionScheduleWithRevertRegionsDimFirstOnly(t *testing.T) { // There is still the solution with one operator after that. ops, _ = hb.Schedule(tc, false) re.Len(ops, 1) - testutil.CheckTransferPeer(re, ops[0], operator.OpHotRegion, 2, 5) + operatorutil.CheckTransferPeer(re, ops[0], operator.OpHotRegion, 2, 5) re.True(hb.searchRevertRegions[writePeer]) clearPendingInfluence(hb) // Two operators can be generated when there is a better solution @@ -198,8 +200,8 @@ func TestHotWriteRegionScheduleWithRevertRegionsDimFirstOnly(t *testing.T) { }) ops, _ = hb.Schedule(tc, false) re.Len(ops, 2) - testutil.CheckTransferPeer(re, ops[0], operator.OpHotRegion, 2, 5) - testutil.CheckTransferPeer(re, ops[1], operator.OpHotRegion, 5, 2) + operatorutil.CheckTransferPeer(re, ops[0], operator.OpHotRegion, 2, 5) + operatorutil.CheckTransferPeer(re, ops[1], operator.OpHotRegion, 5, 2) re.True(hb.searchRevertRegions[writePeer]) clearPendingInfluence(hb) } @@ -208,10 +210,11 @@ func TestHotReadRegionScheduleWithRevertRegionsDimSecond(t *testing.T) { // This is a test that searchRevertRegions finds a solution of rank -1. re := require.New(t) statistics.Denoising = false + statisticsInterval = 0 cancel, _, tc, oc := prepareSchedulersTest() defer cancel() - sche, err := schedule.CreateScheduler(statistics.Read.String(), oc, storage.NewStorageWithMemoryBackend(), nil) + sche, err := CreateScheduler(statistics.Read.String(), oc, storage.NewStorageWithMemoryBackend(), nil, nil) re.NoError(err) hb := sche.(*hotScheduler) hb.conf.SetDstToleranceRatio(0.0) @@ -249,8 +252,8 @@ func TestHotReadRegionScheduleWithRevertRegionsDimSecond(t *testing.T) { ops, _ = hb.Schedule(tc, false) /* The revert region is currently disabled for the -1 case. re.Len(ops, 2) - testutil.CheckTransferLeader(re, ops[0], operator.OpHotRegion, 2, 5) - testutil.CheckTransferLeader(re, ops[1], operator.OpHotRegion, 5, 2) + operatorutil.CheckTransferLeader(re, ops[0], operator.OpHotRegion, 2, 5) + operatorutil.CheckTransferLeader(re, ops[1], operator.OpHotRegion, 5, 2) */ re.Empty(ops) re.True(hb.searchRevertRegions[readLeader]) @@ -261,7 +264,7 @@ func TestHotReadRegionScheduleWithRevertRegionsDimSecond(t *testing.T) { }) ops, _ = hb.Schedule(tc, false) re.Len(ops, 1) - testutil.CheckTransferLeader(re, ops[0], operator.OpHotRegion, 2, 5) + operatorutil.CheckTransferLeader(re, ops[0], operator.OpHotRegion, 2, 5) re.False(hb.searchRevertRegions[readLeader]) clearPendingInfluence(hb) } @@ -269,10 +272,11 @@ func TestHotReadRegionScheduleWithRevertRegionsDimSecond(t *testing.T) { func TestSkipUniformStore(t *testing.T) { re := require.New(t) statistics.Denoising = false + statisticsInterval = 0 cancel, _, tc, oc := prepareSchedulersTest() defer cancel() - hb, err := schedule.CreateScheduler(statistics.Read.String(), oc, storage.NewStorageWithMemoryBackend(), nil) + hb, err := CreateScheduler(statistics.Read.String(), oc, storage.NewStorageWithMemoryBackend(), nil, nil) re.NoError(err) hb.(*hotScheduler).conf.SetSrcToleranceRatio(1) hb.(*hotScheduler).conf.SetDstToleranceRatio(1) @@ -296,7 +300,7 @@ func TestSkipUniformStore(t *testing.T) { stddevThreshold = 0.0 ops, _ := hb.Schedule(tc, false) re.Len(ops, 1) - testutil.CheckTransferLeader(re, ops[0], operator.OpHotRegion, 1, 2) + operatorutil.CheckTransferLeader(re, ops[0], operator.OpHotRegion, 1, 2) clearPendingInfluence(hb.(*hotScheduler)) // when there is uniform store filter, not schedule stddevThreshold = 0.1 @@ -316,13 +320,13 @@ func TestSkipUniformStore(t *testing.T) { stddevThreshold = 0.0 ops, _ = hb.Schedule(tc, false) re.Len(ops, 1) - testutil.CheckTransferLeader(re, ops[0], operator.OpHotRegion, 1, 2) + operatorutil.CheckTransferLeader(re, ops[0], operator.OpHotRegion, 1, 2) clearPendingInfluence(hb.(*hotScheduler)) // when there is uniform store filter, schedule the second dim, which is no uniform stddevThreshold = 0.1 ops, _ = hb.Schedule(tc, false) re.Len(ops, 1) - testutil.CheckTransferLeader(re, ops[0], operator.OpHotRegion, 3, 2) + operatorutil.CheckTransferLeader(re, ops[0], operator.OpHotRegion, 3, 2) clearPendingInfluence(hb.(*hotScheduler)) // Case3: the second dim is enough uniform, we should schedule the first dim, although its rank is higher than the second dim @@ -337,12 +341,132 @@ func TestSkipUniformStore(t *testing.T) { stddevThreshold = 0.0 ops, _ = hb.Schedule(tc, false) re.Len(ops, 1) - testutil.CheckTransferLeader(re, ops[0], operator.OpHotRegion, 3, 2) + operatorutil.CheckTransferLeader(re, ops[0], operator.OpHotRegion, 3, 2) clearPendingInfluence(hb.(*hotScheduler)) // when there is uniform store filter, schedule the first dim, which is no uniform stddevThreshold = 0.1 ops, _ = hb.Schedule(tc, false) re.Len(ops, 1) - testutil.CheckTransferLeader(re, ops[0], operator.OpHotRegion, 3, 2) + operatorutil.CheckTransferLeader(re, ops[0], operator.OpHotRegion, 3, 2) clearPendingInfluence(hb.(*hotScheduler)) } + +func TestHotReadRegionScheduleWithSmallHotRegion(t *testing.T) { + // This is a test that we can schedule small hot region, + // which is smaller than 20% of diff or 2% of low node. (#6645) + // 20% is from `firstPriorityPerceivedRatio`, 2% is from `firstPriorityMinHotRatio`. + // The byte of high node is 2000MB/s, the low node is 200MB/s. + // The query of high node is 2000qps, the low node is 200qps. + // There are all small hot regions in the cluster, which are smaller than 20% of diff or 2% of low node. + re := require.New(t) + emptyFunc := func(*mockcluster.Cluster, *hotScheduler) {} + highLoad, lowLoad := uint64(2000), uint64(200) + bigHotRegionByte := uint64(float64(lowLoad) * firstPriorityMinHotRatio * 10 * units.MiB * statistics.ReadReportInterval) + bigHotRegionQuery := uint64(float64(lowLoad) * firstPriorityMinHotRatio * 10 * statistics.ReadReportInterval) + + // Case1: Before #6827, we only use minHotRatio, so cannot schedule small hot region in this case. + // Because 10000 is larger than the length of hotRegions, so `filterHotPeers` will skip the topn calculation. + origin := topnPosition + topnPosition = 10000 + ops := checkHotReadRegionScheduleWithSmallHotRegion(re, highLoad, lowLoad, emptyFunc) + re.Empty(ops) + topnPosition = origin + + // Case2: After #6827, we use top10 as the threshold of minHotPeer. + ops = checkHotReadRegionScheduleWithSmallHotRegion(re, highLoad, lowLoad, emptyFunc) + re.Len(ops, 1) + ops = checkHotReadRegionScheduleWithSmallHotRegion(re, lowLoad, highLoad, emptyFunc) + re.Len(ops, 0) + + // Case3: If there is larger hot region, we will schedule it. + hotRegionID := uint64(100) + ops = checkHotReadRegionScheduleWithSmallHotRegion(re, highLoad, lowLoad, func(tc *mockcluster.Cluster, _ *hotScheduler) { + tc.AddRegionWithReadInfo(hotRegionID, 1, bigHotRegionByte, 0, bigHotRegionQuery, statistics.ReadReportInterval, []uint64{2, 3}) + }) + re.Len(ops, 1) + re.Equal(hotRegionID, ops[0].RegionID()) + + // Case4: If there is larger hot region, but it need to cool down, we will schedule small hot region. + ops = checkHotReadRegionScheduleWithSmallHotRegion(re, highLoad, lowLoad, func(tc *mockcluster.Cluster, _ *hotScheduler) { + // just transfer leader + tc.AddRegionWithReadInfo(hotRegionID, 2, bigHotRegionByte, 0, bigHotRegionQuery, statistics.ReadReportInterval, []uint64{1, 3}) + tc.AddRegionWithReadInfo(hotRegionID, 1, bigHotRegionByte, 0, bigHotRegionQuery, statistics.ReadReportInterval, []uint64{2, 3}) + }) + re.Len(ops, 1) + re.NotEqual(hotRegionID, ops[0].RegionID()) + + // Case5: If there is larger hot region, but it is pending, we will schedule small hot region. + ops = checkHotReadRegionScheduleWithSmallHotRegion(re, highLoad, lowLoad, func(tc *mockcluster.Cluster, hb *hotScheduler) { + tc.AddRegionWithReadInfo(hotRegionID, 1, bigHotRegionByte, 0, bigHotRegionQuery, statistics.ReadReportInterval, []uint64{2, 3}) + hb.regionPendings[hotRegionID] = &pendingInfluence{} + }) + re.Len(ops, 1) + re.NotEqual(hotRegionID, ops[0].RegionID()) + + // Case5: If there are more than topnPosition hot regions, but them need to cool down, + // we will schedule large hot region rather than small hot region, so there is no operator. + topnPosition = 2 + ops = checkHotReadRegionScheduleWithSmallHotRegion(re, highLoad, lowLoad, func(tc *mockcluster.Cluster, _ *hotScheduler) { + // just transfer leader + tc.AddRegionWithReadInfo(hotRegionID, 2, bigHotRegionByte, 0, bigHotRegionQuery, statistics.ReadReportInterval, []uint64{1, 3}) + tc.AddRegionWithReadInfo(hotRegionID, 1, bigHotRegionByte, 0, bigHotRegionQuery, statistics.ReadReportInterval, []uint64{2, 3}) + // just transfer leader + tc.AddRegionWithReadInfo(hotRegionID+1, 2, bigHotRegionByte, 0, bigHotRegionQuery, statistics.ReadReportInterval, []uint64{1, 3}) + tc.AddRegionWithReadInfo(hotRegionID+1, 1, bigHotRegionByte, 0, bigHotRegionQuery, statistics.ReadReportInterval, []uint64{2, 3}) + }) + re.Len(ops, 0) + topnPosition = origin + + // Case6: If there are more than topnPosition hot regions, but them are pending, + // we will schedule large hot region rather than small hot region, so there is no operator. + topnPosition = 2 + ops = checkHotReadRegionScheduleWithSmallHotRegion(re, highLoad, lowLoad, func(tc *mockcluster.Cluster, hb *hotScheduler) { + tc.AddRegionWithReadInfo(hotRegionID, 1, bigHotRegionByte, 0, bigHotRegionQuery, statistics.ReadReportInterval, []uint64{2, 3}) + hb.regionPendings[hotRegionID] = &pendingInfluence{} + tc.AddRegionWithReadInfo(hotRegionID+1, 1, bigHotRegionByte, 0, bigHotRegionQuery, statistics.ReadReportInterval, []uint64{2, 3}) + hb.regionPendings[hotRegionID+1] = &pendingInfluence{} + }) + re.Len(ops, 0) + topnPosition = origin +} + +func checkHotReadRegionScheduleWithSmallHotRegion(re *require.Assertions, highLoad, lowLoad uint64, + addOtherRegions func(*mockcluster.Cluster, *hotScheduler)) []*operator.Operator { + cancel, _, tc, oc := prepareSchedulersTest() + defer cancel() + statistics.Denoising = false + sche, err := CreateScheduler(statistics.Read.String(), oc, storage.NewStorageWithMemoryBackend(), nil, nil) + re.NoError(err) + hb := sche.(*hotScheduler) + hb.conf.SetSrcToleranceRatio(1) + hb.conf.SetDstToleranceRatio(1) + hb.conf.SetRankFormulaVersion("v2") + hb.conf.ReadPriorities = []string{statistics.QueryPriority, statistics.BytePriority} + tc.SetHotRegionCacheHitsThreshold(0) + tc.AddRegionStore(1, 40) + tc.AddRegionStore(2, 10) + tc.AddRegionStore(3, 10) + + tc.UpdateStorageReadQuery(1, highLoad*statistics.StoreHeartBeatReportInterval) + tc.UpdateStorageReadQuery(2, lowLoad*statistics.StoreHeartBeatReportInterval) + tc.UpdateStorageReadQuery(3, (highLoad+lowLoad)/2*statistics.StoreHeartBeatReportInterval) + tc.UpdateStorageReadStats(1, highLoad*units.MiB*statistics.StoreHeartBeatReportInterval, 0) + tc.UpdateStorageReadStats(2, lowLoad*units.MiB*statistics.StoreHeartBeatReportInterval, 0) + tc.UpdateStorageReadStats(3, (highLoad+lowLoad)/2*units.MiB*statistics.StoreHeartBeatReportInterval, 0) + + smallHotPeerQuery := float64(lowLoad) * firstPriorityMinHotRatio * 0.9 // it's a small hot region than the firstPriorityMinHotRatio + smallHotPeerByte := float64(lowLoad) * secondPriorityMinHotRatio * 0.9 * units.MiB // it's a small hot region than the secondPriorityMinHotRatio + regions := make([]testRegionInfo, 0) + for i := 10; i < 50; i++ { + regions = append(regions, testRegionInfo{uint64(i), []uint64{1, 2, 3}, smallHotPeerByte, 0, smallHotPeerQuery}) + if i < 20 { + regions = append(regions, testRegionInfo{uint64(i), []uint64{2, 1, 3}, smallHotPeerByte, 0, smallHotPeerQuery}) + regions = append(regions, testRegionInfo{uint64(i), []uint64{3, 1, 2}, smallHotPeerByte, 0, smallHotPeerQuery}) + } + } + addRegionInfo(tc, statistics.Read, regions) + tc.SetHotRegionCacheHitsThreshold(1) + addOtherRegions(tc, hb) + ops, _ := hb.Schedule(tc, false) + return ops +} diff --git a/pkg/schedule/schedulers/init.go b/pkg/schedule/schedulers/init.go index 78a35549cd2..8170f86cd2a 100644 --- a/pkg/schedule/schedulers/init.go +++ b/pkg/schedule/schedulers/init.go @@ -21,7 +21,7 @@ import ( "github.com/tikv/pd/pkg/core" "github.com/tikv/pd/pkg/errs" - "github.com/tikv/pd/pkg/schedule" + "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/storage/endpoint" ) @@ -36,7 +36,7 @@ func Register() { func schedulersRegister() { // balance leader - schedule.RegisterSliceDecoderBuilder(BalanceLeaderType, func(args []string) schedule.ConfigDecoder { + RegisterSliceDecoderBuilder(BalanceLeaderType, func(args []string) ConfigDecoder { return func(v interface{}) error { conf, ok := v.(*balanceLeaderSchedulerConfig) if !ok { @@ -52,7 +52,7 @@ func schedulersRegister() { } }) - schedule.RegisterScheduler(BalanceLeaderType, func(opController *schedule.OperatorController, storage endpoint.ConfigStorage, decoder schedule.ConfigDecoder) (schedule.Scheduler, error) { + RegisterScheduler(BalanceLeaderType, func(opController *operator.Controller, storage endpoint.ConfigStorage, decoder ConfigDecoder, removeSchedulerCb ...func(string) error) (Scheduler, error) { conf := &balanceLeaderSchedulerConfig{storage: storage} if err := decoder(conf); err != nil { return nil, err @@ -64,7 +64,7 @@ func schedulersRegister() { }) // balance region - schedule.RegisterSliceDecoderBuilder(BalanceRegionType, func(args []string) schedule.ConfigDecoder { + RegisterSliceDecoderBuilder(BalanceRegionType, func(args []string) ConfigDecoder { return func(v interface{}) error { conf, ok := v.(*balanceRegionSchedulerConfig) if !ok { @@ -80,7 +80,7 @@ func schedulersRegister() { } }) - schedule.RegisterScheduler(BalanceRegionType, func(opController *schedule.OperatorController, storage endpoint.ConfigStorage, decoder schedule.ConfigDecoder) (schedule.Scheduler, error) { + RegisterScheduler(BalanceRegionType, func(opController *operator.Controller, storage endpoint.ConfigStorage, decoder ConfigDecoder, removeSchedulerCb ...func(string) error) (Scheduler, error) { conf := &balanceRegionSchedulerConfig{} if err := decoder(conf); err != nil { return nil, err @@ -89,7 +89,7 @@ func schedulersRegister() { }) // balance witness - schedule.RegisterSliceDecoderBuilder(BalanceWitnessType, func(args []string) schedule.ConfigDecoder { + RegisterSliceDecoderBuilder(BalanceWitnessType, func(args []string) ConfigDecoder { return func(v interface{}) error { conf, ok := v.(*balanceWitnessSchedulerConfig) if !ok { @@ -105,7 +105,7 @@ func schedulersRegister() { } }) - schedule.RegisterScheduler(BalanceWitnessType, func(opController *schedule.OperatorController, storage endpoint.ConfigStorage, decoder schedule.ConfigDecoder) (schedule.Scheduler, error) { + RegisterScheduler(BalanceWitnessType, func(opController *operator.Controller, storage endpoint.ConfigStorage, decoder ConfigDecoder, removeSchedulerCb ...func(string) error) (Scheduler, error) { conf := &balanceWitnessSchedulerConfig{storage: storage} if err := decoder(conf); err != nil { return nil, err @@ -117,7 +117,7 @@ func schedulersRegister() { }) // evict leader - schedule.RegisterSliceDecoderBuilder(EvictLeaderType, func(args []string) schedule.ConfigDecoder { + RegisterSliceDecoderBuilder(EvictLeaderType, func(args []string) ConfigDecoder { return func(v interface{}) error { if len(args) != 1 { return errs.ErrSchedulerConfig.FastGenByArgs("id") @@ -141,23 +141,24 @@ func schedulersRegister() { } }) - schedule.RegisterScheduler(EvictLeaderType, func(opController *schedule.OperatorController, storage endpoint.ConfigStorage, decoder schedule.ConfigDecoder) (schedule.Scheduler, error) { + RegisterScheduler(EvictLeaderType, func(opController *operator.Controller, storage endpoint.ConfigStorage, decoder ConfigDecoder, removeSchedulerCb ...func(string) error) (Scheduler, error) { conf := &evictLeaderSchedulerConfig{StoreIDWithRanges: make(map[uint64][]core.KeyRange), storage: storage} if err := decoder(conf); err != nil { return nil, err } conf.cluster = opController.GetCluster() + conf.removeSchedulerCb = removeSchedulerCb[0] return newEvictLeaderScheduler(opController, conf), nil }) // evict slow store - schedule.RegisterSliceDecoderBuilder(EvictSlowStoreType, func(args []string) schedule.ConfigDecoder { + RegisterSliceDecoderBuilder(EvictSlowStoreType, func(args []string) ConfigDecoder { return func(v interface{}) error { return nil } }) - schedule.RegisterScheduler(EvictSlowStoreType, func(opController *schedule.OperatorController, storage endpoint.ConfigStorage, decoder schedule.ConfigDecoder) (schedule.Scheduler, error) { + RegisterScheduler(EvictSlowStoreType, func(opController *operator.Controller, storage endpoint.ConfigStorage, decoder ConfigDecoder, removeSchedulerCb ...func(string) error) (Scheduler, error) { conf := &evictSlowStoreSchedulerConfig{storage: storage, EvictedStores: make([]uint64, 0)} if err := decoder(conf); err != nil { return nil, err @@ -166,7 +167,7 @@ func schedulersRegister() { }) // grant hot region - schedule.RegisterSliceDecoderBuilder(GrantHotRegionType, func(args []string) schedule.ConfigDecoder { + RegisterSliceDecoderBuilder(GrantHotRegionType, func(args []string) ConfigDecoder { return func(v interface{}) error { if len(args) != 2 { return errs.ErrSchedulerConfig.FastGenByArgs("id") @@ -196,7 +197,7 @@ func schedulersRegister() { } }) - schedule.RegisterScheduler(GrantHotRegionType, func(opController *schedule.OperatorController, storage endpoint.ConfigStorage, decoder schedule.ConfigDecoder) (schedule.Scheduler, error) { + RegisterScheduler(GrantHotRegionType, func(opController *operator.Controller, storage endpoint.ConfigStorage, decoder ConfigDecoder, removeSchedulerCb ...func(string) error) (Scheduler, error) { conf := &grantHotRegionSchedulerConfig{StoreIDs: make([]uint64, 0), storage: storage} conf.cluster = opController.GetCluster() if err := decoder(conf); err != nil { @@ -206,13 +207,13 @@ func schedulersRegister() { }) // hot region - schedule.RegisterSliceDecoderBuilder(HotRegionType, func(args []string) schedule.ConfigDecoder { + RegisterSliceDecoderBuilder(HotRegionType, func(args []string) ConfigDecoder { return func(v interface{}) error { return nil } }) - schedule.RegisterScheduler(HotRegionType, func(opController *schedule.OperatorController, storage endpoint.ConfigStorage, decoder schedule.ConfigDecoder) (schedule.Scheduler, error) { + RegisterScheduler(HotRegionType, func(opController *operator.Controller, storage endpoint.ConfigStorage, decoder ConfigDecoder, removeSchedulerCb ...func(string) error) (Scheduler, error) { conf := initHotRegionScheduleConfig() var data map[string]interface{} if err := decoder(&data); err != nil { @@ -233,7 +234,7 @@ func schedulersRegister() { }) // grant leader - schedule.RegisterSliceDecoderBuilder(GrantLeaderType, func(args []string) schedule.ConfigDecoder { + RegisterSliceDecoderBuilder(GrantLeaderType, func(args []string) ConfigDecoder { return func(v interface{}) error { if len(args) != 1 { return errs.ErrSchedulerConfig.FastGenByArgs("id") @@ -257,9 +258,10 @@ func schedulersRegister() { } }) - schedule.RegisterScheduler(GrantLeaderType, func(opController *schedule.OperatorController, storage endpoint.ConfigStorage, decoder schedule.ConfigDecoder) (schedule.Scheduler, error) { + RegisterScheduler(GrantLeaderType, func(opController *operator.Controller, storage endpoint.ConfigStorage, decoder ConfigDecoder, removeSchedulerCb ...func(string) error) (Scheduler, error) { conf := &grantLeaderSchedulerConfig{StoreIDWithRanges: make(map[uint64][]core.KeyRange), storage: storage} conf.cluster = opController.GetCluster() + conf.removeSchedulerCb = removeSchedulerCb[0] if err := decoder(conf); err != nil { return nil, err } @@ -267,7 +269,7 @@ func schedulersRegister() { }) // label - schedule.RegisterSliceDecoderBuilder(LabelType, func(args []string) schedule.ConfigDecoder { + RegisterSliceDecoderBuilder(LabelType, func(args []string) ConfigDecoder { return func(v interface{}) error { conf, ok := v.(*labelSchedulerConfig) if !ok { @@ -283,7 +285,7 @@ func schedulersRegister() { } }) - schedule.RegisterScheduler(LabelType, func(opController *schedule.OperatorController, storage endpoint.ConfigStorage, decoder schedule.ConfigDecoder) (schedule.Scheduler, error) { + RegisterScheduler(LabelType, func(opController *operator.Controller, storage endpoint.ConfigStorage, decoder ConfigDecoder, removeSchedulerCb ...func(string) error) (Scheduler, error) { conf := &labelSchedulerConfig{} if err := decoder(conf); err != nil { return nil, err @@ -292,7 +294,7 @@ func schedulersRegister() { }) // random merge - schedule.RegisterSliceDecoderBuilder(RandomMergeType, func(args []string) schedule.ConfigDecoder { + RegisterSliceDecoderBuilder(RandomMergeType, func(args []string) ConfigDecoder { return func(v interface{}) error { conf, ok := v.(*randomMergeSchedulerConfig) if !ok { @@ -308,7 +310,7 @@ func schedulersRegister() { } }) - schedule.RegisterScheduler(RandomMergeType, func(opController *schedule.OperatorController, storage endpoint.ConfigStorage, decoder schedule.ConfigDecoder) (schedule.Scheduler, error) { + RegisterScheduler(RandomMergeType, func(opController *operator.Controller, storage endpoint.ConfigStorage, decoder ConfigDecoder, removeSchedulerCb ...func(string) error) (Scheduler, error) { conf := &randomMergeSchedulerConfig{} if err := decoder(conf); err != nil { return nil, err @@ -318,7 +320,7 @@ func schedulersRegister() { // scatter range // args: [start-key, end-key, range-name]. - schedule.RegisterSliceDecoderBuilder(ScatterRangeType, func(args []string) schedule.ConfigDecoder { + RegisterSliceDecoderBuilder(ScatterRangeType, func(args []string) ConfigDecoder { return func(v interface{}) error { if len(args) != 3 { return errs.ErrSchedulerConfig.FastGenByArgs("ranges and name") @@ -337,7 +339,7 @@ func schedulersRegister() { } }) - schedule.RegisterScheduler(ScatterRangeType, func(opController *schedule.OperatorController, storage endpoint.ConfigStorage, decoder schedule.ConfigDecoder) (schedule.Scheduler, error) { + RegisterScheduler(ScatterRangeType, func(opController *operator.Controller, storage endpoint.ConfigStorage, decoder ConfigDecoder, removeSchedulerCb ...func(string) error) (Scheduler, error) { conf := &scatterRangeSchedulerConfig{ storage: storage, } @@ -352,7 +354,7 @@ func schedulersRegister() { }) // shuffle hot region - schedule.RegisterSliceDecoderBuilder(ShuffleHotRegionType, func(args []string) schedule.ConfigDecoder { + RegisterSliceDecoderBuilder(ShuffleHotRegionType, func(args []string) ConfigDecoder { return func(v interface{}) error { conf, ok := v.(*shuffleHotRegionSchedulerConfig) if !ok { @@ -371,7 +373,7 @@ func schedulersRegister() { } }) - schedule.RegisterScheduler(ShuffleHotRegionType, func(opController *schedule.OperatorController, storage endpoint.ConfigStorage, decoder schedule.ConfigDecoder) (schedule.Scheduler, error) { + RegisterScheduler(ShuffleHotRegionType, func(opController *operator.Controller, storage endpoint.ConfigStorage, decoder ConfigDecoder, removeSchedulerCb ...func(string) error) (Scheduler, error) { conf := &shuffleHotRegionSchedulerConfig{Limit: uint64(1)} if err := decoder(conf); err != nil { return nil, err @@ -380,7 +382,7 @@ func schedulersRegister() { }) // shuffle leader - schedule.RegisterSliceDecoderBuilder(ShuffleLeaderType, func(args []string) schedule.ConfigDecoder { + RegisterSliceDecoderBuilder(ShuffleLeaderType, func(args []string) ConfigDecoder { return func(v interface{}) error { conf, ok := v.(*shuffleLeaderSchedulerConfig) if !ok { @@ -396,7 +398,7 @@ func schedulersRegister() { } }) - schedule.RegisterScheduler(ShuffleLeaderType, func(opController *schedule.OperatorController, storage endpoint.ConfigStorage, decoder schedule.ConfigDecoder) (schedule.Scheduler, error) { + RegisterScheduler(ShuffleLeaderType, func(opController *operator.Controller, storage endpoint.ConfigStorage, decoder ConfigDecoder, removeSchedulerCb ...func(string) error) (Scheduler, error) { conf := &shuffleLeaderSchedulerConfig{} if err := decoder(conf); err != nil { return nil, err @@ -405,7 +407,7 @@ func schedulersRegister() { }) // shuffle region - schedule.RegisterSliceDecoderBuilder(ShuffleRegionType, func(args []string) schedule.ConfigDecoder { + RegisterSliceDecoderBuilder(ShuffleRegionType, func(args []string) ConfigDecoder { return func(v interface{}) error { conf, ok := v.(*shuffleRegionSchedulerConfig) if !ok { @@ -421,7 +423,7 @@ func schedulersRegister() { } }) - schedule.RegisterScheduler(ShuffleRegionType, func(opController *schedule.OperatorController, storage endpoint.ConfigStorage, decoder schedule.ConfigDecoder) (schedule.Scheduler, error) { + RegisterScheduler(ShuffleRegionType, func(opController *operator.Controller, storage endpoint.ConfigStorage, decoder ConfigDecoder, removeSchedulerCb ...func(string) error) (Scheduler, error) { conf := &shuffleRegionSchedulerConfig{storage: storage} if err := decoder(conf); err != nil { return nil, err @@ -430,13 +432,13 @@ func schedulersRegister() { }) // split bucket - schedule.RegisterSliceDecoderBuilder(SplitBucketType, func(args []string) schedule.ConfigDecoder { + RegisterSliceDecoderBuilder(SplitBucketType, func(args []string) ConfigDecoder { return func(v interface{}) error { return nil } }) - schedule.RegisterScheduler(SplitBucketType, func(opController *schedule.OperatorController, storage endpoint.ConfigStorage, decoder schedule.ConfigDecoder) (schedule.Scheduler, error) { + RegisterScheduler(SplitBucketType, func(opController *operator.Controller, storage endpoint.ConfigStorage, decoder ConfigDecoder, removeSchedulerCb ...func(string) error) (Scheduler, error) { conf := initSplitBucketConfig() if err := decoder(conf); err != nil { return nil, err @@ -446,24 +448,24 @@ func schedulersRegister() { }) // transfer witness leader - schedule.RegisterSliceDecoderBuilder(TransferWitnessLeaderType, func(args []string) schedule.ConfigDecoder { + RegisterSliceDecoderBuilder(TransferWitnessLeaderType, func(args []string) ConfigDecoder { return func(v interface{}) error { return nil } }) - schedule.RegisterScheduler(TransferWitnessLeaderType, func(opController *schedule.OperatorController, _ endpoint.ConfigStorage, _ schedule.ConfigDecoder) (schedule.Scheduler, error) { + RegisterScheduler(TransferWitnessLeaderType, func(opController *operator.Controller, _ endpoint.ConfigStorage, _ ConfigDecoder, removeSchedulerCb ...func(string) error) (Scheduler, error) { return newTransferWitnessLeaderScheduler(opController), nil }) // evict slow store by trend - schedule.RegisterSliceDecoderBuilder(EvictSlowTrendType, func(args []string) schedule.ConfigDecoder { + RegisterSliceDecoderBuilder(EvictSlowTrendType, func(args []string) ConfigDecoder { return func(v interface{}) error { return nil } }) - schedule.RegisterScheduler(EvictSlowTrendType, func(opController *schedule.OperatorController, storage endpoint.ConfigStorage, decoder schedule.ConfigDecoder) (schedule.Scheduler, error) { + RegisterScheduler(EvictSlowTrendType, func(opController *operator.Controller, storage endpoint.ConfigStorage, decoder ConfigDecoder, removeSchedulerCb ...func(string) error) (Scheduler, error) { conf := &evictSlowTrendSchedulerConfig{storage: storage, EvictedStores: make([]uint64, 0), evictCandidate: 0} if err := decoder(conf); err != nil { return nil, err diff --git a/pkg/schedule/schedulers/label.go b/pkg/schedule/schedulers/label.go index a78e3fa35d8..62a1100d16b 100644 --- a/pkg/schedule/schedulers/label.go +++ b/pkg/schedule/schedulers/label.go @@ -17,9 +17,10 @@ package schedulers import ( "github.com/pingcap/log" "github.com/tikv/pd/pkg/core" + "github.com/tikv/pd/pkg/core/constant" "github.com/tikv/pd/pkg/errs" - "github.com/tikv/pd/pkg/schedule" "github.com/tikv/pd/pkg/schedule/config" + sche "github.com/tikv/pd/pkg/schedule/core" "github.com/tikv/pd/pkg/schedule/filter" "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/plan" @@ -55,7 +56,7 @@ type labelScheduler struct { // LabelScheduler is mainly based on the store's label information for scheduling. // Now only used for reject leader schedule, that will move the leader out of // the store with the specific label. -func newLabelScheduler(opController *schedule.OperatorController, conf *labelSchedulerConfig) schedule.Scheduler { +func newLabelScheduler(opController *operator.Controller, conf *labelSchedulerConfig) Scheduler { return &labelScheduler{ BaseScheduler: NewBaseScheduler(opController), conf: conf, @@ -71,23 +72,23 @@ func (s *labelScheduler) GetType() string { } func (s *labelScheduler) EncodeConfig() ([]byte, error) { - return schedule.EncodeConfig(s.conf) + return EncodeConfig(s.conf) } -func (s *labelScheduler) IsScheduleAllowed(cluster schedule.Cluster) bool { - allowed := s.OpController.OperatorCount(operator.OpLeader) < cluster.GetOpts().GetLeaderScheduleLimit() +func (s *labelScheduler) IsScheduleAllowed(cluster sche.SchedulerCluster) bool { + allowed := s.OpController.OperatorCount(operator.OpLeader) < cluster.GetSchedulerConfig().GetLeaderScheduleLimit() if !allowed { operator.OperatorLimitCounter.WithLabelValues(s.GetType(), operator.OpLeader.String()).Inc() } return allowed } -func (s *labelScheduler) Schedule(cluster schedule.Cluster, dryRun bool) ([]*operator.Operator, []plan.Plan) { +func (s *labelScheduler) Schedule(cluster sche.SchedulerCluster, dryRun bool) ([]*operator.Operator, []plan.Plan) { labelCounter.Inc() stores := cluster.GetStores() rejectLeaderStores := make(map[uint64]struct{}) for _, s := range stores { - if cluster.GetOpts().CheckLabelProperty(config.RejectLeader, s.GetLabels()) { + if cluster.GetSchedulerConfig().CheckLabelProperty(config.RejectLeader, s.GetLabels()) { rejectLeaderStores[s.GetID()] = struct{}{} } } @@ -109,7 +110,7 @@ func (s *labelScheduler) Schedule(cluster schedule.Cluster, dryRun bool) ([]*ope f := filter.NewExcludedFilter(s.GetName(), nil, excludeStores) target := filter.NewCandidates(cluster.GetFollowerStores(region)). - FilterTarget(cluster.GetOpts(), nil, nil, &filter.StoreStateFilter{ActionScope: LabelName, TransferLeader: true}, f). + FilterTarget(cluster.GetSchedulerConfig(), nil, nil, &filter.StoreStateFilter{ActionScope: LabelName, TransferLeader: true, OperatorLevel: constant.Medium}, f). RandomPick() if target == nil { log.Debug("label scheduler no target found for region", zap.Uint64("region-id", region.GetID())) diff --git a/pkg/schedule/schedulers/metrics.go b/pkg/schedule/schedulers/metrics.go index 47512450ae5..96b1766a833 100644 --- a/pkg/schedule/schedulers/metrics.go +++ b/pkg/schedule/schedulers/metrics.go @@ -16,148 +16,168 @@ package schedulers import "github.com/prometheus/client_golang/prometheus" -var schedulerCounter = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Namespace: "pd", - Subsystem: "scheduler", - Name: "event_count", - Help: "Counter of scheduler events.", - }, []string{"type", "name"}) - -var schedulerStatus = prometheus.NewGaugeVec( - prometheus.GaugeOpts{ - Namespace: "pd", - Subsystem: "scheduler", - Name: "inner_status", - Help: "Inner status of the scheduler.", - }, []string{"type", "name"}) - -// TODO: pre-allocate gauge metrics -var opInfluenceStatus = prometheus.NewGaugeVec( - prometheus.GaugeOpts{ - Namespace: "pd", - Subsystem: "scheduler", - Name: "op_influence", - Help: "Store status for schedule", - }, []string{"scheduler", "store", "type"}) - -// TODO: pre-allocate gauge metrics -var tolerantResourceStatus = prometheus.NewGaugeVec( - prometheus.GaugeOpts{ - Namespace: "pd", - Subsystem: "scheduler", - Name: "tolerant_resource", - Help: "Store status for schedule", - }, []string{"scheduler"}) - -var balanceLeaderCounter = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Namespace: "pd", - Subsystem: "scheduler", - Name: "balance_leader", - Help: "Counter of balance leader scheduler.", - }, []string{"type", "store"}) - -var balanceWitnessCounter = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Namespace: "pd", - Subsystem: "scheduler", - Name: "balance_witness", - Help: "Counter of balance witness scheduler.", - }, []string{"type", "store"}) - -var balanceRegionCounter = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Namespace: "pd", - Subsystem: "scheduler", - Name: "balance_region", - Help: "Counter of balance region scheduler.", - }, []string{"type", "store"}) - -// TODO: pre-allocate gauge metrics -var hotSchedulerResultCounter = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Namespace: "pd", - Subsystem: "scheduler", - Name: "hot_region", - Help: "Counter of hot region scheduler.", - }, []string{"type", "store"}) - -var balanceDirectionCounter = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Namespace: "pd", - Subsystem: "scheduler", - Name: "balance_direction", - Help: "Counter of direction of balance related schedulers.", - }, []string{"type", "source", "target"}) - -// TODO: pre-allocate gauge metrics -var hotDirectionCounter = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Namespace: "pd", - Subsystem: "scheduler", - Name: "hot_region_direction", - Help: "Counter of hot region scheduler.", - }, []string{"type", "rw", "store", "direction", "dim"}) - -var scatterRangeLeaderCounter = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Namespace: "pd", - Subsystem: "scheduler", - Name: "scatter_range_leader", - Help: "Counter of scatter range leader scheduler.", - }, []string{"type", "store"}) - -var scatterRangeRegionCounter = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Namespace: "pd", - Subsystem: "scheduler", - Name: "scatter_range_region", - Help: "Counter of scatter range region scheduler.", - }, []string{"type", "store"}) - -var hotPendingStatus = prometheus.NewGaugeVec( - prometheus.GaugeOpts{ - Namespace: "pd", - Subsystem: "scheduler", - Name: "hot_pending", - Help: "Pending influence status in hot region scheduler.", - }, []string{"type", "source", "target"}) - -var hotPeerHist = prometheus.NewHistogramVec( - prometheus.HistogramOpts{ - Namespace: "pd", - Subsystem: "scheduler", - Name: "hot_peer", - Help: "Bucketed histogram of the scheduling hot peer.", - Buckets: prometheus.ExponentialBuckets(1, 2, 30), - }, []string{"type", "rw", "dim"}) - -var storeSlowTrendEvictedStatusGauge = prometheus.NewGaugeVec( - prometheus.GaugeOpts{ - Namespace: "pd", - Subsystem: "scheduler", - Name: "store_slow_trend_evicted_status", - Help: "Store evited by slow trend status for schedule", - }, []string{"address", "store"}) - -var storeSlowTrendActionStatusGauge = prometheus.NewGaugeVec( - prometheus.GaugeOpts{ - Namespace: "pd", - Subsystem: "scheduler", - Name: "store_slow_trend_action_status", - Help: "Store trend scheduler calculating actions", - }, []string{"reason"}) - -var storeSlowTrendMiscGauge = prometheus.NewGaugeVec( - prometheus.GaugeOpts{ - Namespace: "pd", - Subsystem: "scheduler", - Name: "store_slow_trend_misc", - Help: "Store trend internal uncatelogued values", - }, []string{"type"}) +var ( + schedulerStatusGauge = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: "pd", + Subsystem: "scheduler", + Name: "status", + Help: "Status of the scheduler.", + }, []string{"kind", "type"}) + + schedulerCounter = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "pd", + Subsystem: "scheduler", + Name: "event_count", + Help: "Counter of scheduler events.", + }, []string{"type", "name"}) + + schedulerStatus = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: "pd", + Subsystem: "scheduler", + Name: "inner_status", + Help: "Inner status of the scheduler.", + }, []string{"type", "name"}) + + // TODO: pre-allocate gauge metrics + opInfluenceStatus = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: "pd", + Subsystem: "scheduler", + Name: "op_influence", + Help: "Store status for schedule", + }, []string{"scheduler", "store", "type"}) + + // TODO: pre-allocate gauge metrics + tolerantResourceStatus = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: "pd", + Subsystem: "scheduler", + Name: "tolerant_resource", + Help: "Store status for schedule", + }, []string{"scheduler"}) + + balanceLeaderCounter = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "pd", + Subsystem: "scheduler", + Name: "balance_leader", + Help: "Counter of balance leader scheduler.", + }, []string{"type", "store"}) + + balanceWitnessCounter = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "pd", + Subsystem: "scheduler", + Name: "balance_witness", + Help: "Counter of balance witness scheduler.", + }, []string{"type", "store"}) + + balanceRegionCounter = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "pd", + Subsystem: "scheduler", + Name: "balance_region", + Help: "Counter of balance region scheduler.", + }, []string{"type", "store"}) + + // TODO: pre-allocate gauge metrics + hotSchedulerResultCounter = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "pd", + Subsystem: "scheduler", + Name: "hot_region", + Help: "Counter of hot region scheduler.", + }, []string{"type", "store"}) + + balanceDirectionCounter = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "pd", + Subsystem: "scheduler", + Name: "balance_direction", + Help: "Counter of direction of balance related schedulers.", + }, []string{"type", "source", "target"}) + + // TODO: pre-allocate gauge metrics + hotDirectionCounter = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "pd", + Subsystem: "scheduler", + Name: "hot_region_direction", + Help: "Counter of hot region scheduler.", + }, []string{"type", "rw", "store", "direction", "dim"}) + + scatterRangeLeaderCounter = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "pd", + Subsystem: "scheduler", + Name: "scatter_range_leader", + Help: "Counter of scatter range leader scheduler.", + }, []string{"type", "store"}) + + scatterRangeRegionCounter = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "pd", + Subsystem: "scheduler", + Name: "scatter_range_region", + Help: "Counter of scatter range region scheduler.", + }, []string{"type", "store"}) + + hotPendingStatus = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: "pd", + Subsystem: "scheduler", + Name: "hot_pending", + Help: "Pending influence status in hot region scheduler.", + }, []string{"type", "source", "target"}) + + hotPeerHist = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Namespace: "pd", + Subsystem: "scheduler", + Name: "hot_peer", + Help: "Bucketed histogram of the scheduling hot peer.", + Buckets: prometheus.ExponentialBuckets(1, 2, 30), + }, []string{"type", "rw", "dim"}) + + storeSlowTrendEvictedStatusGauge = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: "pd", + Subsystem: "scheduler", + Name: "store_slow_trend_evicted_status", + Help: "Store evited by slow trend status for schedule", + }, []string{"address", "store"}) + + storeSlowTrendActionStatusGauge = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: "pd", + Subsystem: "scheduler", + Name: "store_slow_trend_action_status", + Help: "Store trend scheduler calculating actions", + }, []string{"reason"}) + + storeSlowTrendMiscGauge = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: "pd", + Subsystem: "scheduler", + Name: "store_slow_trend_misc", + Help: "Store trend internal uncatelogued values", + }, []string{"type"}) + + // HotPendingSum is the sum of pending influence in hot region scheduler. + HotPendingSum = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: "pd", + Subsystem: "scheduler", + Name: "hot_pending_sum", + Help: "Pending influence sum of store in hot region scheduler.", + }, []string{"store", "rw", "dim"}) +) func init() { + prometheus.MustRegister(schedulerStatusGauge) prometheus.MustRegister(schedulerCounter) prometheus.MustRegister(schedulerStatus) prometheus.MustRegister(balanceLeaderCounter) @@ -175,4 +195,5 @@ func init() { prometheus.MustRegister(storeSlowTrendEvictedStatusGauge) prometheus.MustRegister(storeSlowTrendActionStatusGauge) prometheus.MustRegister(storeSlowTrendMiscGauge) + prometheus.MustRegister(HotPendingSum) } diff --git a/pkg/schedule/schedulers/random_merge.go b/pkg/schedule/schedulers/random_merge.go index be6aa40c20f..a621b595198 100644 --- a/pkg/schedule/schedulers/random_merge.go +++ b/pkg/schedule/schedulers/random_merge.go @@ -21,8 +21,8 @@ import ( "github.com/tikv/pd/pkg/core" "github.com/tikv/pd/pkg/core/constant" "github.com/tikv/pd/pkg/errs" - "github.com/tikv/pd/pkg/schedule" "github.com/tikv/pd/pkg/schedule/checker" + sche "github.com/tikv/pd/pkg/schedule/core" "github.com/tikv/pd/pkg/schedule/filter" "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/plan" @@ -57,7 +57,7 @@ type randomMergeScheduler struct { // newRandomMergeScheduler creates an admin scheduler that randomly picks two adjacent regions // then merges them. -func newRandomMergeScheduler(opController *schedule.OperatorController, conf *randomMergeSchedulerConfig) schedule.Scheduler { +func newRandomMergeScheduler(opController *operator.Controller, conf *randomMergeSchedulerConfig) Scheduler { base := NewBaseScheduler(opController) return &randomMergeScheduler{ BaseScheduler: base, @@ -74,22 +74,22 @@ func (s *randomMergeScheduler) GetType() string { } func (s *randomMergeScheduler) EncodeConfig() ([]byte, error) { - return schedule.EncodeConfig(s.conf) + return EncodeConfig(s.conf) } -func (s *randomMergeScheduler) IsScheduleAllowed(cluster schedule.Cluster) bool { - allowed := s.OpController.OperatorCount(operator.OpMerge) < cluster.GetOpts().GetMergeScheduleLimit() +func (s *randomMergeScheduler) IsScheduleAllowed(cluster sche.SchedulerCluster) bool { + allowed := s.OpController.OperatorCount(operator.OpMerge) < cluster.GetSchedulerConfig().GetMergeScheduleLimit() if !allowed { operator.OperatorLimitCounter.WithLabelValues(s.GetType(), operator.OpMerge.String()).Inc() } return allowed } -func (s *randomMergeScheduler) Schedule(cluster schedule.Cluster, dryRun bool) ([]*operator.Operator, []plan.Plan) { +func (s *randomMergeScheduler) Schedule(cluster sche.SchedulerCluster, dryRun bool) ([]*operator.Operator, []plan.Plan) { randomMergeCounter.Inc() store := filter.NewCandidates(cluster.GetStores()). - FilterSource(cluster.GetOpts(), nil, nil, &filter.StoreStateFilter{ActionScope: s.conf.Name, MoveRegion: true}). + FilterSource(cluster.GetSchedulerConfig(), nil, nil, &filter.StoreStateFilter{ActionScope: s.conf.Name, MoveRegion: true, OperatorLevel: constant.Low}). RandomPick() if store == nil { randomMergeNoSourceStoreCounter.Inc() @@ -104,7 +104,7 @@ func (s *randomMergeScheduler) Schedule(cluster schedule.Cluster, dryRun bool) ( } other, target := cluster.GetAdjacentRegions(region) - if !cluster.GetOpts().IsOneWayMergeEnabled() && ((rand.Int()%2 == 0 && other != nil) || target == nil) { + if !cluster.GetSchedulerConfig().IsOneWayMergeEnabled() && ((rand.Int()%2 == 0 && other != nil) || target == nil) { target = other } if target == nil { @@ -128,7 +128,7 @@ func (s *randomMergeScheduler) Schedule(cluster schedule.Cluster, dryRun bool) ( return ops, nil } -func (s *randomMergeScheduler) allowMerge(cluster schedule.Cluster, region, target *core.RegionInfo) bool { +func (s *randomMergeScheduler) allowMerge(cluster sche.SchedulerCluster, region, target *core.RegionInfo) bool { if !filter.IsRegionHealthy(region) || !filter.IsRegionHealthy(target) { return false } diff --git a/pkg/schedule/range_cluster.go b/pkg/schedule/schedulers/range_cluster.go similarity index 72% rename from pkg/schedule/range_cluster.go rename to pkg/schedule/schedulers/range_cluster.go index 5c2645c4a3e..e83e74145f6 100644 --- a/pkg/schedule/range_cluster.go +++ b/pkg/schedule/schedulers/range_cluster.go @@ -12,35 +12,36 @@ // See the License for the specific language governing permissions and // limitations under the License. -package schedule +package schedulers import ( "github.com/docker/go-units" "github.com/tikv/pd/pkg/core" + sche "github.com/tikv/pd/pkg/schedule/core" ) -// RangeCluster isolates the cluster by range. -type RangeCluster struct { - Cluster +// rangeCluster isolates the cluster by range. +type rangeCluster struct { + sche.SchedulerCluster subCluster *core.BasicCluster // Collect all regions belong to the range. tolerantSizeRatio float64 } -// GenRangeCluster gets a range cluster by specifying start key and end key. -// The cluster can only know the regions within [startKey, endKey]. -func GenRangeCluster(cluster Cluster, startKey, endKey []byte) *RangeCluster { +// genRangeCluster gets a range cluster by specifying start key and end key. +// The cluster can only know the regions within [startKey, endKey). +func genRangeCluster(cluster sche.SchedulerCluster, startKey, endKey []byte) *rangeCluster { subCluster := core.NewBasicCluster() for _, r := range cluster.ScanRegions(startKey, endKey, -1) { origin, overlaps, rangeChanged := subCluster.SetRegion(r) subCluster.UpdateSubTree(r, origin, overlaps, rangeChanged) } - return &RangeCluster{ - Cluster: cluster, - subCluster: subCluster, + return &rangeCluster{ + SchedulerCluster: cluster, + subCluster: subCluster, } } -func (r *RangeCluster) updateStoreInfo(s *core.StoreInfo) *core.StoreInfo { +func (r *rangeCluster) updateStoreInfo(s *core.StoreInfo) *core.StoreInfo { id := s.GetID() used := float64(s.GetUsedSize()) / units.MiB @@ -68,8 +69,8 @@ func (r *RangeCluster) updateStoreInfo(s *core.StoreInfo) *core.StoreInfo { } // GetStore searches for a store by ID. -func (r *RangeCluster) GetStore(id uint64) *core.StoreInfo { - s := r.Cluster.GetStore(id) +func (r *rangeCluster) GetStore(id uint64) *core.StoreInfo { + s := r.SchedulerCluster.GetStore(id) if s == nil { return nil } @@ -77,8 +78,8 @@ func (r *RangeCluster) GetStore(id uint64) *core.StoreInfo { } // GetStores returns all Stores in the cluster. -func (r *RangeCluster) GetStores() []*core.StoreInfo { - stores := r.Cluster.GetStores() +func (r *rangeCluster) GetStores() []*core.StoreInfo { + stores := r.SchedulerCluster.GetStores() newStores := make([]*core.StoreInfo, 0, len(stores)) for _, s := range stores { newStores = append(newStores, r.updateStoreInfo(s)) @@ -87,36 +88,36 @@ func (r *RangeCluster) GetStores() []*core.StoreInfo { } // SetTolerantSizeRatio sets the tolerant size ratio. -func (r *RangeCluster) SetTolerantSizeRatio(ratio float64) { +func (r *rangeCluster) SetTolerantSizeRatio(ratio float64) { r.tolerantSizeRatio = ratio } // GetTolerantSizeRatio gets the tolerant size ratio. -func (r *RangeCluster) GetTolerantSizeRatio() float64 { +func (r *rangeCluster) GetTolerantSizeRatio() float64 { if r.tolerantSizeRatio != 0 { return r.tolerantSizeRatio } - return r.Cluster.GetOpts().GetTolerantSizeRatio() + return r.SchedulerCluster.GetSchedulerConfig().GetTolerantSizeRatio() } // RandFollowerRegions returns a random region that has a follower on the store. -func (r *RangeCluster) RandFollowerRegions(storeID uint64, ranges []core.KeyRange) []*core.RegionInfo { +func (r *rangeCluster) RandFollowerRegions(storeID uint64, ranges []core.KeyRange) []*core.RegionInfo { return r.subCluster.RandFollowerRegions(storeID, ranges) } // RandLeaderRegions returns a random region that has leader on the store. -func (r *RangeCluster) RandLeaderRegions(storeID uint64, ranges []core.KeyRange) []*core.RegionInfo { +func (r *rangeCluster) RandLeaderRegions(storeID uint64, ranges []core.KeyRange) []*core.RegionInfo { return r.subCluster.RandLeaderRegions(storeID, ranges) } // GetAverageRegionSize returns the average region approximate size. -func (r *RangeCluster) GetAverageRegionSize() int64 { +func (r *rangeCluster) GetAverageRegionSize() int64 { return r.subCluster.GetAverageRegionSize() } // GetRegionStores returns all stores that contains the region's peer. -func (r *RangeCluster) GetRegionStores(region *core.RegionInfo) []*core.StoreInfo { - stores := r.Cluster.GetRegionStores(region) +func (r *rangeCluster) GetRegionStores(region *core.RegionInfo) []*core.StoreInfo { + stores := r.SchedulerCluster.GetRegionStores(region) newStores := make([]*core.StoreInfo, 0, len(stores)) for _, s := range stores { newStores = append(newStores, r.updateStoreInfo(s)) @@ -125,8 +126,8 @@ func (r *RangeCluster) GetRegionStores(region *core.RegionInfo) []*core.StoreInf } // GetFollowerStores returns all stores that contains the region's follower peer. -func (r *RangeCluster) GetFollowerStores(region *core.RegionInfo) []*core.StoreInfo { - stores := r.Cluster.GetFollowerStores(region) +func (r *rangeCluster) GetFollowerStores(region *core.RegionInfo) []*core.StoreInfo { + stores := r.SchedulerCluster.GetFollowerStores(region) newStores := make([]*core.StoreInfo, 0, len(stores)) for _, s := range stores { newStores = append(newStores, r.updateStoreInfo(s)) @@ -135,8 +136,8 @@ func (r *RangeCluster) GetFollowerStores(region *core.RegionInfo) []*core.StoreI } // GetLeaderStore returns all stores that contains the region's leader peer. -func (r *RangeCluster) GetLeaderStore(region *core.RegionInfo) *core.StoreInfo { - s := r.Cluster.GetLeaderStore(region) +func (r *rangeCluster) GetLeaderStore(region *core.RegionInfo) *core.StoreInfo { + s := r.SchedulerCluster.GetLeaderStore(region) if s != nil { return r.updateStoreInfo(s) } diff --git a/pkg/schedule/schedulers/scatter_range.go b/pkg/schedule/schedulers/scatter_range.go index 8ca06ed50d8..19b606f8d28 100644 --- a/pkg/schedule/schedulers/scatter_range.go +++ b/pkg/schedule/schedulers/scatter_range.go @@ -22,7 +22,7 @@ import ( "github.com/pingcap/errors" "github.com/tikv/pd/pkg/core" "github.com/tikv/pd/pkg/errs" - "github.com/tikv/pd/pkg/schedule" + sche "github.com/tikv/pd/pkg/schedule/core" "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/plan" "github.com/tikv/pd/pkg/storage/endpoint" @@ -83,7 +83,7 @@ func (conf *scatterRangeSchedulerConfig) Persist() error { name := conf.getSchedulerName() conf.mu.RLock() defer conf.mu.RUnlock() - data, err := schedule.EncodeConfig(conf) + data, err := EncodeConfig(conf) if err != nil { return err } @@ -118,13 +118,13 @@ type scatterRangeScheduler struct { *BaseScheduler name string config *scatterRangeSchedulerConfig - balanceLeader schedule.Scheduler - balanceRegion schedule.Scheduler + balanceLeader Scheduler + balanceRegion Scheduler handler http.Handler } // newScatterRangeScheduler creates a scheduler that balances the distribution of leaders and regions that in the specified key range. -func newScatterRangeScheduler(opController *schedule.OperatorController, config *scatterRangeSchedulerConfig) schedule.Scheduler { +func newScatterRangeScheduler(opController *operator.Controller, config *scatterRangeSchedulerConfig) Scheduler { base := NewBaseScheduler(opController) name := config.getSchedulerName() @@ -165,33 +165,33 @@ func (l *scatterRangeScheduler) GetType() string { func (l *scatterRangeScheduler) EncodeConfig() ([]byte, error) { l.config.mu.RLock() defer l.config.mu.RUnlock() - return schedule.EncodeConfig(l.config) + return EncodeConfig(l.config) } -func (l *scatterRangeScheduler) IsScheduleAllowed(cluster schedule.Cluster) bool { +func (l *scatterRangeScheduler) IsScheduleAllowed(cluster sche.SchedulerCluster) bool { return l.allowBalanceLeader(cluster) || l.allowBalanceRegion(cluster) } -func (l *scatterRangeScheduler) allowBalanceLeader(cluster schedule.Cluster) bool { - allowed := l.OpController.OperatorCount(operator.OpRange) < cluster.GetOpts().GetLeaderScheduleLimit() +func (l *scatterRangeScheduler) allowBalanceLeader(cluster sche.SchedulerCluster) bool { + allowed := l.OpController.OperatorCount(operator.OpRange) < cluster.GetSchedulerConfig().GetLeaderScheduleLimit() if !allowed { operator.OperatorLimitCounter.WithLabelValues(l.GetType(), operator.OpLeader.String()).Inc() } return allowed } -func (l *scatterRangeScheduler) allowBalanceRegion(cluster schedule.Cluster) bool { - allowed := l.OpController.OperatorCount(operator.OpRange) < cluster.GetOpts().GetRegionScheduleLimit() +func (l *scatterRangeScheduler) allowBalanceRegion(cluster sche.SchedulerCluster) bool { + allowed := l.OpController.OperatorCount(operator.OpRange) < cluster.GetSchedulerConfig().GetRegionScheduleLimit() if !allowed { operator.OperatorLimitCounter.WithLabelValues(l.GetType(), operator.OpRegion.String()).Inc() } return allowed } -func (l *scatterRangeScheduler) Schedule(cluster schedule.Cluster, dryRun bool) ([]*operator.Operator, []plan.Plan) { +func (l *scatterRangeScheduler) Schedule(cluster sche.SchedulerCluster, dryRun bool) ([]*operator.Operator, []plan.Plan) { scatterRangeCounter.Inc() // isolate a new cluster according to the key range - c := schedule.GenRangeCluster(cluster, l.config.GetStartKey(), l.config.GetEndKey()) + c := genRangeCluster(cluster, l.config.GetStartKey(), l.config.GetEndKey()) c.SetTolerantSizeRatio(2) if l.allowBalanceLeader(cluster) { ops, _ := l.balanceLeader.Schedule(c, false) diff --git a/pkg/schedule/scheduler.go b/pkg/schedule/schedulers/scheduler.go similarity index 83% rename from pkg/schedule/scheduler.go rename to pkg/schedule/schedulers/scheduler.go index cb16177d87e..b4c425047cd 100644 --- a/pkg/schedule/scheduler.go +++ b/pkg/schedule/schedulers/scheduler.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package schedule +package schedulers import ( "encoding/json" @@ -24,6 +24,7 @@ import ( "github.com/pingcap/log" "github.com/tikv/pd/pkg/errs" "github.com/tikv/pd/pkg/schedule/config" + sche "github.com/tikv/pd/pkg/schedule/core" "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/plan" "github.com/tikv/pd/pkg/storage/endpoint" @@ -34,15 +35,15 @@ import ( type Scheduler interface { http.Handler GetName() string - // GetType should in accordance with the name passing to schedule.RegisterScheduler() + // GetType should in accordance with the name passing to RegisterScheduler() GetType() string EncodeConfig() ([]byte, error) GetMinInterval() time.Duration GetNextInterval(interval time.Duration) time.Duration - Prepare(cluster Cluster) error - Cleanup(cluster Cluster) - Schedule(cluster Cluster, dryRun bool) ([]*operator.Operator, []plan.Plan) - IsScheduleAllowed(cluster Cluster) bool + Prepare(cluster sche.SchedulerCluster) error + Cleanup(cluster sche.SchedulerCluster) + Schedule(cluster sche.SchedulerCluster, dryRun bool) ([]*operator.Operator, []plan.Plan) + IsScheduleAllowed(cluster sche.SchedulerCluster) bool } // EncodeConfig encode the custom config for each scheduler. @@ -88,7 +89,7 @@ func ConfigSliceDecoder(name string, args []string) ConfigDecoder { } // CreateSchedulerFunc is for creating scheduler. -type CreateSchedulerFunc func(opController *OperatorController, storage endpoint.ConfigStorage, dec ConfigDecoder) (Scheduler, error) +type CreateSchedulerFunc func(opController *operator.Controller, storage endpoint.ConfigStorage, dec ConfigDecoder, removeSchedulerCb ...func(string) error) (Scheduler, error) var schedulerMap = make(map[string]CreateSchedulerFunc) var schedulerArgsToDecoder = make(map[string]ConfigSliceDecoderBuilder) @@ -113,13 +114,13 @@ func RegisterSliceDecoderBuilder(typ string, builder ConfigSliceDecoderBuilder) } // CreateScheduler creates a scheduler with registered creator func. -func CreateScheduler(typ string, opController *OperatorController, storage endpoint.ConfigStorage, dec ConfigDecoder) (Scheduler, error) { +func CreateScheduler(typ string, oc *operator.Controller, storage endpoint.ConfigStorage, dec ConfigDecoder, removeSchedulerCb ...func(string) error) (Scheduler, error) { fn, ok := schedulerMap[typ] if !ok { return nil, errs.ErrSchedulerCreateFuncNotRegistered.FastGenByArgs(typ) } - s, err := fn(opController, storage, dec) + s, err := fn(oc, storage, dec, removeSchedulerCb...) if err != nil { return nil, err } diff --git a/pkg/schedule/schedulers/scheduler_controller.go b/pkg/schedule/schedulers/scheduler_controller.go new file mode 100644 index 00000000000..1c6329fb0b1 --- /dev/null +++ b/pkg/schedule/schedulers/scheduler_controller.go @@ -0,0 +1,511 @@ +// Copyright 2023 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package schedulers + +import ( + "context" + "net/http" + "sync" + "sync/atomic" + "time" + + "github.com/pingcap/log" + "github.com/tikv/pd/pkg/core" + "github.com/tikv/pd/pkg/errs" + sche "github.com/tikv/pd/pkg/schedule/core" + "github.com/tikv/pd/pkg/schedule/labeler" + "github.com/tikv/pd/pkg/schedule/operator" + "github.com/tikv/pd/pkg/schedule/plan" + "github.com/tikv/pd/pkg/storage/endpoint" + "github.com/tikv/pd/pkg/utils/logutil" + "go.uber.org/zap" +) + +const maxScheduleRetries = 10 + +var denySchedulersByLabelerCounter = labeler.LabelerEventCounter.WithLabelValues("schedulers", "deny") + +// Controller is used to manage all schedulers. +type Controller struct { + sync.RWMutex + wg sync.WaitGroup + ctx context.Context + cluster sche.SchedulerCluster + storage endpoint.ConfigStorage + schedulers map[string]*ScheduleController + opController *operator.Controller +} + +// NewController creates a scheduler controller. +func NewController(ctx context.Context, cluster sche.SchedulerCluster, storage endpoint.ConfigStorage, opController *operator.Controller) *Controller { + return &Controller{ + ctx: ctx, + cluster: cluster, + storage: storage, + schedulers: make(map[string]*ScheduleController), + opController: opController, + } +} + +// Wait waits on all schedulers to exit. +func (c *Controller) Wait() { + c.wg.Wait() +} + +// GetScheduler returns a schedule controller by name. +func (c *Controller) GetScheduler(name string) *ScheduleController { + c.RLock() + defer c.RUnlock() + return c.schedulers[name] +} + +// GetSchedulerNames returns all names of schedulers. +func (c *Controller) GetSchedulerNames() []string { + c.RLock() + defer c.RUnlock() + names := make([]string, 0, len(c.schedulers)) + for name := range c.schedulers { + names = append(names, name) + } + return names +} + +// GetSchedulerHandlers returns all handlers of schedulers. +func (c *Controller) GetSchedulerHandlers() map[string]http.Handler { + c.RLock() + defer c.RUnlock() + handlers := make(map[string]http.Handler, len(c.schedulers)) + for name, scheduler := range c.schedulers { + handlers[name] = scheduler.Scheduler + } + return handlers +} + +// CollectSchedulerMetrics collects metrics of all schedulers. +func (c *Controller) CollectSchedulerMetrics() { + c.RLock() + defer c.RUnlock() + for _, s := range c.schedulers { + var allowScheduler float64 + // If the scheduler is not allowed to schedule, it will disappear in Grafana panel. + // See issue #1341. + if !s.IsPaused() && !c.isSchedulingHalted() { + allowScheduler = 1 + } + schedulerStatusGauge.WithLabelValues(s.Scheduler.GetName(), "allow").Set(allowScheduler) + } +} + +func (c *Controller) isSchedulingHalted() bool { + return c.cluster.GetSchedulerConfig().IsSchedulingHalted() +} + +// ResetSchedulerMetrics resets metrics of all schedulers. +func (c *Controller) ResetSchedulerMetrics() { + schedulerStatusGauge.Reset() +} + +// AddScheduler adds a scheduler. +func (c *Controller) AddScheduler(scheduler Scheduler, args ...string) error { + c.Lock() + defer c.Unlock() + + if _, ok := c.schedulers[scheduler.GetName()]; ok { + return errs.ErrSchedulerExisted.FastGenByArgs() + } + + s := NewScheduleController(c.ctx, c.cluster, c.opController, scheduler) + if err := s.Scheduler.Prepare(c.cluster); err != nil { + return err + } + + c.wg.Add(1) + go c.runScheduler(s) + c.schedulers[s.Scheduler.GetName()] = s + c.cluster.GetSchedulerConfig().AddSchedulerCfg(s.Scheduler.GetType(), args) + return nil +} + +// RemoveScheduler removes a scheduler by name. +func (c *Controller) RemoveScheduler(name string) error { + c.Lock() + defer c.Unlock() + if c.cluster == nil { + return errs.ErrNotBootstrapped.FastGenByArgs() + } + s, ok := c.schedulers[name] + if !ok { + return errs.ErrSchedulerNotFound.FastGenByArgs() + } + + conf := c.cluster.GetSchedulerConfig() + conf.RemoveSchedulerCfg(s.Scheduler.GetType()) + if err := conf.Persist(c.storage); err != nil { + log.Error("the option can not persist scheduler config", errs.ZapError(err)) + return err + } + + if err := c.storage.RemoveScheduleConfig(name); err != nil { + log.Error("can not remove the scheduler config", errs.ZapError(err)) + return err + } + + s.Stop() + schedulerStatusGauge.DeleteLabelValues(name, "allow") + delete(c.schedulers, name) + + return nil +} + +// PauseOrResumeScheduler pauses or resumes a scheduler by name. +func (c *Controller) PauseOrResumeScheduler(name string, t int64) error { + c.Lock() + defer c.Unlock() + if c.cluster == nil { + return errs.ErrNotBootstrapped.FastGenByArgs() + } + var s []*ScheduleController + if name != "all" { + sc, ok := c.schedulers[name] + if !ok { + return errs.ErrSchedulerNotFound.FastGenByArgs() + } + s = append(s, sc) + } else { + for _, sc := range c.schedulers { + s = append(s, sc) + } + } + var err error + for _, sc := range s { + var delayAt, delayUntil int64 + if t > 0 { + delayAt = time.Now().Unix() + delayUntil = delayAt + t + } + sc.SetDelay(delayAt, delayUntil) + } + return err +} + +// IsSchedulerAllowed returns whether a scheduler is allowed to schedule, a scheduler is not allowed to schedule if it is paused or blocked by unsafe recovery. +func (c *Controller) IsSchedulerAllowed(name string) (bool, error) { + c.RLock() + defer c.RUnlock() + if c.cluster == nil { + return false, errs.ErrNotBootstrapped.FastGenByArgs() + } + s, ok := c.schedulers[name] + if !ok { + return false, errs.ErrSchedulerNotFound.FastGenByArgs() + } + return s.AllowSchedule(false), nil +} + +// IsSchedulerPaused returns whether a scheduler is paused. +func (c *Controller) IsSchedulerPaused(name string) (bool, error) { + c.RLock() + defer c.RUnlock() + if c.cluster == nil { + return false, errs.ErrNotBootstrapped.FastGenByArgs() + } + s, ok := c.schedulers[name] + if !ok { + return false, errs.ErrSchedulerNotFound.FastGenByArgs() + } + return s.IsPaused(), nil +} + +// IsSchedulerDisabled returns whether a scheduler is disabled. +func (c *Controller) IsSchedulerDisabled(name string) (bool, error) { + c.RLock() + defer c.RUnlock() + if c.cluster == nil { + return false, errs.ErrNotBootstrapped.FastGenByArgs() + } + s, ok := c.schedulers[name] + if !ok { + return false, errs.ErrSchedulerNotFound.FastGenByArgs() + } + return c.cluster.GetSchedulerConfig().IsSchedulerDisabled(s.Scheduler.GetType()), nil +} + +// IsSchedulerExisted returns whether a scheduler is existed. +func (c *Controller) IsSchedulerExisted(name string) (bool, error) { + c.RLock() + defer c.RUnlock() + if c.cluster == nil { + return false, errs.ErrNotBootstrapped.FastGenByArgs() + } + _, ok := c.schedulers[name] + if !ok { + return false, errs.ErrSchedulerNotFound.FastGenByArgs() + } + return true, nil +} + +func (c *Controller) runScheduler(s *ScheduleController) { + defer logutil.LogPanic() + defer c.wg.Done() + defer s.Scheduler.Cleanup(c.cluster) + + ticker := time.NewTicker(s.GetInterval()) + defer ticker.Stop() + for { + select { + case <-ticker.C: + diagnosable := s.IsDiagnosticAllowed() + if !s.AllowSchedule(diagnosable) { + continue + } + if op := s.Schedule(diagnosable); len(op) > 0 { + added := c.opController.AddWaitingOperator(op...) + log.Debug("add operator", zap.Int("added", added), zap.Int("total", len(op)), zap.String("scheduler", s.Scheduler.GetName())) + } + // Note: we reset the ticker here to support updating configuration dynamically. + ticker.Reset(s.GetInterval()) + case <-s.Ctx().Done(): + log.Info("scheduler has been stopped", + zap.String("scheduler-name", s.Scheduler.GetName()), + errs.ZapError(s.Ctx().Err())) + return + } + } +} + +// GetPausedSchedulerDelayAt returns paused timestamp of a paused scheduler +func (c *Controller) GetPausedSchedulerDelayAt(name string) (int64, error) { + c.RLock() + defer c.RUnlock() + if c.cluster == nil { + return -1, errs.ErrNotBootstrapped.FastGenByArgs() + } + s, ok := c.schedulers[name] + if !ok { + return -1, errs.ErrSchedulerNotFound.FastGenByArgs() + } + return s.GetDelayAt(), nil +} + +// GetPausedSchedulerDelayUntil returns the delay time until the scheduler is paused. +func (c *Controller) GetPausedSchedulerDelayUntil(name string) (int64, error) { + c.RLock() + defer c.RUnlock() + if c.cluster == nil { + return -1, errs.ErrNotBootstrapped.FastGenByArgs() + } + s, ok := c.schedulers[name] + if !ok { + return -1, errs.ErrSchedulerNotFound.FastGenByArgs() + } + return s.GetDelayUntil(), nil +} + +// CheckTransferWitnessLeader determines if transfer leader is required, then sends to the scheduler if needed +func (c *Controller) CheckTransferWitnessLeader(region *core.RegionInfo) { + if core.NeedTransferWitnessLeader(region) { + c.RLock() + s, ok := c.schedulers[TransferWitnessLeaderName] + c.RUnlock() + if ok { + select { + case RecvRegionInfo(s.Scheduler) <- region: + default: + log.Warn("drop transfer witness leader due to recv region channel full", zap.Uint64("region-id", region.GetID())) + } + } + } +} + +// ScheduleController is used to manage a scheduler. +type ScheduleController struct { + Scheduler + cluster sche.SchedulerCluster + opController *operator.Controller + nextInterval time.Duration + ctx context.Context + cancel context.CancelFunc + delayAt int64 + delayUntil int64 + diagnosticRecorder *DiagnosticRecorder +} + +// NewScheduleController creates a new ScheduleController. +func NewScheduleController(ctx context.Context, cluster sche.SchedulerCluster, opController *operator.Controller, s Scheduler) *ScheduleController { + ctx, cancel := context.WithCancel(ctx) + return &ScheduleController{ + Scheduler: s, + cluster: cluster, + opController: opController, + nextInterval: s.GetMinInterval(), + ctx: ctx, + cancel: cancel, + diagnosticRecorder: NewDiagnosticRecorder(s.GetName(), cluster.GetSchedulerConfig()), + } +} + +// Ctx returns the context of ScheduleController +func (s *ScheduleController) Ctx() context.Context { + return s.ctx +} + +// Stop stops the ScheduleController +func (s *ScheduleController) Stop() { + s.cancel() +} + +// Schedule tries to create some operators. +func (s *ScheduleController) Schedule(diagnosable bool) []*operator.Operator { + for i := 0; i < maxScheduleRetries; i++ { + // no need to retry if schedule should stop to speed exit + select { + case <-s.ctx.Done(): + return nil + default: + } + cacheCluster := newCacheCluster(s.cluster) + // we need only process diagnostic once in the retry loop + diagnosable = diagnosable && i == 0 + ops, plans := s.Scheduler.Schedule(cacheCluster, diagnosable) + if diagnosable { + s.diagnosticRecorder.SetResultFromPlans(ops, plans) + } + foundDisabled := false + for _, op := range ops { + if labelMgr := s.cluster.GetRegionLabeler(); labelMgr != nil { + region := s.cluster.GetRegion(op.RegionID()) + if region == nil { + continue + } + if labelMgr.ScheduleDisabled(region) { + denySchedulersByLabelerCounter.Inc() + foundDisabled = true + break + } + } + } + if len(ops) > 0 { + // If we have schedule, reset interval to the minimal interval. + s.nextInterval = s.Scheduler.GetMinInterval() + // try regenerating operators + if foundDisabled { + continue + } + return ops + } + } + s.nextInterval = s.Scheduler.GetNextInterval(s.nextInterval) + return nil +} + +// DiagnoseDryRun returns the operators and plans of a scheduler. +func (s *ScheduleController) DiagnoseDryRun() ([]*operator.Operator, []plan.Plan) { + cacheCluster := newCacheCluster(s.cluster) + return s.Scheduler.Schedule(cacheCluster, true) +} + +// GetInterval returns the interval of scheduling for a scheduler. +func (s *ScheduleController) GetInterval() time.Duration { + return s.nextInterval +} + +// SetInterval sets the interval of scheduling for a scheduler. for test purpose. +func (s *ScheduleController) SetInterval(interval time.Duration) { + s.nextInterval = interval +} + +// AllowSchedule returns if a scheduler is allowed to +func (s *ScheduleController) AllowSchedule(diagnosable bool) bool { + if !s.Scheduler.IsScheduleAllowed(s.cluster) { + if diagnosable { + s.diagnosticRecorder.SetResultFromStatus(Pending) + } + return false + } + if s.isSchedulingHalted() { + if diagnosable { + s.diagnosticRecorder.SetResultFromStatus(Halted) + } + return false + } + if s.IsPaused() { + if diagnosable { + s.diagnosticRecorder.SetResultFromStatus(Paused) + } + return false + } + return true +} + +func (s *ScheduleController) isSchedulingHalted() bool { + return s.cluster.GetSchedulerConfig().IsSchedulingHalted() +} + +// IsPaused returns if a scheduler is paused. +func (s *ScheduleController) IsPaused() bool { + delayUntil := atomic.LoadInt64(&s.delayUntil) + return time.Now().Unix() < delayUntil +} + +// GetDelayAt returns paused timestamp of a paused scheduler +func (s *ScheduleController) GetDelayAt() int64 { + if s.IsPaused() { + return atomic.LoadInt64(&s.delayAt) + } + return 0 +} + +// GetDelayUntil returns resume timestamp of a paused scheduler +func (s *ScheduleController) GetDelayUntil() int64 { + if s.IsPaused() { + return atomic.LoadInt64(&s.delayUntil) + } + return 0 +} + +// SetDelay sets the delay of a scheduler. +func (s *ScheduleController) SetDelay(delayAt, delayUntil int64) { + atomic.StoreInt64(&s.delayAt, delayAt) + atomic.StoreInt64(&s.delayUntil, delayUntil) +} + +// GetDiagnosticRecorder returns the diagnostic recorder of a scheduler. +func (s *ScheduleController) GetDiagnosticRecorder() *DiagnosticRecorder { + return s.diagnosticRecorder +} + +// IsDiagnosticAllowed returns if a scheduler is allowed to do diagnostic. +func (s *ScheduleController) IsDiagnosticAllowed() bool { + return s.diagnosticRecorder.IsAllowed() +} + +// cacheCluster include cache info to improve the performance. +type cacheCluster struct { + sche.SchedulerCluster + stores []*core.StoreInfo +} + +// GetStores returns store infos from cache +func (c *cacheCluster) GetStores() []*core.StoreInfo { + return c.stores +} + +// newCacheCluster constructor for cache +func newCacheCluster(c sche.SchedulerCluster) *cacheCluster { + return &cacheCluster{ + SchedulerCluster: c, + stores: c.GetStores(), + } +} diff --git a/pkg/schedule/schedulers/scheduler_test.go b/pkg/schedule/schedulers/scheduler_test.go index b26de523be8..54ee9366f45 100644 --- a/pkg/schedule/schedulers/scheduler_test.go +++ b/pkg/schedule/schedulers/scheduler_test.go @@ -24,18 +24,17 @@ import ( "github.com/tikv/pd/pkg/core" "github.com/tikv/pd/pkg/mock/mockcluster" "github.com/tikv/pd/pkg/mock/mockconfig" - "github.com/tikv/pd/pkg/schedule" "github.com/tikv/pd/pkg/schedule/config" "github.com/tikv/pd/pkg/schedule/hbstream" "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/placement" "github.com/tikv/pd/pkg/statistics" "github.com/tikv/pd/pkg/storage" - "github.com/tikv/pd/pkg/utils/testutil" + "github.com/tikv/pd/pkg/utils/operatorutil" "github.com/tikv/pd/pkg/versioninfo" ) -func prepareSchedulersTest(needToRunStream ...bool) (context.CancelFunc, config.Config, *mockcluster.Cluster, *schedule.OperatorController) { +func prepareSchedulersTest(needToRunStream ...bool) (context.CancelFunc, config.SchedulerConfigProvider, *mockcluster.Cluster, *operator.Controller) { Register() ctx, cancel := context.WithCancel(context.Background()) opt := mockconfig.NewTestOptions() @@ -46,7 +45,7 @@ func prepareSchedulersTest(needToRunStream ...bool) (context.CancelFunc, config. } else { stream = hbstream.NewTestHeartbeatStreams(ctx, tc.ID, tc, needToRunStream[0]) } - oc := schedule.NewOperatorController(ctx, tc, stream) + oc := operator.NewController(ctx, tc.GetBasicCluster(), tc.GetSchedulerConfig(), stream) return cancel, opt, tc, oc } @@ -55,7 +54,7 @@ func TestShuffleLeader(t *testing.T) { cancel, _, tc, oc := prepareSchedulersTest() defer cancel() - sl, err := schedule.CreateScheduler(ShuffleLeaderType, oc, storage.NewStorageWithMemoryBackend(), schedule.ConfigSliceDecoder(ShuffleLeaderType, []string{"", ""})) + sl, err := CreateScheduler(ShuffleLeaderType, oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(ShuffleLeaderType, []string{"", ""})) re.NoError(err) ops, _ := sl.Schedule(tc, false) re.Empty(ops) @@ -93,25 +92,25 @@ func TestRejectLeader(t *testing.T) { tc.AddLeaderRegion(2, 2, 1, 3) // The label scheduler transfers leader out of store1. - sl, err := schedule.CreateScheduler(LabelType, oc, storage.NewStorageWithMemoryBackend(), schedule.ConfigSliceDecoder(LabelType, []string{"", ""})) + sl, err := CreateScheduler(LabelType, oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(LabelType, []string{"", ""})) re.NoError(err) ops, _ := sl.Schedule(tc, false) - testutil.CheckTransferLeaderFrom(re, ops[0], operator.OpLeader, 1) + operatorutil.CheckTransferLeaderFrom(re, ops[0], operator.OpLeader, 1) // If store3 is disconnected, transfer leader to store 2. tc.SetStoreDisconnect(3) ops, _ = sl.Schedule(tc, false) - testutil.CheckTransferLeader(re, ops[0], operator.OpLeader, 1, 2) + operatorutil.CheckTransferLeader(re, ops[0], operator.OpLeader, 1, 2) // As store3 is disconnected, store1 rejects leader. Balancer will not create // any operators. - bs, err := schedule.CreateScheduler(BalanceLeaderType, oc, storage.NewStorageWithMemoryBackend(), schedule.ConfigSliceDecoder(BalanceLeaderType, []string{"", ""})) + bs, err := CreateScheduler(BalanceLeaderType, oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(BalanceLeaderType, []string{"", ""})) re.NoError(err) ops, _ = bs.Schedule(tc, false) re.Empty(ops) // Can't evict leader from store2, neither. - el, err := schedule.CreateScheduler(EvictLeaderType, oc, storage.NewStorageWithMemoryBackend(), schedule.ConfigSliceDecoder(EvictLeaderType, []string{"2"})) + el, err := CreateScheduler(EvictLeaderType, oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(EvictLeaderType, []string{"2"}), func(string) error { return nil }) re.NoError(err) ops, _ = el.Schedule(tc, false) re.Empty(ops) @@ -128,7 +127,7 @@ func TestRejectLeader(t *testing.T) { origin, overlaps, rangeChanged := tc.SetRegion(region) tc.UpdateSubTree(region, origin, overlaps, rangeChanged) ops, _ = sl.Schedule(tc, false) - testutil.CheckTransferLeader(re, ops[0], operator.OpLeader, 1, 2) + operatorutil.CheckTransferLeader(re, ops[0], operator.OpLeader, 1, 2) } func TestRemoveRejectLeader(t *testing.T) { @@ -137,7 +136,7 @@ func TestRemoveRejectLeader(t *testing.T) { defer cancel() tc.AddRegionStore(1, 0) tc.AddRegionStore(2, 1) - el, err := schedule.CreateScheduler(EvictLeaderType, oc, storage.NewStorageWithMemoryBackend(), schedule.ConfigSliceDecoder(EvictLeaderType, []string{"1"})) + el, err := CreateScheduler(EvictLeaderType, oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(EvictLeaderType, []string{"1"}), func(string) error { return nil }) re.NoError(err) tc.DeleteStore(tc.GetStore(1)) succ, _ := el.(*evictLeaderScheduler).conf.removeStore(1) @@ -157,7 +156,7 @@ func checkBalance(re *require.Assertions, enablePlacementRules bool) { tc.SetEnablePlacementRules(enablePlacementRules) labels := []string{"zone", "host"} tc.SetMaxReplicasWithLabel(enablePlacementRules, 3, labels...) - hb, err := schedule.CreateScheduler(ShuffleHotRegionType, oc, storage.NewStorageWithMemoryBackend(), schedule.ConfigSliceDecoder("shuffle-hot-region", []string{"", ""})) + hb, err := CreateScheduler(ShuffleHotRegionType, oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder("shuffle-hot-region", []string{"", ""})) re.NoError(err) // Add stores 1, 2, 3, 4, 5, 6 with hot peer counts 3, 2, 2, 2, 0, 0. tc.AddLabelsStore(1, 3, map[string]string{"zone": "z1", "host": "h1"}) @@ -204,7 +203,7 @@ func TestHotRegionScheduleAbnormalReplica(t *testing.T) { cancel, _, tc, oc := prepareSchedulersTest() defer cancel() tc.SetHotRegionScheduleLimit(0) - hb, err := schedule.CreateScheduler(statistics.Read.String(), oc, storage.NewStorageWithMemoryBackend(), nil) + hb, err := CreateScheduler(statistics.Read.String(), oc, storage.NewStorageWithMemoryBackend(), nil) re.NoError(err) tc.AddRegionStore(1, 3) @@ -229,7 +228,7 @@ func TestShuffleRegion(t *testing.T) { cancel, _, tc, oc := prepareSchedulersTest() defer cancel() - sl, err := schedule.CreateScheduler(ShuffleRegionType, oc, storage.NewStorageWithMemoryBackend(), schedule.ConfigSliceDecoder(ShuffleRegionType, []string{"", ""})) + sl, err := CreateScheduler(ShuffleRegionType, oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(ShuffleRegionType, []string{"", ""})) re.NoError(err) re.True(sl.IsScheduleAllowed(tc)) ops, _ := sl.Schedule(tc, false) @@ -293,18 +292,18 @@ func TestShuffleRegionRole(t *testing.T) { }, peers[0]) tc.PutRegion(region) - sl, err := schedule.CreateScheduler(ShuffleRegionType, oc, storage.NewStorageWithMemoryBackend(), schedule.ConfigSliceDecoder(ShuffleRegionType, []string{"", ""})) + sl, err := CreateScheduler(ShuffleRegionType, oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(ShuffleRegionType, []string{"", ""})) re.NoError(err) conf := sl.(*shuffleRegionScheduler).conf conf.Roles = []string{"follower"} ops, _ := sl.Schedule(tc, false) re.Len(ops, 1) - testutil.CheckTransferPeer(re, ops[0], operator.OpKind(0), 2, 4) // transfer follower + operatorutil.CheckTransferPeer(re, ops[0], operator.OpKind(0), 2, 4) // transfer follower conf.Roles = []string{"learner"} ops, _ = sl.Schedule(tc, false) re.Len(ops, 1) - testutil.CheckTransferLearner(re, ops[0], operator.OpRegion, 3, 4) + operatorutil.CheckTransferLearner(re, ops[0], operator.OpRegion, 3, 4) } func TestSpecialUseHotRegion(t *testing.T) { @@ -313,10 +312,10 @@ func TestSpecialUseHotRegion(t *testing.T) { defer cancel() storage := storage.NewStorageWithMemoryBackend() - cd := schedule.ConfigSliceDecoder(BalanceRegionType, []string{"", ""}) - bs, err := schedule.CreateScheduler(BalanceRegionType, oc, storage, cd) + cd := ConfigSliceDecoder(BalanceRegionType, []string{"", ""}) + bs, err := CreateScheduler(BalanceRegionType, oc, storage, cd) re.NoError(err) - hs, err := schedule.CreateScheduler(statistics.Write.String(), oc, storage, cd) + hs, err := CreateScheduler(statistics.Write.String(), oc, storage, cd) re.NoError(err) tc.SetHotRegionCacheHitsThreshold(0) @@ -335,7 +334,7 @@ func TestSpecialUseHotRegion(t *testing.T) { // balance region without label ops, _ := bs.Schedule(tc, false) re.Len(ops, 1) - testutil.CheckTransferPeer(re, ops[0], operator.OpKind(0), 1, 4) + operatorutil.CheckTransferPeer(re, ops[0], operator.OpKind(0), 1, 4) // cannot balance to store 4 and 5 with label tc.AddLabelsStore(4, 0, map[string]string{"specialUse": "hotRegion"}) @@ -356,7 +355,7 @@ func TestSpecialUseHotRegion(t *testing.T) { tc.AddLeaderRegionWithWriteInfo(5, 3, 512*units.KiB*statistics.WriteReportInterval, 0, 0, statistics.WriteReportInterval, []uint64{1, 2}) ops, _ = hs.Schedule(tc, false) re.Len(ops, 1) - testutil.CheckTransferPeer(re, ops[0], operator.OpHotRegion, 1, 4) + operatorutil.CheckTransferPeer(re, ops[0], operator.OpHotRegion, 1, 4) } func TestSpecialUseReserved(t *testing.T) { @@ -365,8 +364,8 @@ func TestSpecialUseReserved(t *testing.T) { defer cancel() storage := storage.NewStorageWithMemoryBackend() - cd := schedule.ConfigSliceDecoder(BalanceRegionType, []string{"", ""}) - bs, err := schedule.CreateScheduler(BalanceRegionType, oc, storage, cd) + cd := ConfigSliceDecoder(BalanceRegionType, []string{"", ""}) + bs, err := CreateScheduler(BalanceRegionType, oc, storage, cd) re.NoError(err) tc.SetHotRegionCacheHitsThreshold(0) @@ -384,7 +383,7 @@ func TestSpecialUseReserved(t *testing.T) { // balance region without label ops, _ := bs.Schedule(tc, false) re.Len(ops, 1) - testutil.CheckTransferPeer(re, ops[0], operator.OpKind(0), 1, 4) + operatorutil.CheckTransferPeer(re, ops[0], operator.OpKind(0), 1, 4) // cannot balance to store 4 with label tc.AddLabelsStore(4, 0, map[string]string{"specialUse": "reserved"}) @@ -400,7 +399,7 @@ func TestBalanceLeaderWithConflictRule(t *testing.T) { cancel, _, tc, oc := prepareSchedulersTest() defer cancel() tc.SetEnablePlacementRules(true) - lb, err := schedule.CreateScheduler(BalanceLeaderType, oc, storage.NewStorageWithMemoryBackend(), schedule.ConfigSliceDecoder(BalanceLeaderType, []string{"", ""})) + lb, err := CreateScheduler(BalanceLeaderType, oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(BalanceLeaderType, []string{"", ""})) re.NoError(err) tc.AddLeaderStore(1, 1) diff --git a/pkg/schedule/schedulers/shuffle_hot_region.go b/pkg/schedule/schedulers/shuffle_hot_region.go index dfe4970bd6b..d5264b90428 100644 --- a/pkg/schedule/schedulers/shuffle_hot_region.go +++ b/pkg/schedule/schedulers/shuffle_hot_region.go @@ -19,7 +19,7 @@ import ( "github.com/pingcap/log" "github.com/tikv/pd/pkg/core/constant" "github.com/tikv/pd/pkg/errs" - "github.com/tikv/pd/pkg/schedule" + sche "github.com/tikv/pd/pkg/schedule/core" "github.com/tikv/pd/pkg/schedule/filter" "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/plan" @@ -56,7 +56,7 @@ type shuffleHotRegionScheduler struct { } // newShuffleHotRegionScheduler creates an admin scheduler that random balance hot regions -func newShuffleHotRegionScheduler(opController *schedule.OperatorController, conf *shuffleHotRegionSchedulerConfig) schedule.Scheduler { +func newShuffleHotRegionScheduler(opController *operator.Controller, conf *shuffleHotRegionSchedulerConfig) Scheduler { base := newBaseHotScheduler(opController) ret := &shuffleHotRegionScheduler{ baseHotScheduler: base, @@ -74,13 +74,14 @@ func (s *shuffleHotRegionScheduler) GetType() string { } func (s *shuffleHotRegionScheduler) EncodeConfig() ([]byte, error) { - return schedule.EncodeConfig(s.conf) + return EncodeConfig(s.conf) } -func (s *shuffleHotRegionScheduler) IsScheduleAllowed(cluster schedule.Cluster) bool { +func (s *shuffleHotRegionScheduler) IsScheduleAllowed(cluster sche.SchedulerCluster) bool { hotRegionAllowed := s.OpController.OperatorCount(operator.OpHotRegion) < s.conf.Limit - regionAllowed := s.OpController.OperatorCount(operator.OpRegion) < cluster.GetOpts().GetRegionScheduleLimit() - leaderAllowed := s.OpController.OperatorCount(operator.OpLeader) < cluster.GetOpts().GetLeaderScheduleLimit() + conf := cluster.GetSchedulerConfig() + regionAllowed := s.OpController.OperatorCount(operator.OpRegion) < conf.GetRegionScheduleLimit() + leaderAllowed := s.OpController.OperatorCount(operator.OpLeader) < conf.GetLeaderScheduleLimit() if !hotRegionAllowed { operator.OperatorLimitCounter.WithLabelValues(s.GetType(), operator.OpHotRegion.String()).Inc() } @@ -93,7 +94,7 @@ func (s *shuffleHotRegionScheduler) IsScheduleAllowed(cluster schedule.Cluster) return hotRegionAllowed && regionAllowed && leaderAllowed } -func (s *shuffleHotRegionScheduler) Schedule(cluster schedule.Cluster, dryRun bool) ([]*operator.Operator, []plan.Plan) { +func (s *shuffleHotRegionScheduler) Schedule(cluster sche.SchedulerCluster, dryRun bool) ([]*operator.Operator, []plan.Plan) { shuffleHotRegionCounter.Inc() rw := s.randomRWType() s.prepareForBalance(rw, cluster) @@ -101,7 +102,7 @@ func (s *shuffleHotRegionScheduler) Schedule(cluster schedule.Cluster, dryRun bo return operators, nil } -func (s *shuffleHotRegionScheduler) randomSchedule(cluster schedule.Cluster, loadDetail map[uint64]*statistics.StoreLoadDetail) []*operator.Operator { +func (s *shuffleHotRegionScheduler) randomSchedule(cluster sche.SchedulerCluster, loadDetail map[uint64]*statistics.StoreLoadDetail) []*operator.Operator { for _, detail := range loadDetail { if len(detail.HotPeers) < 1 { continue @@ -120,14 +121,14 @@ func (s *shuffleHotRegionScheduler) randomSchedule(cluster schedule.Cluster, loa } filters := []filter.Filter{ - &filter.StoreStateFilter{ActionScope: s.GetName(), MoveRegion: true}, + &filter.StoreStateFilter{ActionScope: s.GetName(), MoveRegion: true, OperatorLevel: constant.Low}, filter.NewExcludedFilter(s.GetName(), srcRegion.GetStoreIDs(), srcRegion.GetStoreIDs()), - filter.NewPlacementSafeguard(s.GetName(), cluster.GetOpts(), cluster.GetBasicCluster(), cluster.GetRuleManager(), srcRegion, srcStore, nil), + filter.NewPlacementSafeguard(s.GetName(), cluster.GetSchedulerConfig(), cluster.GetBasicCluster(), cluster.GetRuleManager(), srcRegion, srcStore, nil), } stores := cluster.GetStores() destStoreIDs := make([]uint64, 0, len(stores)) for _, store := range stores { - if !filter.Target(cluster.GetOpts(), store, filters) { + if !filter.Target(cluster.GetSchedulerConfig(), store, filters) { continue } destStoreIDs = append(destStoreIDs, store.GetID()) diff --git a/pkg/schedule/schedulers/shuffle_leader.go b/pkg/schedule/schedulers/shuffle_leader.go index ff6d7debdd6..0e33fa802db 100644 --- a/pkg/schedule/schedulers/shuffle_leader.go +++ b/pkg/schedule/schedulers/shuffle_leader.go @@ -19,7 +19,7 @@ import ( "github.com/tikv/pd/pkg/core" "github.com/tikv/pd/pkg/core/constant" "github.com/tikv/pd/pkg/errs" - "github.com/tikv/pd/pkg/schedule" + sche "github.com/tikv/pd/pkg/schedule/core" "github.com/tikv/pd/pkg/schedule/filter" "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/plan" @@ -53,9 +53,9 @@ type shuffleLeaderScheduler struct { // newShuffleLeaderScheduler creates an admin scheduler that shuffles leaders // between stores. -func newShuffleLeaderScheduler(opController *schedule.OperatorController, conf *shuffleLeaderSchedulerConfig) schedule.Scheduler { +func newShuffleLeaderScheduler(opController *operator.Controller, conf *shuffleLeaderSchedulerConfig) Scheduler { filters := []filter.Filter{ - &filter.StoreStateFilter{ActionScope: conf.Name, TransferLeader: true}, + &filter.StoreStateFilter{ActionScope: conf.Name, TransferLeader: true, OperatorLevel: constant.Low}, filter.NewSpecialUseFilter(conf.Name), } base := NewBaseScheduler(opController) @@ -75,24 +75,24 @@ func (s *shuffleLeaderScheduler) GetType() string { } func (s *shuffleLeaderScheduler) EncodeConfig() ([]byte, error) { - return schedule.EncodeConfig(s.conf) + return EncodeConfig(s.conf) } -func (s *shuffleLeaderScheduler) IsScheduleAllowed(cluster schedule.Cluster) bool { - allowed := s.OpController.OperatorCount(operator.OpLeader) < cluster.GetOpts().GetLeaderScheduleLimit() +func (s *shuffleLeaderScheduler) IsScheduleAllowed(cluster sche.SchedulerCluster) bool { + allowed := s.OpController.OperatorCount(operator.OpLeader) < cluster.GetSchedulerConfig().GetLeaderScheduleLimit() if !allowed { operator.OperatorLimitCounter.WithLabelValues(s.GetType(), operator.OpLeader.String()).Inc() } return allowed } -func (s *shuffleLeaderScheduler) Schedule(cluster schedule.Cluster, dryRun bool) ([]*operator.Operator, []plan.Plan) { +func (s *shuffleLeaderScheduler) Schedule(cluster sche.SchedulerCluster, dryRun bool) ([]*operator.Operator, []plan.Plan) { // We shuffle leaders between stores by: // 1. random select a valid store. // 2. transfer a leader to the store. shuffleLeaderCounter.Inc() targetStore := filter.NewCandidates(cluster.GetStores()). - FilterTarget(cluster.GetOpts(), nil, nil, s.filters...). + FilterTarget(cluster.GetSchedulerConfig(), nil, nil, s.filters...). RandomPick() if targetStore == nil { shuffleLeaderNoTargetStoreCounter.Inc() diff --git a/pkg/schedule/schedulers/shuffle_region.go b/pkg/schedule/schedulers/shuffle_region.go index d29147a3c02..08570fe9f20 100644 --- a/pkg/schedule/schedulers/shuffle_region.go +++ b/pkg/schedule/schedulers/shuffle_region.go @@ -20,7 +20,7 @@ import ( "github.com/pingcap/kvproto/pkg/metapb" "github.com/tikv/pd/pkg/core" "github.com/tikv/pd/pkg/core/constant" - "github.com/tikv/pd/pkg/schedule" + sche "github.com/tikv/pd/pkg/schedule/core" "github.com/tikv/pd/pkg/schedule/filter" "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/plan" @@ -51,9 +51,9 @@ type shuffleRegionScheduler struct { // newShuffleRegionScheduler creates an admin scheduler that shuffles regions // between stores. -func newShuffleRegionScheduler(opController *schedule.OperatorController, conf *shuffleRegionSchedulerConfig) schedule.Scheduler { +func newShuffleRegionScheduler(opController *operator.Controller, conf *shuffleRegionSchedulerConfig) Scheduler { filters := []filter.Filter{ - &filter.StoreStateFilter{ActionScope: ShuffleRegionName, MoveRegion: true}, + &filter.StoreStateFilter{ActionScope: ShuffleRegionName, MoveRegion: true, OperatorLevel: constant.Low}, filter.NewSpecialUseFilter(ShuffleRegionName), } base := NewBaseScheduler(opController) @@ -80,15 +80,15 @@ func (s *shuffleRegionScheduler) EncodeConfig() ([]byte, error) { return s.conf.EncodeConfig() } -func (s *shuffleRegionScheduler) IsScheduleAllowed(cluster schedule.Cluster) bool { - allowed := s.OpController.OperatorCount(operator.OpRegion) < cluster.GetOpts().GetRegionScheduleLimit() +func (s *shuffleRegionScheduler) IsScheduleAllowed(cluster sche.SchedulerCluster) bool { + allowed := s.OpController.OperatorCount(operator.OpRegion) < cluster.GetSchedulerConfig().GetRegionScheduleLimit() if !allowed { operator.OperatorLimitCounter.WithLabelValues(s.GetType(), operator.OpRegion.String()).Inc() } return allowed } -func (s *shuffleRegionScheduler) Schedule(cluster schedule.Cluster, dryRun bool) ([]*operator.Operator, []plan.Plan) { +func (s *shuffleRegionScheduler) Schedule(cluster sche.SchedulerCluster, dryRun bool) ([]*operator.Operator, []plan.Plan) { shuffleRegionCounter.Inc() region, oldPeer := s.scheduleRemovePeer(cluster) if region == nil { @@ -112,9 +112,9 @@ func (s *shuffleRegionScheduler) Schedule(cluster schedule.Cluster, dryRun bool) return []*operator.Operator{op}, nil } -func (s *shuffleRegionScheduler) scheduleRemovePeer(cluster schedule.Cluster) (*core.RegionInfo, *metapb.Peer) { +func (s *shuffleRegionScheduler) scheduleRemovePeer(cluster sche.SchedulerCluster) (*core.RegionInfo, *metapb.Peer) { candidates := filter.NewCandidates(cluster.GetStores()). - FilterSource(cluster.GetOpts(), nil, nil, s.filters...). + FilterSource(cluster.GetSchedulerConfig(), nil, nil, s.filters...). Shuffle() pendingFilter := filter.NewRegionPendingFilter() @@ -144,16 +144,16 @@ func (s *shuffleRegionScheduler) scheduleRemovePeer(cluster schedule.Cluster) (* return nil, nil } -func (s *shuffleRegionScheduler) scheduleAddPeer(cluster schedule.Cluster, region *core.RegionInfo, oldPeer *metapb.Peer) *metapb.Peer { +func (s *shuffleRegionScheduler) scheduleAddPeer(cluster sche.SchedulerCluster, region *core.RegionInfo, oldPeer *metapb.Peer) *metapb.Peer { store := cluster.GetStore(oldPeer.GetStoreId()) if store == nil { return nil } - scoreGuard := filter.NewPlacementSafeguard(s.GetName(), cluster.GetOpts(), cluster.GetBasicCluster(), cluster.GetRuleManager(), region, store, nil) + scoreGuard := filter.NewPlacementSafeguard(s.GetName(), cluster.GetSchedulerConfig(), cluster.GetBasicCluster(), cluster.GetRuleManager(), region, store, nil) excludedFilter := filter.NewExcludedFilter(s.GetName(), nil, region.GetStoreIDs()) target := filter.NewCandidates(cluster.GetStores()). - FilterTarget(cluster.GetOpts(), nil, nil, append(s.filters, scoreGuard, excludedFilter)...). + FilterTarget(cluster.GetSchedulerConfig(), nil, nil, append(s.filters, scoreGuard, excludedFilter)...). RandomPick() if target == nil { return nil diff --git a/pkg/schedule/schedulers/shuffle_region_config.go b/pkg/schedule/schedulers/shuffle_region_config.go index d36fbc7674f..97ccab136b5 100644 --- a/pkg/schedule/schedulers/shuffle_region_config.go +++ b/pkg/schedule/schedulers/shuffle_region_config.go @@ -19,7 +19,6 @@ import ( "github.com/gorilla/mux" "github.com/tikv/pd/pkg/core" - "github.com/tikv/pd/pkg/schedule" "github.com/tikv/pd/pkg/schedule/placement" "github.com/tikv/pd/pkg/slice" "github.com/tikv/pd/pkg/storage/endpoint" @@ -47,7 +46,7 @@ type shuffleRegionSchedulerConfig struct { func (conf *shuffleRegionSchedulerConfig) EncodeConfig() ([]byte, error) { conf.RLock() defer conf.RUnlock() - return schedule.EncodeConfig(conf) + return EncodeConfig(conf) } func (conf *shuffleRegionSchedulerConfig) GetRoles() []string { @@ -106,7 +105,7 @@ func (conf *shuffleRegionSchedulerConfig) handleSetRoles(w http.ResponseWriter, } func (conf *shuffleRegionSchedulerConfig) persist() error { - data, err := schedule.EncodeConfig(conf) + data, err := EncodeConfig(conf) if err != nil { return err } diff --git a/pkg/schedule/schedulers/split_bucket.go b/pkg/schedule/schedulers/split_bucket.go index 95831a1f4c1..415e6207752 100644 --- a/pkg/schedule/schedulers/split_bucket.go +++ b/pkg/schedule/schedulers/split_bucket.go @@ -23,8 +23,7 @@ import ( "github.com/gorilla/mux" "github.com/pingcap/kvproto/pkg/pdpb" - "github.com/tikv/pd/pkg/core" - "github.com/tikv/pd/pkg/schedule" + sche "github.com/tikv/pd/pkg/schedule/core" "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/plan" "github.com/tikv/pd/pkg/statistics/buckets" @@ -80,7 +79,7 @@ func (conf *splitBucketSchedulerConfig) Clone() *splitBucketSchedulerConfig { } func (conf *splitBucketSchedulerConfig) persistLocked() error { - data, err := schedule.EncodeConfig(conf) + data, err := EncodeConfig(conf) if err != nil { return err } @@ -139,7 +138,7 @@ func newSplitBucketHandler(conf *splitBucketSchedulerConfig) http.Handler { return router } -func newSplitBucketScheduler(opController *schedule.OperatorController, conf *splitBucketSchedulerConfig) *splitBucketScheduler { +func newSplitBucketScheduler(opController *operator.Controller, conf *splitBucketSchedulerConfig) *splitBucketScheduler { base := NewBaseScheduler(opController) handler := newSplitBucketHandler(conf) ret := &splitBucketScheduler{ @@ -166,7 +165,7 @@ func (s *splitBucketScheduler) ServeHTTP(w http.ResponseWriter, r *http.Request) } // IsScheduleAllowed return true if the sum of executing opSplit operator is less . -func (s *splitBucketScheduler) IsScheduleAllowed(cluster schedule.Cluster) bool { +func (s *splitBucketScheduler) IsScheduleAllowed(cluster sche.SchedulerCluster) bool { if !cluster.GetStoreConfig().IsEnableRegionBucket() { splitBucketDisableCounter.Inc() return false @@ -181,20 +180,20 @@ func (s *splitBucketScheduler) IsScheduleAllowed(cluster schedule.Cluster) bool type splitBucketPlan struct { hotBuckets map[uint64][]*buckets.BucketStat - cluster schedule.Cluster + cluster sche.SchedulerCluster conf *splitBucketSchedulerConfig hotRegionSplitSize int64 } // Schedule return operators if some bucket is too hot. -func (s *splitBucketScheduler) Schedule(cluster schedule.Cluster, dryRun bool) ([]*operator.Operator, []plan.Plan) { +func (s *splitBucketScheduler) Schedule(cluster sche.SchedulerCluster, dryRun bool) ([]*operator.Operator, []plan.Plan) { splitBucketScheduleCounter.Inc() conf := s.conf.Clone() plan := &splitBucketPlan{ conf: conf, cluster: cluster, hotBuckets: cluster.BucketsStats(conf.Degree), - hotRegionSplitSize: cluster.GetOpts().GetMaxMovableHotPeerSize(), + hotRegionSplitSize: cluster.GetSchedulerConfig().GetMaxMovableHotPeerSize(), } return s.splitBucket(plan), nil } @@ -251,8 +250,6 @@ func (s *splitBucketScheduler) splitBucket(plan *splitBucketPlan) []*operator.Op return nil } splitBucketNewOperatorCounter.Inc() - op.AdditionalInfos["region-start-key"] = core.HexRegionKeyStr(region.GetStartKey()) - op.AdditionalInfos["region-end-key"] = core.HexRegionKeyStr(region.GetEndKey()) op.AdditionalInfos["hot-degree"] = strconv.FormatInt(int64(splitBucket.HotDegree), 10) return []*operator.Operator{op} } diff --git a/pkg/schedule/schedulers/transfer_witness_leader.go b/pkg/schedule/schedulers/transfer_witness_leader.go index b14d0bc7d41..bddbc7cf0ec 100644 --- a/pkg/schedule/schedulers/transfer_witness_leader.go +++ b/pkg/schedule/schedulers/transfer_witness_leader.go @@ -20,7 +20,7 @@ import ( "github.com/tikv/pd/pkg/core" "github.com/tikv/pd/pkg/core/constant" "github.com/tikv/pd/pkg/errs" - "github.com/tikv/pd/pkg/schedule" + sche "github.com/tikv/pd/pkg/schedule/core" "github.com/tikv/pd/pkg/schedule/filter" "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/plan" @@ -52,7 +52,7 @@ type trasferWitnessLeaderScheduler struct { } // newTransferWitnessLeaderScheduler creates an admin scheduler that transfers witness leader of a region. -func newTransferWitnessLeaderScheduler(opController *schedule.OperatorController) schedule.Scheduler { +func newTransferWitnessLeaderScheduler(opController *operator.Controller) Scheduler { return &trasferWitnessLeaderScheduler{ BaseScheduler: NewBaseScheduler(opController), regions: make(chan *core.RegionInfo, transferWitnessLeaderRecvMaxRegionSize), @@ -67,16 +67,16 @@ func (s *trasferWitnessLeaderScheduler) GetType() string { return TransferWitnessLeaderType } -func (s *trasferWitnessLeaderScheduler) IsScheduleAllowed(cluster schedule.Cluster) bool { +func (s *trasferWitnessLeaderScheduler) IsScheduleAllowed(cluster sche.SchedulerCluster) bool { return true } -func (s *trasferWitnessLeaderScheduler) Schedule(cluster schedule.Cluster, dryRun bool) ([]*operator.Operator, []plan.Plan) { +func (s *trasferWitnessLeaderScheduler) Schedule(cluster sche.SchedulerCluster, dryRun bool) ([]*operator.Operator, []plan.Plan) { transferWitnessLeaderCounter.Inc() return s.scheduleTransferWitnessLeaderBatch(s.GetName(), s.GetType(), cluster, transferWitnessLeaderBatchSize), nil } -func (s *trasferWitnessLeaderScheduler) scheduleTransferWitnessLeaderBatch(name, typ string, cluster schedule.Cluster, batchSize int) []*operator.Operator { +func (s *trasferWitnessLeaderScheduler) scheduleTransferWitnessLeaderBatch(name, typ string, cluster sche.SchedulerCluster, batchSize int) []*operator.Operator { var ops []*operator.Operator for i := 0; i < batchSize; i++ { select { @@ -98,7 +98,7 @@ func (s *trasferWitnessLeaderScheduler) scheduleTransferWitnessLeaderBatch(name, return ops } -func (s *trasferWitnessLeaderScheduler) scheduleTransferWitnessLeader(name, typ string, cluster schedule.Cluster, region *core.RegionInfo) (*operator.Operator, error) { +func (s *trasferWitnessLeaderScheduler) scheduleTransferWitnessLeader(name, typ string, cluster sche.SchedulerCluster, region *core.RegionInfo) (*operator.Operator, error) { var filters []filter.Filter unhealthyPeerStores := make(map[uint64]struct{}) for _, peer := range region.GetDownPeers() { @@ -107,8 +107,8 @@ func (s *trasferWitnessLeaderScheduler) scheduleTransferWitnessLeader(name, typ for _, peer := range region.GetPendingPeers() { unhealthyPeerStores[peer.GetStoreId()] = struct{}{} } - filters = append(filters, filter.NewExcludedFilter(name, nil, unhealthyPeerStores), &filter.StoreStateFilter{ActionScope: name, TransferLeader: true}) - candidates := filter.NewCandidates(cluster.GetFollowerStores(region)).FilterTarget(cluster.GetOpts(), nil, nil, filters...) + filters = append(filters, filter.NewExcludedFilter(name, nil, unhealthyPeerStores), &filter.StoreStateFilter{ActionScope: name, TransferLeader: true, OperatorLevel: constant.Urgent}) + candidates := filter.NewCandidates(cluster.GetFollowerStores(region)).FilterTarget(cluster.GetSchedulerConfig(), nil, nil, filters...) // Compatible with old TiKV transfer leader logic. target := candidates.RandomPick() targets := candidates.PickAll() @@ -125,6 +125,6 @@ func (s *trasferWitnessLeaderScheduler) scheduleTransferWitnessLeader(name, typ } // RecvRegionInfo receives a checked region from coordinator -func RecvRegionInfo(s schedule.Scheduler) chan<- *core.RegionInfo { +func RecvRegionInfo(s Scheduler) chan<- *core.RegionInfo { return s.(*trasferWitnessLeaderScheduler).regions } diff --git a/pkg/schedule/schedulers/transfer_witness_leader_test.go b/pkg/schedule/schedulers/transfer_witness_leader_test.go index 54384abb04b..1da968d8dc2 100644 --- a/pkg/schedule/schedulers/transfer_witness_leader_test.go +++ b/pkg/schedule/schedulers/transfer_witness_leader_test.go @@ -21,10 +21,9 @@ import ( "github.com/pingcap/kvproto/pkg/pdpb" "github.com/stretchr/testify/require" "github.com/tikv/pd/pkg/core" - "github.com/tikv/pd/pkg/schedule" "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/storage" - "github.com/tikv/pd/pkg/utils/testutil" + "github.com/tikv/pd/pkg/utils/operatorutil" ) func TestTransferWitnessLeader(t *testing.T) { @@ -39,12 +38,12 @@ func TestTransferWitnessLeader(t *testing.T) { // Add regions 1 with leader in stores 1 tc.AddLeaderRegion(1, 1, 2, 3) - sl, err := schedule.CreateScheduler(TransferWitnessLeaderType, oc, storage.NewStorageWithMemoryBackend(), nil) + sl, err := CreateScheduler(TransferWitnessLeaderType, oc, storage.NewStorageWithMemoryBackend(), nil) re.NoError(err) RecvRegionInfo(sl) <- tc.GetRegion(1) re.True(sl.IsScheduleAllowed(tc)) ops, _ := sl.Schedule(tc, false) - testutil.CheckMultiTargetTransferLeader(re, ops[0], operator.OpLeader, 1, []uint64{2, 3}) + operatorutil.CheckMultiTargetTransferLeader(re, ops[0], operator.OpLeader, 1, []uint64{2, 3}) re.False(ops[0].Step(0).(operator.TransferLeader).IsFinish(tc.MockRegionInfo(1, 1, []uint64{2, 3}, []uint64{}, &metapb.RegionEpoch{ConfVer: 0, Version: 0}))) re.True(ops[0].Step(0).(operator.TransferLeader).IsFinish(tc.MockRegionInfo(1, 2, []uint64{1, 3}, []uint64{}, &metapb.RegionEpoch{ConfVer: 0, Version: 0}))) } @@ -54,7 +53,7 @@ func TestTransferWitnessLeaderWithUnhealthyPeer(t *testing.T) { cancel, _, tc, oc := prepareSchedulersTest() defer cancel() - sl, err := schedule.CreateScheduler(TransferWitnessLeaderType, oc, storage.NewStorageWithMemoryBackend(), nil) + sl, err := CreateScheduler(TransferWitnessLeaderType, oc, storage.NewStorageWithMemoryBackend(), nil) re.NoError(err) // Add stores 1, 2, 3 @@ -74,14 +73,14 @@ func TestTransferWitnessLeaderWithUnhealthyPeer(t *testing.T) { tc.PutRegion(region.Clone(withPendingPeer)) RecvRegionInfo(sl) <- tc.GetRegion(1) ops, _ := sl.Schedule(tc, false) - testutil.CheckMultiTargetTransferLeader(re, ops[0], operator.OpLeader, 1, []uint64{3}) + operatorutil.CheckMultiTargetTransferLeader(re, ops[0], operator.OpLeader, 1, []uint64{3}) ops, _ = sl.Schedule(tc, false) re.Nil(ops) // only down tc.PutRegion(region.Clone(withDownPeer)) RecvRegionInfo(sl) <- tc.GetRegion(1) ops, _ = sl.Schedule(tc, false) - testutil.CheckMultiTargetTransferLeader(re, ops[0], operator.OpLeader, 1, []uint64{2}) + operatorutil.CheckMultiTargetTransferLeader(re, ops[0], operator.OpLeader, 1, []uint64{2}) // pending + down tc.PutRegion(region.Clone(withPendingPeer, withDownPeer)) ops, _ = sl.Schedule(tc, false) diff --git a/pkg/schedule/schedulers/utils.go b/pkg/schedule/schedulers/utils.go index 7f2ba3281f5..998b0307530 100644 --- a/pkg/schedule/schedulers/utils.go +++ b/pkg/schedule/schedulers/utils.go @@ -23,9 +23,10 @@ import ( "github.com/tikv/pd/pkg/core" "github.com/tikv/pd/pkg/core/constant" "github.com/tikv/pd/pkg/errs" - "github.com/tikv/pd/pkg/schedule" + sche "github.com/tikv/pd/pkg/schedule/core" "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/placement" + "github.com/tikv/pd/pkg/schedule/plan" "github.com/tikv/pd/pkg/statistics" "go.uber.org/zap" ) @@ -42,8 +43,8 @@ const ( ) type solver struct { - *balanceSchedulerPlan - schedule.Cluster + *plan.BalanceSchedulerPlan + sche.SchedulerCluster kind constant.ScheduleKind opInfluence operator.OpInfluence tolerantSizeRatio float64 @@ -54,10 +55,10 @@ type solver struct { targetScore float64 } -func newSolver(basePlan *balanceSchedulerPlan, kind constant.ScheduleKind, cluster schedule.Cluster, opInfluence operator.OpInfluence) *solver { +func newSolver(basePlan *plan.BalanceSchedulerPlan, kind constant.ScheduleKind, cluster sche.SchedulerCluster, opInfluence operator.OpInfluence) *solver { return &solver{ - balanceSchedulerPlan: basePlan, - Cluster: cluster, + BalanceSchedulerPlan: basePlan, + SchedulerCluster: cluster, kind: kind, opInfluence: opInfluence, tolerantSizeRatio: adjustTolerantRatio(cluster, kind), @@ -69,7 +70,7 @@ func (p *solver) GetOpInfluence(storeID uint64) int64 { } func (p *solver) SourceStoreID() uint64 { - return p.source.GetID() + return p.Source.GetID() } func (p *solver) SourceMetricLabel() string { @@ -77,7 +78,7 @@ func (p *solver) SourceMetricLabel() string { } func (p *solver) TargetStoreID() uint64 { - return p.target.GetID() + return p.Target.GetID() } func (p *solver) TargetMetricLabel() string { @@ -85,7 +86,7 @@ func (p *solver) TargetMetricLabel() string { } func (p *solver) sourceStoreScore(scheduleName string) float64 { - sourceID := p.source.GetID() + sourceID := p.Source.GetID() tolerantResource := p.getTolerantResource() // to avoid schedule too much, if A's core greater than B and C a little // we want that A should be moved out one region not two @@ -94,8 +95,7 @@ func (p *solver) sourceStoreScore(scheduleName string) float64 { influence = -influence } - opts := p.GetOpts() - if opts.IsDebugMetricsEnabled() { + if p.GetSchedulerConfig().IsDebugMetricsEnabled() { opInfluenceStatus.WithLabelValues(scheduleName, strconv.FormatUint(sourceID, 10), "source").Set(float64(influence)) tolerantResourceStatus.WithLabelValues(scheduleName).Set(float64(tolerantResource)) } @@ -103,19 +103,19 @@ func (p *solver) sourceStoreScore(scheduleName string) float64 { switch p.kind.Resource { case constant.LeaderKind: sourceDelta := influence - tolerantResource - score = p.source.LeaderScore(p.kind.Policy, sourceDelta) + score = p.Source.LeaderScore(p.kind.Policy, sourceDelta) case constant.RegionKind: sourceDelta := influence*influenceAmp - tolerantResource - score = p.source.RegionScore(opts.GetRegionScoreFormulaVersion(), opts.GetHighSpaceRatio(), opts.GetLowSpaceRatio(), sourceDelta) + score = p.Source.RegionScore(p.GetSchedulerConfig().GetRegionScoreFormulaVersion(), p.GetSchedulerConfig().GetHighSpaceRatio(), p.GetSchedulerConfig().GetLowSpaceRatio(), sourceDelta) case constant.WitnessKind: sourceDelta := influence - tolerantResource - score = p.source.WitnessScore(sourceDelta) + score = p.Source.WitnessScore(sourceDelta) } return score } func (p *solver) targetStoreScore(scheduleName string) float64 { - targetID := p.target.GetID() + targetID := p.Target.GetID() // to avoid schedule too much, if A's score less than B and C in small range, // we want that A can be moved in one region not two tolerantResource := p.getTolerantResource() @@ -126,21 +126,20 @@ func (p *solver) targetStoreScore(scheduleName string) float64 { influence = -influence } - opts := p.GetOpts() - if opts.IsDebugMetricsEnabled() { + if p.GetSchedulerConfig().IsDebugMetricsEnabled() { opInfluenceStatus.WithLabelValues(scheduleName, strconv.FormatUint(targetID, 10), "target").Set(float64(influence)) } var score float64 switch p.kind.Resource { case constant.LeaderKind: targetDelta := influence + tolerantResource - score = p.target.LeaderScore(p.kind.Policy, targetDelta) + score = p.Target.LeaderScore(p.kind.Policy, targetDelta) case constant.RegionKind: targetDelta := influence*influenceAmp + tolerantResource - score = p.target.RegionScore(opts.GetRegionScoreFormulaVersion(), opts.GetHighSpaceRatio(), opts.GetLowSpaceRatio(), targetDelta) + score = p.Target.RegionScore(p.GetSchedulerConfig().GetRegionScoreFormulaVersion(), p.GetSchedulerConfig().GetHighSpaceRatio(), p.GetSchedulerConfig().GetLowSpaceRatio(), targetDelta) case constant.WitnessKind: targetDelta := influence + tolerantResource - score = p.target.WitnessScore(targetDelta) + score = p.Target.WitnessScore(targetDelta) } return score } @@ -151,16 +150,16 @@ func (p *solver) shouldBalance(scheduleName string) bool { // The reason we use max(regionSize, averageRegionSize) to check is: // 1. prevent moving small regions between stores with close scores, leading to unnecessary balance. // 2. prevent moving huge regions, leading to over balance. - sourceID := p.source.GetID() - targetID := p.target.GetID() + sourceID := p.Source.GetID() + targetID := p.Target.GetID() // Make sure after move, source score is still greater than target score. shouldBalance := p.sourceScore > p.targetScore if !shouldBalance && log.GetLevel() <= zap.DebugLevel { log.Debug("skip balance "+p.kind.Resource.String(), - zap.String("scheduler", scheduleName), zap.Uint64("region-id", p.region.GetID()), zap.Uint64("source-store", sourceID), zap.Uint64("target-store", targetID), - zap.Int64("source-size", p.source.GetRegionSize()), zap.Float64("source-score", p.sourceScore), - zap.Int64("target-size", p.target.GetRegionSize()), zap.Float64("target-score", p.targetScore), + zap.String("scheduler", scheduleName), zap.Uint64("region-id", p.Region.GetID()), zap.Uint64("source-store", sourceID), zap.Uint64("target-store", targetID), + zap.Int64("source-size", p.Source.GetRegionSize()), zap.Float64("source-score", p.sourceScore), + zap.Int64("target-size", p.Target.GetRegionSize()), zap.Float64("target-score", p.targetScore), zap.Int64("average-region-size", p.GetAverageRegionSize()), zap.Int64("tolerant-resource", p.getTolerantResource())) } @@ -181,14 +180,14 @@ func (p *solver) getTolerantResource() int64 { return p.tolerantSource } -func adjustTolerantRatio(cluster schedule.Cluster, kind constant.ScheduleKind) float64 { +func adjustTolerantRatio(cluster sche.SchedulerCluster, kind constant.ScheduleKind) float64 { var tolerantSizeRatio float64 switch c := cluster.(type) { - case *schedule.RangeCluster: + case *rangeCluster: // range cluster use a separate configuration tolerantSizeRatio = c.GetTolerantSizeRatio() default: - tolerantSizeRatio = cluster.GetOpts().GetTolerantSizeRatio() + tolerantSizeRatio = cluster.GetSchedulerConfig().GetTolerantSizeRatio() } if kind.Resource == constant.LeaderKind && kind.Policy == constant.ByCount { if tolerantSizeRatio == 0 { @@ -236,15 +235,16 @@ func getKeyRanges(args []string) ([]core.KeyRange, error) { type pendingInfluence struct { op *operator.Operator - from, to uint64 + froms []uint64 + to uint64 origin statistics.Influence maxZombieDuration time.Duration } -func newPendingInfluence(op *operator.Operator, from, to uint64, infl statistics.Influence, maxZombieDur time.Duration) *pendingInfluence { +func newPendingInfluence(op *operator.Operator, froms []uint64, to uint64, infl statistics.Influence, maxZombieDur time.Duration) *pendingInfluence { return &pendingInfluence{ op: op, - from: from, + froms: froms, to: to, origin: infl, maxZombieDuration: maxZombieDur, diff --git a/pkg/schedule/region_splitter.go b/pkg/schedule/splitter/region_splitter.go similarity index 92% rename from pkg/schedule/region_splitter.go rename to pkg/schedule/splitter/region_splitter.go index 8817fb56121..f20346d5b59 100644 --- a/pkg/schedule/region_splitter.go +++ b/pkg/schedule/splitter/region_splitter.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package schedule +package splitter import ( "bytes" @@ -25,6 +25,7 @@ import ( "github.com/pingcap/kvproto/pkg/pdpb" "github.com/pingcap/log" "github.com/tikv/pd/pkg/core" + sche "github.com/tikv/pd/pkg/schedule/core" "github.com/tikv/pd/pkg/schedule/filter" "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/utils/logutil" @@ -33,8 +34,10 @@ import ( ) const ( - watchInterval = 100 * time.Millisecond - timeout = time.Minute + watchInterval = 100 * time.Millisecond + timeout = time.Minute + maxSleepDuration = time.Minute + initialSleepDuration = 100 * time.Millisecond ) // SplitRegionsHandler used to handle region splitting @@ -44,7 +47,7 @@ type SplitRegionsHandler interface { } // NewSplitRegionsHandler return SplitRegionsHandler -func NewSplitRegionsHandler(cluster Cluster, oc *OperatorController) SplitRegionsHandler { +func NewSplitRegionsHandler(cluster sche.ClusterInformer, oc *operator.Controller) SplitRegionsHandler { return &splitRegionsHandler{ cluster: cluster, oc: oc, @@ -53,12 +56,12 @@ func NewSplitRegionsHandler(cluster Cluster, oc *OperatorController) SplitRegion // RegionSplitter handles split regions type RegionSplitter struct { - cluster Cluster + cluster sche.ClusterInformer handler SplitRegionsHandler } // NewRegionSplitter return a region splitter -func NewRegionSplitter(cluster Cluster, handler SplitRegionsHandler) *RegionSplitter { +func NewRegionSplitter(cluster sche.ClusterInformer, handler SplitRegionsHandler) *RegionSplitter { return &RegionSplitter{ cluster: cluster, handler: handler, @@ -144,6 +147,9 @@ func (r *RegionSplitter) groupKeysByRegion(keys [][]byte) map[uint64]*regionGrou if !r.checkRegionValid(region) { continue } + if bytes.Equal(region.GetStartKey(), key) { + continue + } log.Info("found region", zap.Uint64("region-id", region.GetID()), logutil.ZapRedactByteString("key", key)) @@ -163,9 +169,6 @@ func (r *RegionSplitter) groupKeysByRegion(keys [][]byte) map[uint64]*regionGrou } func (r *RegionSplitter) checkRegionValid(region *core.RegionInfo) bool { - if r.cluster.IsRegionHot(region) { - return false - } if !filter.IsRegionReplicated(r.cluster, region) { r.cluster.AddSuspectRegions(region.GetID()) return false @@ -177,8 +180,8 @@ func (r *RegionSplitter) checkRegionValid(region *core.RegionInfo) bool { } type splitRegionsHandler struct { - cluster Cluster - oc *OperatorController + cluster sche.ClusterInformer + oc *operator.Controller } func (h *splitRegionsHandler) SplitRegionByKeys(region *core.RegionInfo, splitKeys [][]byte) error { diff --git a/pkg/schedule/region_splitter_test.go b/pkg/schedule/splitter/region_splitter_test.go similarity index 97% rename from pkg/schedule/region_splitter_test.go rename to pkg/schedule/splitter/region_splitter_test.go index eb91421d342..f293446e6cd 100644 --- a/pkg/schedule/region_splitter_test.go +++ b/pkg/schedule/splitter/region_splitter_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package schedule +package splitter import ( "bytes" @@ -122,7 +122,7 @@ func (suite *regionSplitterTestSuite) TestGroupKeysByRegion() { []byte("fff"), []byte("zzz"), }) - suite.Len(groupKeys, 3) + suite.Len(groupKeys, 2) for k, v := range groupKeys { switch k { case uint64(1): @@ -131,9 +131,6 @@ func (suite *regionSplitterTestSuite) TestGroupKeysByRegion() { case uint64(2): suite.Len(v.keys, 1) suite.Equal([]byte("ddd"), v.keys[0]) - case uint64(3): - suite.Len(v.keys, 1) - suite.Equal([]byte("fff"), v.keys[0]) } } } diff --git a/pkg/slice/slice.go b/pkg/slice/slice.go index f26f6ca6459..b3741593670 100644 --- a/pkg/slice/slice.go +++ b/pkg/slice/slice.go @@ -46,3 +46,15 @@ func Contains[T comparable](slice []T, value T) bool { } return false } + +// Remove removes the value from the slice. +func Remove[T comparable](slice []T, value T) []T { + i, j := 0, 0 + for ; i < len(slice); i++ { + if slice[i] != value { + slice[j] = slice[i] + j++ + } + } + return slice[:j] +} diff --git a/pkg/slice/slice_test.go b/pkg/slice/slice_test.go index d8ba709eb66..1fe3fe79dcf 100644 --- a/pkg/slice/slice_test.go +++ b/pkg/slice/slice_test.go @@ -48,14 +48,53 @@ func TestSliceContains(t *testing.T) { t.Parallel() re := require.New(t) ss := []string{"a", "b", "c"} - re.Contains(ss, "a") - re.NotContains(ss, "d") + re.True(slice.Contains(ss, "a")) + re.False(slice.Contains(ss, "d")) us := []uint64{1, 2, 3} - re.Contains(us, uint64(1)) - re.NotContains(us, uint64(4)) + re.True(slice.Contains(us, uint64(1))) + re.False(slice.Contains(us, uint64(4))) is := []int64{1, 2, 3} - re.Contains(is, int64(1)) - re.NotContains(is, int64(4)) + re.True(slice.Contains(is, int64(1))) + re.False(slice.Contains(is, int64(4))) +} + +func TestSliceRemoveGenericTypes(t *testing.T) { + t.Parallel() + re := require.New(t) + ss := []string{"a", "b", "c"} + ss = slice.Remove(ss, "a") + re.Equal([]string{"b", "c"}, ss) + + us := []uint64{1, 2, 3} + us = slice.Remove(us, 1) + re.Equal([]uint64{2, 3}, us) + + is := []int64{1, 2, 3} + is = slice.Remove(is, 1) + re.Equal([]int64{2, 3}, is) +} + +func TestSliceRemove(t *testing.T) { + t.Parallel() + re := require.New(t) + + is := []int64{} + is = slice.Remove(is, 1) + re.Equal([]int64{}, is) + + is = []int64{1} + is = slice.Remove(is, 2) + re.Equal([]int64{1}, is) + is = slice.Remove(is, 1) + re.Equal([]int64{}, is) + + is = []int64{1, 2, 3} + is = slice.Remove(is, 1) + re.Equal([]int64{2, 3}, is) + + is = []int64{1, 1, 1} + is = slice.Remove(is, 1) + re.Equal([]int64{}, is) } diff --git a/pkg/statistics/buckets/bucket_stat_informer.go b/pkg/statistics/buckets/bucket_stat_informer.go index f01ad5f0978..ba917a864c7 100644 --- a/pkg/statistics/buckets/bucket_stat_informer.go +++ b/pkg/statistics/buckets/bucket_stat_informer.go @@ -35,7 +35,7 @@ func init() { // BucketStatInformer is used to get the bucket statistics. type BucketStatInformer interface { - BucketsStats(degree int) map[uint64][]*BucketStat + BucketsStats(degree int, regions ...uint64) map[uint64][]*BucketStat } // BucketStat is the record the bucket statistics. @@ -190,11 +190,13 @@ func (b *BucketTreeItem) calculateHotDegree() { // TODO: qps should be considered, tikv will report this in next sprint // the order: read [bytes keys qps] and write[bytes keys qps] readLoads := stat.Loads[:2] - readHot := slice.AllOf(readLoads, func(i int) bool { + // keep same with the hot region hot degree + // https://github.com/tikv/pd/blob/6f6f545a6716840f7e2c7f4d8ed9b49f613a5cd8/pkg/statistics/hot_peer_cache.go#L220-L222 + readHot := slice.AnyOf(readLoads, func(i int) bool { return readLoads[i] > minHotThresholds[i] }) writeLoads := stat.Loads[3:5] - writeHot := slice.AllOf(writeLoads, func(i int) bool { + writeHot := slice.AnyOf(writeLoads, func(i int) bool { return writeLoads[i] > minHotThresholds[3+i] }) hot := readHot || writeHot diff --git a/pkg/statistics/buckets/hot_bucket_cache.go b/pkg/statistics/buckets/hot_bucket_cache.go index 4de3ec99f7c..f8b666cf4be 100644 --- a/pkg/statistics/buckets/hot_bucket_cache.go +++ b/pkg/statistics/buckets/hot_bucket_cache.go @@ -53,10 +53,10 @@ type HotBucketCache struct { ctx context.Context } -// GetHotBucketStats returns the hot stats of the regions that great than degree. -func (h *HotBucketCache) GetHotBucketStats(degree int) map[uint64][]*BucketStat { +// GetHotBucketStats returns the hot stats of the regionIDs that great than degree. +func (h *HotBucketCache) GetHotBucketStats(degree int, regionIDs []uint64) map[uint64][]*BucketStat { rst := make(map[uint64][]*BucketStat) - for _, item := range h.bucketsOfRegion { + appendItems := func(item *BucketTreeItem) { stats := make([]*BucketStat, 0) for _, b := range item.stats { if b.HotDegree >= degree { @@ -67,6 +67,18 @@ func (h *HotBucketCache) GetHotBucketStats(degree int) map[uint64][]*BucketStat rst[item.regionID] = stats } } + if len(regionIDs) == 0 { + for _, item := range h.bucketsOfRegion { + appendItems(item) + } + } else { + for _, region := range regionIDs { + if item, ok := h.bucketsOfRegion[region]; ok { + appendItems(item) + } + } + } + return rst } @@ -150,6 +162,8 @@ func (h *HotBucketCache) CheckAsync(task flowBucketsItemTask) bool { } func (h *HotBucketCache) schedule() { + defer logutil.LogPanic() + for { select { case <-h.ctx.Done(): diff --git a/pkg/statistics/buckets/hot_bucket_task.go b/pkg/statistics/buckets/hot_bucket_task.go index 4ceecdefd01..d6a43a6f8ae 100644 --- a/pkg/statistics/buckets/hot_bucket_task.go +++ b/pkg/statistics/buckets/hot_bucket_task.go @@ -66,13 +66,15 @@ func (t *checkBucketsTask) runTask(cache *HotBucketCache) { type collectBucketStatsTask struct { minDegree int + regionIDs []uint64 ret chan map[uint64][]*BucketStat // RegionID ==>Buckets } // NewCollectBucketStatsTask creates task to collect bucket stats. -func NewCollectBucketStatsTask(minDegree int) *collectBucketStatsTask { +func NewCollectBucketStatsTask(minDegree int, regionIDs ...uint64) *collectBucketStatsTask { return &collectBucketStatsTask{ minDegree: minDegree, + regionIDs: regionIDs, ret: make(chan map[uint64][]*BucketStat, 1), } } @@ -82,7 +84,7 @@ func (t *collectBucketStatsTask) taskType() flowItemTaskKind { } func (t *collectBucketStatsTask) runTask(cache *HotBucketCache) { - t.ret <- cache.GetHotBucketStats(t.minDegree) + t.ret <- cache.GetHotBucketStats(t.minDegree, t.regionIDs) } // WaitRet returns the result of the task. diff --git a/pkg/statistics/buckets/hot_bucket_task_test.go b/pkg/statistics/buckets/hot_bucket_task_test.go index f2f28ef3d02..49741fdc83f 100644 --- a/pkg/statistics/buckets/hot_bucket_task_test.go +++ b/pkg/statistics/buckets/hot_bucket_task_test.go @@ -68,7 +68,7 @@ func TestCheckBucketsTask(t *testing.T) { ctx, cancelFn := context.WithCancel(context.Background()) defer cancelFn() hotCache := NewBucketsCache(ctx) - // case1： add bucket successfully + // case1: add bucket successfully buckets := newTestBuckets(1, 1, [][]byte{[]byte("10"), []byte("20"), []byte("30")}, 0) task := NewCheckPeerTask(buckets) re.True(hotCache.CheckAsync(task)) @@ -93,7 +93,7 @@ func TestCheckBucketsTask(t *testing.T) { re.Len(item, 1) re.Equal(-2, item[0].HotDegree) - // case3：add bucket successful and the hot degree should inherit from the old one. + // case3: add bucket successful and the hot degree should inherit from the old one. buckets = newTestBuckets(1, 1, [][]byte{[]byte("10"), []byte("20")}, 0) task = NewCheckPeerTask(buckets) re.True(hotCache.CheckAsync(task)) @@ -109,7 +109,7 @@ func TestCollectBucketStatsTask(t *testing.T) { ctx, cancelFn := context.WithCancel(context.Background()) defer cancelFn() hotCache := NewBucketsCache(ctx) - // case1： add bucket successfully + // case1: add bucket successfully for i := uint64(0); i < 10; i++ { buckets := convertToBucketTreeItem(newTestBuckets(i, 1, [][]byte{[]byte(strconv.FormatUint(i*10, 10)), []byte(strconv.FormatUint((i+1)*10, 10))}, 0)) diff --git a/pkg/statistics/hot_cache.go b/pkg/statistics/hot_cache.go index f95683fbb57..42c1e6c49a7 100644 --- a/pkg/statistics/hot_cache.go +++ b/pkg/statistics/hot_cache.go @@ -19,6 +19,7 @@ import ( "github.com/smallnest/chanx" "github.com/tikv/pd/pkg/core" + "github.com/tikv/pd/pkg/utils/logutil" ) const chanMaxLength = 6000000 @@ -129,6 +130,8 @@ func (w *HotCache) ResetMetrics() { } func (w *HotCache) updateItems(queue *chanx.UnboundedChan[FlowItemTask], runTask func(task FlowItemTask)) { + defer logutil.LogPanic() + for { select { case <-w.ctx.Done(): diff --git a/pkg/statistics/hot_peer_cache.go b/pkg/statistics/hot_peer_cache.go index 478a9f506d1..16c64b752e0 100644 --- a/pkg/statistics/hot_peer_cache.go +++ b/pkg/statistics/hot_peer_cache.go @@ -234,8 +234,8 @@ func (f *hotPeerCache) checkPeerFlow(peer *core.PeerInfo, region *core.RegionInf actionType: Update, stores: make([]uint64, len(peers)), } - for _, peer := range peers { - newItem.stores = append(newItem.stores, peer.GetStoreId()) + for i, peer := range peers { + newItem.stores[i] = peer.GetStoreId() } if oldItem == nil { diff --git a/pkg/statistics/hot_regions_stat.go b/pkg/statistics/hot_regions_stat.go index d606a0d8bb4..d30a153492b 100644 --- a/pkg/statistics/hot_regions_stat.go +++ b/pkg/statistics/hot_regions_stat.go @@ -14,11 +14,7 @@ package statistics -import ( - "time" - - "github.com/tikv/pd/pkg/core" -) +import "time" // HotPeersStat records all hot regions statistics type HotPeersStat struct { @@ -44,14 +40,5 @@ type HotPeerStatShow struct { KeyRate float64 `json:"flow_keys"` QueryRate float64 `json:"flow_query"` AntiCount int `json:"anti_count"` - LastUpdateTime time.Time `json:"last_update_time"` -} - -// UpdateHotPeerStatShow updates the region information, such as `IsLearner` and `LastUpdateTime`. -func (h *HotPeerStatShow) UpdateHotPeerStatShow(region *core.RegionInfo) { - if region == nil { - return - } - h.IsLearner = core.IsLearner(region.GetPeer(h.StoreID)) - h.LastUpdateTime = time.Unix(int64(region.GetInterval().GetEndTimestamp()), 0) + LastUpdateTime time.Time `json:"last_update_time,omitempty"` } diff --git a/pkg/statistics/kind.go b/pkg/statistics/kind.go index 66e4fe92b50..6a942e3feac 100644 --- a/pkg/statistics/kind.go +++ b/pkg/statistics/kind.go @@ -170,6 +170,7 @@ type RWType int const ( Write RWType = iota Read + RWTypeLen ) func (rw RWType) String() string { diff --git a/pkg/statistics/metrics.go b/pkg/statistics/metrics.go index bd4c897e258..a5ea07f4f55 100644 --- a/pkg/statistics/metrics.go +++ b/pkg/statistics/metrics.go @@ -41,14 +41,6 @@ var ( Help: "Status of the regions.", }, []string{"type"}) - offlineRegionStatusGauge = prometheus.NewGaugeVec( - prometheus.GaugeOpts{ - Namespace: "pd", - Subsystem: "regions", - Name: "offline_status", - Help: "Status of the offline regions.", - }, []string{"type"}) - clusterStatusGauge = prometheus.NewGaugeVec( prometheus.GaugeOpts{ Namespace: "pd", @@ -190,7 +182,6 @@ func init() { prometheus.MustRegister(hotCacheStatusGauge) prometheus.MustRegister(storeStatusGauge) prometheus.MustRegister(regionStatusGauge) - prometheus.MustRegister(offlineRegionStatusGauge) prometheus.MustRegister(clusterStatusGauge) prometheus.MustRegister(placementStatusGauge) prometheus.MustRegister(configStatusGauge) diff --git a/pkg/statistics/region.go b/pkg/statistics/region.go index 51030b676d2..f39c58ed81c 100644 --- a/pkg/statistics/region.go +++ b/pkg/statistics/region.go @@ -23,6 +23,7 @@ type RegionStats struct { Count int `json:"count"` EmptyCount int `json:"empty_count"` StorageSize int64 `json:"storage_size"` + UserStorageSize int64 `json:"user_storage_size"` StorageKeys int64 `json:"storage_keys"` StoreLeaderCount map[uint64]int `json:"store_leader_count"` StorePeerCount map[uint64]int `json:"store_peer_count"` @@ -57,10 +58,12 @@ func (s *RegionStats) Observe(r *core.RegionInfo) { s.Count++ approximateKeys := r.GetApproximateKeys() approximateSize := r.GetApproximateSize() + approximateKvSize := r.GetApproximateKvSize() if approximateSize <= core.EmptyRegionApproximateSize { s.EmptyCount++ } s.StorageSize += approximateSize + s.UserStorageSize += approximateKvSize s.StorageKeys += approximateKeys leader := r.GetLeader() if leader != nil { diff --git a/pkg/statistics/region_collection.go b/pkg/statistics/region_collection.go index 72c2bc1bb91..29171eb6b7e 100644 --- a/pkg/statistics/region_collection.go +++ b/pkg/statistics/region_collection.go @@ -25,6 +25,12 @@ import ( "github.com/tikv/pd/server/config" ) +// RegionInfoProvider is an interface to provide the region information. +type RegionInfoProvider interface { + // GetRegion returns the region information according to the given region ID. + GetRegion(regionID uint64) *core.RegionInfo +} + // RegionStatisticType represents the type of the region's status. type RegionStatisticType uint32 @@ -42,31 +48,38 @@ const ( WitnessLeader ) +var regionStatisticTypes = []RegionStatisticType{ + MissPeer, + ExtraPeer, + DownPeer, + PendingPeer, + OfflinePeer, + LearnerPeer, + EmptyRegion, + OversizedRegion, + UndersizedRegion, + WitnessLeader, +} + const nonIsolation = "none" var ( // WithLabelValues is a heavy operation, define variable to avoid call it every time. - regionMissPeerRegionCounter = regionStatusGauge.WithLabelValues("miss-peer-region-count") - regionExtraPeerRegionCounter = regionStatusGauge.WithLabelValues("extra-peer-region-count") - regionDownPeerRegionCounter = regionStatusGauge.WithLabelValues("down-peer-region-count") - regionPendingPeerRegionCounter = regionStatusGauge.WithLabelValues("pending-peer-region-count") - regionLearnerPeerRegionCounter = regionStatusGauge.WithLabelValues("learner-peer-region-count") - regionEmptyRegionCounter = regionStatusGauge.WithLabelValues("empty-region-count") - regionOversizedRegionCounter = regionStatusGauge.WithLabelValues("oversized-region-count") - regionUndersizedRegionCounter = regionStatusGauge.WithLabelValues("undersized-region-count") - regionWitnesssLeaderRegionCounter = regionStatusGauge.WithLabelValues("witness-leader-region-count") - - offlineMissPeerRegionCounter = offlineRegionStatusGauge.WithLabelValues("miss-peer-region-count") - offlineExtraPeerRegionCounter = offlineRegionStatusGauge.WithLabelValues("extra-peer-region-count") - offlineDownPeerRegionCounter = offlineRegionStatusGauge.WithLabelValues("down-peer-region-count") - offlinePendingPeerRegionCounter = offlineRegionStatusGauge.WithLabelValues("pending-peer-region-count") - offlineLearnerPeerRegionCounter = offlineRegionStatusGauge.WithLabelValues("learner-peer-region-count") - offlineOfflinePeerRegionCounter = offlineRegionStatusGauge.WithLabelValues("offline-peer-region-count") + regionMissPeerRegionCounter = regionStatusGauge.WithLabelValues("miss-peer-region-count") + regionExtraPeerRegionCounter = regionStatusGauge.WithLabelValues("extra-peer-region-count") + regionDownPeerRegionCounter = regionStatusGauge.WithLabelValues("down-peer-region-count") + regionPendingPeerRegionCounter = regionStatusGauge.WithLabelValues("pending-peer-region-count") + regionOfflinePeerRegionCounter = regionStatusGauge.WithLabelValues("offline-peer-region-count") + regionLearnerPeerRegionCounter = regionStatusGauge.WithLabelValues("learner-peer-region-count") + regionEmptyRegionCounter = regionStatusGauge.WithLabelValues("empty-region-count") + regionOversizedRegionCounter = regionStatusGauge.WithLabelValues("oversized-region-count") + regionUndersizedRegionCounter = regionStatusGauge.WithLabelValues("undersized-region-count") + regionWitnessLeaderRegionCounter = regionStatusGauge.WithLabelValues("witness-leader-region-count") ) -// RegionInfo is used to record the status of region. -type RegionInfo struct { - *core.RegionInfo +// RegionInfoWithTS is used to record the extra timestamp status of a region. +type RegionInfoWithTS struct { + id uint64 startMissVoterPeerTS int64 startDownPeerTS int64 } @@ -74,52 +87,43 @@ type RegionInfo struct { // RegionStatistics is used to record the status of regions. type RegionStatistics struct { sync.RWMutex - conf sc.Config - stats map[RegionStatisticType]map[uint64]*RegionInfo - offlineStats map[RegionStatisticType]map[uint64]*core.RegionInfo + rip RegionInfoProvider + conf sc.CheckerConfigProvider + stats map[RegionStatisticType]map[uint64]*RegionInfoWithTS index map[uint64]RegionStatisticType - offlineIndex map[uint64]RegionStatisticType ruleManager *placement.RuleManager storeConfigManager *config.StoreConfigManager } // NewRegionStatistics creates a new RegionStatistics. -func NewRegionStatistics(conf sc.Config, ruleManager *placement.RuleManager, storeConfigManager *config.StoreConfigManager) *RegionStatistics { +func NewRegionStatistics( + rip RegionInfoProvider, + conf sc.CheckerConfigProvider, + ruleManager *placement.RuleManager, + storeConfigManager *config.StoreConfigManager, +) *RegionStatistics { r := &RegionStatistics{ + rip: rip, conf: conf, ruleManager: ruleManager, storeConfigManager: storeConfigManager, - stats: make(map[RegionStatisticType]map[uint64]*RegionInfo), - offlineStats: make(map[RegionStatisticType]map[uint64]*core.RegionInfo), + stats: make(map[RegionStatisticType]map[uint64]*RegionInfoWithTS), index: make(map[uint64]RegionStatisticType), - offlineIndex: make(map[uint64]RegionStatisticType), } - r.stats[MissPeer] = make(map[uint64]*RegionInfo) - r.stats[ExtraPeer] = make(map[uint64]*RegionInfo) - r.stats[DownPeer] = make(map[uint64]*RegionInfo) - r.stats[PendingPeer] = make(map[uint64]*RegionInfo) - r.stats[LearnerPeer] = make(map[uint64]*RegionInfo) - r.stats[EmptyRegion] = make(map[uint64]*RegionInfo) - r.stats[OversizedRegion] = make(map[uint64]*RegionInfo) - r.stats[UndersizedRegion] = make(map[uint64]*RegionInfo) - r.stats[WitnessLeader] = make(map[uint64]*RegionInfo) - - r.offlineStats[MissPeer] = make(map[uint64]*core.RegionInfo) - r.offlineStats[ExtraPeer] = make(map[uint64]*core.RegionInfo) - r.offlineStats[DownPeer] = make(map[uint64]*core.RegionInfo) - r.offlineStats[PendingPeer] = make(map[uint64]*core.RegionInfo) - r.offlineStats[LearnerPeer] = make(map[uint64]*core.RegionInfo) - r.offlineStats[OfflinePeer] = make(map[uint64]*core.RegionInfo) + for _, typ := range regionStatisticTypes { + r.stats[typ] = make(map[uint64]*RegionInfoWithTS) + } return r } -// GetRegionStatsByType gets the status of the region by types. The regions here need to be cloned, otherwise, it may cause data race problems. +// GetRegionStatsByType gets the status of the region by types. +// The regions here need to be cloned, otherwise, it may cause data race problems. func (r *RegionStatistics) GetRegionStatsByType(typ RegionStatisticType) []*core.RegionInfo { r.RLock() defer r.RUnlock() res := make([]*core.RegionInfo, 0, len(r.stats[typ])) - for _, r := range r.stats[typ] { - res = append(res, r.RegionInfo.Clone()) + for regionID := range r.stats[typ] { + res = append(res, r.rip.GetRegion(regionID).Clone()) } return res } @@ -132,17 +136,6 @@ func (r *RegionStatistics) IsRegionStatsType(regionID uint64, typ RegionStatisti return exist } -// GetOfflineRegionStatsByType gets the status of the offline region by types. The regions here need to be cloned, otherwise, it may cause data race problems. -func (r *RegionStatistics) GetOfflineRegionStatsByType(typ RegionStatisticType) []*core.RegionInfo { - r.RLock() - defer r.RUnlock() - res := make([]*core.RegionInfo, 0, len(r.stats[typ])) - for _, r := range r.offlineStats[typ] { - res = append(res, r.Clone()) - } - return res -} - func (r *RegionStatistics) deleteEntry(deleteIndex RegionStatisticType, regionID uint64) { for typ := RegionStatisticType(1); typ <= deleteIndex; typ <<= 1 { if deleteIndex&typ != 0 { @@ -151,14 +144,6 @@ func (r *RegionStatistics) deleteEntry(deleteIndex RegionStatisticType, regionID } } -func (r *RegionStatistics) deleteOfflineEntry(deleteIndex RegionStatisticType, regionID uint64) { - for typ := RegionStatisticType(1); typ <= deleteIndex; typ <<= 1 { - if deleteIndex&typ != 0 { - delete(r.offlineStats[typ], regionID) - } - } -} - // RegionStatsNeedUpdate checks whether the region's status need to be updated // due to some special state types. func (r *RegionStatistics) RegionStatsNeedUpdate(region *core.RegionInfo) bool { @@ -175,15 +160,13 @@ func (r *RegionStatistics) RegionStatsNeedUpdate(region *core.RegionInfo) bool { func (r *RegionStatistics) Observe(region *core.RegionInfo, stores []*core.StoreInfo) { r.Lock() defer r.Unlock() - // Region state. - regionID := region.GetID() var ( - peerTypeIndex RegionStatisticType - offlinePeerTypeIndex RegionStatisticType - deleteIndex RegionStatisticType + desiredReplicas = r.conf.GetMaxReplicas() + desiredVoters = desiredReplicas + peerTypeIndex RegionStatisticType + deleteIndex RegionStatisticType ) - desiredReplicas := r.conf.GetMaxReplicas() - desiredVoters := desiredReplicas + // Check if the region meets count requirements of its rules. if r.conf.IsPlacementRulesEnabled() { if !r.ruleManager.IsInitialized() { log.Warn("ruleManager haven't been initialized") @@ -199,19 +182,6 @@ func (r *RegionStatistics) Observe(region *core.RegionInfo, stores []*core.Store } } } - - var isRemoving bool - - for _, store := range stores { - if store.IsRemoving() { - peer := region.GetStorePeer(store.GetID()) - if peer != nil { - isRemoving = true - break - } - } - } - // Better to make sure once any of these conditions changes, it will trigger the heartbeat `save_cache`. // Otherwise, the state may be out-of-date for a long time, which needs another way to apply the change ASAP. // For example, see `RegionStatsNeedUpdate` above to know how `OversizedRegion` and `UndersizedRegion` are updated. @@ -220,6 +190,17 @@ func (r *RegionStatistics) Observe(region *core.RegionInfo, stores []*core.Store ExtraPeer: len(region.GetPeers()) > desiredReplicas, DownPeer: len(region.GetDownPeers()) > 0, PendingPeer: len(region.GetPendingPeers()) > 0, + OfflinePeer: func() bool { + for _, store := range stores { + if store.IsRemoving() { + peer := region.GetStorePeer(store.GetID()) + if peer != nil { + return true + } + } + } + return false + }(), LearnerPeer: len(region.GetLearners()) > 0, EmptyRegion: region.GetApproximateSize() <= core.EmptyRegionApproximateSize, OversizedRegion: region.IsOversized( @@ -232,18 +213,13 @@ func (r *RegionStatistics) Observe(region *core.RegionInfo, stores []*core.Store ), WitnessLeader: region.GetLeader().GetIsWitness(), } - + // Check if the region meets any of the conditions and update the corresponding info. + regionID := region.GetID() for typ, c := range conditions { if c { - if isRemoving && typ < EmptyRegion { - r.offlineStats[typ][regionID] = region - offlinePeerTypeIndex |= typ - } info := r.stats[typ][regionID] if info == nil { - info = &RegionInfo{ - RegionInfo: region, - } + info = &RegionInfoWithTS{id: regionID} } if typ == DownPeer { if info.startDownPeerTS != 0 { @@ -263,18 +239,7 @@ func (r *RegionStatistics) Observe(region *core.RegionInfo, stores []*core.Store peerTypeIndex |= typ } } - - if isRemoving { - r.offlineStats[OfflinePeer][regionID] = region - offlinePeerTypeIndex |= OfflinePeer - } - - if oldIndex, ok := r.offlineIndex[regionID]; ok { - deleteIndex = oldIndex &^ offlinePeerTypeIndex - } - r.deleteOfflineEntry(deleteIndex, regionID) - r.offlineIndex[regionID] = offlinePeerTypeIndex - + // Remove the info if any of the conditions are not met any more. if oldIndex, ok := r.index[regionID]; ok { deleteIndex = oldIndex &^ peerTypeIndex } @@ -289,9 +254,6 @@ func (r *RegionStatistics) ClearDefunctRegion(regionID uint64) { if oldIndex, ok := r.index[regionID]; ok { r.deleteEntry(oldIndex, regionID) } - if oldIndex, ok := r.offlineIndex[regionID]; ok { - r.deleteOfflineEntry(oldIndex, regionID) - } } // Collect collects the metrics of the regions' status. @@ -302,24 +264,26 @@ func (r *RegionStatistics) Collect() { regionExtraPeerRegionCounter.Set(float64(len(r.stats[ExtraPeer]))) regionDownPeerRegionCounter.Set(float64(len(r.stats[DownPeer]))) regionPendingPeerRegionCounter.Set(float64(len(r.stats[PendingPeer]))) + regionOfflinePeerRegionCounter.Set(float64(len(r.stats[OfflinePeer]))) regionLearnerPeerRegionCounter.Set(float64(len(r.stats[LearnerPeer]))) regionEmptyRegionCounter.Set(float64(len(r.stats[EmptyRegion]))) regionOversizedRegionCounter.Set(float64(len(r.stats[OversizedRegion]))) regionUndersizedRegionCounter.Set(float64(len(r.stats[UndersizedRegion]))) - regionWitnesssLeaderRegionCounter.Set(float64(len(r.stats[WitnessLeader]))) - - offlineMissPeerRegionCounter.Set(float64(len(r.offlineStats[MissPeer]))) - offlineExtraPeerRegionCounter.Set(float64(len(r.offlineStats[ExtraPeer]))) - offlineDownPeerRegionCounter.Set(float64(len(r.offlineStats[DownPeer]))) - offlinePendingPeerRegionCounter.Set(float64(len(r.offlineStats[PendingPeer]))) - offlineLearnerPeerRegionCounter.Set(float64(len(r.offlineStats[LearnerPeer]))) - offlineOfflinePeerRegionCounter.Set(float64(len(r.offlineStats[OfflinePeer]))) + regionWitnessLeaderRegionCounter.Set(float64(len(r.stats[WitnessLeader]))) } // Reset resets the metrics of the regions' status. func (r *RegionStatistics) Reset() { - regionStatusGauge.Reset() - offlineRegionStatusGauge.Reset() + regionMissPeerRegionCounter.Set(0) + regionExtraPeerRegionCounter.Set(0) + regionDownPeerRegionCounter.Set(0) + regionPendingPeerRegionCounter.Set(0) + regionOfflinePeerRegionCounter.Set(0) + regionLearnerPeerRegionCounter.Set(0) + regionEmptyRegionCounter.Set(0) + regionOversizedRegionCounter.Set(0) + regionUndersizedRegionCounter.Set(0) + regionWitnessLeaderRegionCounter.Set(0) } // LabelStatistics is the statistics of the level of labels. diff --git a/pkg/statistics/region_collection_test.go b/pkg/statistics/region_collection_test.go index 1e071900708..f767c30fcd4 100644 --- a/pkg/statistics/region_collection_test.go +++ b/pkg/statistics/region_collection_test.go @@ -65,14 +65,13 @@ func TestRegionStatistics(t *testing.T) { r2 := &metapb.Region{Id: 2, Peers: peers[0:2], StartKey: []byte("cc"), EndKey: []byte("dd")} region1 := core.NewRegionInfo(r1, peers[0]) region2 := core.NewRegionInfo(r2, peers[0]) - regionStats := NewRegionStatistics(opt, manager, nil) + regionStats := NewRegionStatistics(nil, opt, manager, nil) regionStats.Observe(region1, stores) re.Len(regionStats.stats[ExtraPeer], 1) re.Len(regionStats.stats[LearnerPeer], 1) re.Len(regionStats.stats[EmptyRegion], 1) re.Len(regionStats.stats[UndersizedRegion], 1) - re.Len(regionStats.offlineStats[ExtraPeer], 1) - re.Len(regionStats.offlineStats[LearnerPeer], 1) + re.Len(regionStats.stats[OfflinePeer], 1) region1 = region1.Clone( core.WithDownPeers(downPeers), @@ -88,12 +87,7 @@ func TestRegionStatistics(t *testing.T) { re.Empty(regionStats.stats[EmptyRegion]) re.Len(regionStats.stats[OversizedRegion], 1) re.Empty(regionStats.stats[UndersizedRegion]) - re.Len(regionStats.offlineStats[ExtraPeer], 1) - re.Empty(regionStats.offlineStats[MissPeer]) - re.Len(regionStats.offlineStats[DownPeer], 1) - re.Len(regionStats.offlineStats[PendingPeer], 1) - re.Len(regionStats.offlineStats[LearnerPeer], 1) - re.Len(regionStats.offlineStats[OfflinePeer], 1) + re.Len(regionStats.stats[OfflinePeer], 1) region2 = region2.Clone(core.WithDownPeers(downPeers[0:1])) regionStats.Observe(region2, stores[0:2]) @@ -104,12 +98,7 @@ func TestRegionStatistics(t *testing.T) { re.Len(regionStats.stats[LearnerPeer], 1) re.Len(regionStats.stats[OversizedRegion], 1) re.Len(regionStats.stats[UndersizedRegion], 1) - re.Len(regionStats.offlineStats[ExtraPeer], 1) - re.Empty(regionStats.offlineStats[MissPeer]) - re.Len(regionStats.offlineStats[DownPeer], 1) - re.Len(regionStats.offlineStats[PendingPeer], 1) - re.Len(regionStats.offlineStats[LearnerPeer], 1) - re.Len(regionStats.offlineStats[OfflinePeer], 1) + re.Len(regionStats.stats[OfflinePeer], 1) region1 = region1.Clone(core.WithRemoveStorePeer(7)) regionStats.Observe(region1, stores[0:3]) @@ -118,12 +107,7 @@ func TestRegionStatistics(t *testing.T) { re.Len(regionStats.stats[DownPeer], 2) re.Len(regionStats.stats[PendingPeer], 1) re.Empty(regionStats.stats[LearnerPeer]) - re.Empty(regionStats.offlineStats[ExtraPeer]) - re.Empty(regionStats.offlineStats[MissPeer]) - re.Empty(regionStats.offlineStats[DownPeer]) - re.Empty(regionStats.offlineStats[PendingPeer]) - re.Empty(regionStats.offlineStats[LearnerPeer]) - re.Empty(regionStats.offlineStats[OfflinePeer]) + re.Empty(regionStats.stats[OfflinePeer]) store3 = stores[3].Clone(core.UpStore()) stores[3] = store3 @@ -166,7 +150,7 @@ func TestRegionStatisticsWithPlacementRule(t *testing.T) { region3 := core.NewRegionInfo(r3, peers[0]) region4 := core.NewRegionInfo(r4, peers[0]) region5 := core.NewRegionInfo(r5, peers[4]) - regionStats := NewRegionStatistics(opt, manager, nil) + regionStats := NewRegionStatistics(nil, opt, manager, nil) // r2 didn't match the rules regionStats.Observe(region2, stores) re.Len(regionStats.stats[MissPeer], 1) diff --git a/pkg/statistics/store_collection.go b/pkg/statistics/store_collection.go index 98f6832c05c..c362041ca00 100644 --- a/pkg/statistics/store_collection.go +++ b/pkg/statistics/store_collection.go @@ -20,6 +20,8 @@ import ( "github.com/pingcap/kvproto/pkg/metapb" "github.com/tikv/pd/pkg/core" + "github.com/tikv/pd/pkg/core/constant" + "github.com/tikv/pd/pkg/core/storelimit" "github.com/tikv/pd/server/config" ) @@ -43,6 +45,7 @@ type storeStatistics struct { StorageCapacity uint64 RegionCount int LeaderCount int + LearnerCount int WitnessCount int LabelCounter map[string]int Preparing int @@ -74,9 +77,11 @@ func (s *storeStatistics) Observe(store *core.StoreInfo, stats *StoresStats) { storeAddress := store.GetAddress() id := strconv.FormatUint(store.GetID(), 10) // Store state. + isDown := false switch store.GetNodeState() { case metapb.NodeState_Preparing, metapb.NodeState_Serving: if store.DownTime() >= s.opt.GetMaxStoreDownTime() { + isDown = true s.Down++ } else if store.IsUnhealthy() { s.Unhealthy++ @@ -101,7 +106,8 @@ func (s *storeStatistics) Observe(store *core.StoreInfo, stats *StoresStats) { s.resetStoreStatistics(storeAddress, id) return } - if store.IsLowSpace(s.opt.GetLowSpaceRatio()) { + + if !isDown && store.IsLowSpace(s.opt.GetLowSpaceRatio()) { s.LowSpace++ } @@ -111,6 +117,17 @@ func (s *storeStatistics) Observe(store *core.StoreInfo, stats *StoresStats) { s.RegionCount += store.GetRegionCount() s.LeaderCount += store.GetLeaderCount() s.WitnessCount += store.GetWitnessCount() + s.LearnerCount += store.GetLearnerCount() + limit, ok := store.GetStoreLimit().(*storelimit.SlidingWindows) + if ok { + cap := limit.GetCap() + storeStatusGauge.WithLabelValues(storeAddress, id, "windows_size").Set(float64(cap)) + for i, use := range limit.GetUsed() { + priority := constant.PriorityLevel(i).String() + storeStatusGauge.WithLabelValues(storeAddress, id, "windows_used_level_"+priority).Set(float64(use)) + } + } + // TODO: pre-allocate gauge metrics storeStatusGauge.WithLabelValues(storeAddress, id, "region_score").Set(store.RegionScore(s.opt.GetRegionScoreFormulaVersion(), s.opt.GetHighSpaceRatio(), s.opt.GetLowSpaceRatio(), 0)) storeStatusGauge.WithLabelValues(storeAddress, id, "leader_score").Set(store.LeaderScore(s.opt.GetLeaderSchedulePolicy(), 0)) @@ -119,6 +136,7 @@ func (s *storeStatistics) Observe(store *core.StoreInfo, stats *StoresStats) { storeStatusGauge.WithLabelValues(storeAddress, id, "leader_size").Set(float64(store.GetLeaderSize())) storeStatusGauge.WithLabelValues(storeAddress, id, "leader_count").Set(float64(store.GetLeaderCount())) storeStatusGauge.WithLabelValues(storeAddress, id, "witness_count").Set(float64(store.GetWitnessCount())) + storeStatusGauge.WithLabelValues(storeAddress, id, "learner_count").Set(float64(store.GetLearnerCount())) storeStatusGauge.WithLabelValues(storeAddress, id, "store_available").Set(float64(store.GetAvailable())) storeStatusGauge.WithLabelValues(storeAddress, id, "store_used").Set(float64(store.GetUsedSize())) storeStatusGauge.WithLabelValues(storeAddress, id, "store_capacity").Set(float64(store.GetCapacity())) @@ -177,6 +195,7 @@ func (s *storeStatistics) Collect() { metrics["region_count"] = float64(s.RegionCount) metrics["leader_count"] = float64(s.LeaderCount) metrics["witness_count"] = float64(s.WitnessCount) + metrics["learner_count"] = float64(s.LearnerCount) metrics["storage_size"] = float64(s.StorageSize) metrics["storage_capacity"] = float64(s.StorageCapacity) @@ -248,6 +267,7 @@ func (s *storeStatistics) resetStoreStatistics(storeAddress string, id string) { "leader_size", "leader_count", "witness_count", + "learner_count", "store_available", "store_used", "store_capacity", diff --git a/pkg/statistics/store_collection_test.go b/pkg/statistics/store_collection_test.go index 878ebf65627..76a0dbd1b32 100644 --- a/pkg/statistics/store_collection_test.go +++ b/pkg/statistics/store_collection_test.go @@ -15,12 +15,16 @@ package statistics import ( + "strconv" "testing" "time" + "github.com/docker/go-units" "github.com/pingcap/kvproto/pkg/metapb" + "github.com/pingcap/kvproto/pkg/pdpb" "github.com/stretchr/testify/require" "github.com/tikv/pd/pkg/core" + "github.com/tikv/pd/pkg/core/constant" "github.com/tikv/pd/pkg/mock/mockconfig" ) @@ -54,6 +58,12 @@ func TestStoreStatistics(t *testing.T) { stores[3] = store3 store4 := stores[4].Clone(core.SetLastHeartbeatTS(stores[4].GetLastHeartbeatTS().Add(-time.Hour))) stores[4] = store4 + store5 := stores[5].Clone(core.SetStoreStats(&pdpb.StoreStats{ + Capacity: 512 * units.MiB, + Available: 100 * units.MiB, + UsedSize: 0, + })) + stores[5] = store5 storeStats := NewStoreStatisticsMap(opt, nil) for _, store := range stores { storeStats.Observe(store, storesStats) @@ -72,7 +82,7 @@ func TestStoreStatistics(t *testing.T) { re.Equal(0, stats.Unhealthy) re.Equal(0, stats.Disconnect) re.Equal(1, stats.Tombstone) - re.Equal(8, stats.LowSpace) + re.Equal(1, stats.LowSpace) re.Equal(2, stats.LabelCounter["zone:z1"]) re.Equal(2, stats.LabelCounter["zone:z2"]) re.Equal(2, stats.LabelCounter["zone:z3"]) @@ -80,3 +90,66 @@ func TestStoreStatistics(t *testing.T) { re.Equal(4, stats.LabelCounter["host:h2"]) re.Equal(2, stats.LabelCounter["zone:unknown"]) } + +func TestSummaryStoreInfos(t *testing.T) { + re := require.New(t) + rw := Read + kind := constant.LeaderKind + collector := newTikvCollector() + storeHistoryLoad := NewStoreHistoryLoads(DimLen) + storeInfos := make(map[uint64]*StoreSummaryInfo) + storeLoads := make(map[uint64][]float64) + for _, storeID := range []int{1, 3} { + storeInfos[uint64(storeID)] = &StoreSummaryInfo{ + isTiFlash: false, + StoreInfo: core.NewStoreInfo(&metapb.Store{Id: uint64(storeID), Address: "mock://tikv" + strconv.Itoa(storeID)}, core.SetLastHeartbeatTS(time.Now())), + } + storeLoads[uint64(storeID)] = []float64{1, 2, 0, 0, 5} + for i, v := range storeLoads[uint64(storeID)] { + storeLoads[uint64(storeID)][i] = v * float64(storeID) + } + } + + // case 1: put one element into history load + details := summaryStoresLoadByEngine(storeInfos, storeLoads, storeHistoryLoad, nil, rw, kind, collector) + re.Len(details, 2) + re.Empty(details[0].LoadPred.Current.HistoryLoads) + re.Empty(details[1].LoadPred.Current.HistoryLoads) + expectHistoryLoads := []float64{1, 2, 5} + for _, storeID := range []uint64{1, 3} { + loads := storeHistoryLoad.Get(storeID, rw, kind) + for i := 0; i < len(loads); i++ { + for j := 0; j < len(loads[0]); j++ { + if loads[i][j] != 0 { + re.Equal(loads[i][j]/float64(storeID), expectHistoryLoads[i]) + } + } + } + } + + // case 2: put many elements into history load + historySampleInterval = 0 + for i := 1; i < 10; i++ { + details = summaryStoresLoadByEngine(storeInfos, storeLoads, storeHistoryLoad, nil, rw, kind, collector) + expect := []float64{2, 4, 10} + for _, detail := range details { + loads := detail.LoadPred.Current.HistoryLoads + storeID := detail.GetID() + for i := 0; i < len(loads); i++ { + for j := 0; j < len(loads[0]); j++ { + if loads[i][j] != 0 { + re.Equal(loads[i][j]/float64(storeID), expectHistoryLoads[i]) + } + } + } + + for i, loads := range detail.LoadPred.Expect.HistoryLoads { + for _, load := range loads { + if load != 0 { + re.Equal(load, expect[i]) + } + } + } + } + } +} diff --git a/pkg/statistics/store_hot_peers_infos.go b/pkg/statistics/store_hot_peers_infos.go index ecbee239a69..9480c0ed879 100644 --- a/pkg/statistics/store_hot_peers_infos.go +++ b/pkg/statistics/store_hot_peers_infos.go @@ -75,12 +75,14 @@ func GetHotStatus(stores []*core.StoreInfo, storesLoads map[uint64][]float64, re stLoadInfosAsLeader := SummaryStoresLoad( stInfos, storesLoads, + nil, regionStats, isTraceRegionFlow, typ, constant.LeaderKind) stLoadInfosAsPeer := SummaryStoresLoad( stInfos, storesLoads, + nil, regionStats, isTraceRegionFlow, typ, constant.RegionKind) @@ -105,6 +107,7 @@ func GetHotStatus(stores []*core.StoreInfo, storesLoads map[uint64][]float64, re func SummaryStoresLoad( storeInfos map[uint64]*StoreSummaryInfo, storesLoads map[uint64][]float64, + storesHistoryLoads *StoreHistoryLoads, storeHotPeers map[uint64][]*HotPeerStat, isTraceRegionFlow bool, rwTy RWType, @@ -116,6 +119,7 @@ func SummaryStoresLoad( tikvLoadDetail := summaryStoresLoadByEngine( storeInfos, storesLoads, + storesHistoryLoads, storeHotPeers, rwTy, kind, newTikvCollector(), @@ -123,6 +127,7 @@ func SummaryStoresLoad( tiflashLoadDetail := summaryStoresLoadByEngine( storeInfos, storesLoads, + storesHistoryLoads, storeHotPeers, rwTy, kind, newTiFlashCollector(isTraceRegionFlow), @@ -137,6 +142,7 @@ func SummaryStoresLoad( func summaryStoresLoadByEngine( storeInfos map[uint64]*StoreSummaryInfo, storesLoads map[uint64][]float64, + storesHistoryLoads *StoreHistoryLoads, storeHotPeers map[uint64][]*HotPeerStat, rwTy RWType, kind constant.ResourceKind, @@ -144,6 +150,7 @@ func summaryStoresLoadByEngine( ) []*StoreLoadDetail { loadDetail := make([]*StoreLoadDetail, 0, len(storeInfos)) allStoreLoadSum := make([]float64, DimLen) + allStoreHistoryLoadSum := make([][]float64, DimLen) allStoreCount := 0 allHotPeersCount := 0 @@ -176,8 +183,22 @@ func summaryStoresLoadByEngine( ty = "query-rate-" + rwTy.String() + "-" + kind.String() hotPeerSummary.WithLabelValues(ty, fmt.Sprintf("%v", id)).Set(peerLoadSum[QueryDim]) } - loads := collector.GetLoads(storeLoads, peerLoadSum, rwTy, kind) + + var historyLoads [][]float64 + if storesHistoryLoads != nil { + historyLoads = storesHistoryLoads.Get(id, rwTy, kind) + for i, loads := range historyLoads { + if allStoreHistoryLoadSum[i] == nil || len(allStoreHistoryLoadSum[i]) < len(loads) { + allStoreHistoryLoadSum[i] = make([]float64, len(loads)) + } + for j, load := range loads { + allStoreHistoryLoadSum[i][j] += load + } + } + storesHistoryLoads.Add(id, rwTy, kind, loads) + } + for i := range allStoreLoadSum { allStoreLoadSum[i] += loads[i] } @@ -186,8 +207,9 @@ func summaryStoresLoadByEngine( // Build store load prediction from current load and pending influence. stLoadPred := (&StoreLoad{ - Loads: loads, - Count: float64(len(hotPeers)), + Loads: loads, + Count: float64(len(hotPeers)), + HistoryLoads: historyLoads, }).ToLoadPred(rwTy, info.PendingSum) // Construct store load info. @@ -208,6 +230,14 @@ func summaryStoresLoadByEngine( expectLoads[i] = allStoreLoadSum[i] / float64(allStoreCount) } + // todo: remove some the max value or min value to avoid the effect of extreme value. + expectHistoryLoads := make([][]float64, DimLen) + for i := range allStoreHistoryLoadSum { + expectHistoryLoads[i] = make([]float64, len(allStoreHistoryLoadSum[i])) + for j := range allStoreHistoryLoadSum[i] { + expectHistoryLoads[i][j] = allStoreHistoryLoadSum[i][j] / float64(allStoreCount) + } + } stddevLoads := make([]float64, len(allStoreLoadSum)) if allHotPeersCount != 0 { for _, detail := range loadDetail { @@ -239,8 +269,9 @@ func summaryStoresLoadByEngine( hotPeerSummary.WithLabelValues(ty, engine).Set(stddevLoads[QueryDim]) } expect := StoreLoad{ - Loads: expectLoads, - Count: expectCount, + Loads: expectLoads, + Count: expectCount, + HistoryLoads: expectHistoryLoads, } stddev := StoreLoad{ Loads: stddevLoads, diff --git a/pkg/statistics/store_load.go b/pkg/statistics/store_load.go index 13c4aa57635..d110ec4a7f3 100644 --- a/pkg/statistics/store_load.go +++ b/pkg/statistics/store_load.go @@ -16,8 +16,10 @@ package statistics import ( "math" + "time" "github.com/tikv/pd/pkg/core" + "github.com/tikv/pd/pkg/core/constant" ) // StoreLoadDetail records store load information. @@ -144,8 +146,9 @@ func (s *StoreSummaryInfo) SetEngineAsTiFlash() { // StoreLoad records the current load. type StoreLoad struct { - Loads []float64 - Count float64 + Loads []float64 + Count float64 + HistoryLoads [][]float64 } // ToLoadPred returns the current load and future predictive load. @@ -240,3 +243,91 @@ func MaxLoad(a, b *StoreLoad) *StoreLoad { Count: math.Max(a.Count, b.Count), } } + +var ( + // historySampleInterval is the sampling interval for history load. + historySampleInterval = 30 * time.Second + // HistorySampleDuration is the duration for saving history load. + HistorySampleDuration = 5 * time.Minute + defaultSize = 10 +) + +// StoreHistoryLoads records the history load of a store. +type StoreHistoryLoads struct { + // loads[read/write][leader/follower]-->[store id]-->history load + loads [RWTypeLen][constant.ResourceKindLen]map[uint64]*storeHistoryLoad + dim int +} + +// NewStoreHistoryLoads creates a StoreHistoryLoads. +func NewStoreHistoryLoads(dim int) *StoreHistoryLoads { + st := StoreHistoryLoads{dim: dim} + for i := RWType(0); i < RWTypeLen; i++ { + for j := constant.ResourceKind(0); j < constant.ResourceKindLen; j++ { + st.loads[i][j] = make(map[uint64]*storeHistoryLoad) + } + } + return &st +} + +// Add adds the store load to the history. +func (s *StoreHistoryLoads) Add(storeID uint64, rwTp RWType, kind constant.ResourceKind, loads []float64) { + load, ok := s.loads[rwTp][kind][storeID] + if !ok { + size := defaultSize + if historySampleInterval != 0 { + size = int(HistorySampleDuration / historySampleInterval) + } + load = newStoreHistoryLoad(size, s.dim) + s.loads[rwTp][kind][storeID] = load + } + load.add(loads) +} + +// Get returns the store loads from the history, not one time point. +func (s *StoreHistoryLoads) Get(storeID uint64, rwTp RWType, kind constant.ResourceKind) [][]float64 { + load, ok := s.loads[rwTp][kind][storeID] + if !ok { + return [][]float64{} + } + return load.get() +} + +type storeHistoryLoad struct { + update time.Time + // loads is a circular buffer. + // [dim] --> [1,2,3...] + loads [][]float64 + size int + count int +} + +func newStoreHistoryLoad(size int, dim int) *storeHistoryLoad { + return &storeHistoryLoad{ + loads: make([][]float64, dim), + size: size, + } +} + +// add adds the store load to the history. +// eg. add([1,2,3]) --> [][]float64{{1}, {2}, {3}} +func (s *storeHistoryLoad) add(loads []float64) { + // reject if the loads length is not equal to the dimension. + if time.Since(s.update) < historySampleInterval || s.size == 0 || len(loads) != len(s.loads) { + return + } + if s.count == 0 { + for i := range s.loads { + s.loads[i] = make([]float64, s.size) + } + } + for i, v := range loads { + s.loads[i][s.count%s.size] = v + } + s.count++ + s.update = time.Now() +} + +func (s *storeHistoryLoad) get() [][]float64 { + return s.loads +} diff --git a/pkg/statistics/store_load_test.go b/pkg/statistics/store_load_test.go new file mode 100644 index 00000000000..18441f00dbc --- /dev/null +++ b/pkg/statistics/store_load_test.go @@ -0,0 +1,45 @@ +// Copyright 2023 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package statistics + +import ( + "testing" + + "github.com/stretchr/testify/require" + "github.com/tikv/pd/pkg/core/constant" +) + +func TestHistoryLoads(t *testing.T) { + re := require.New(t) + historySampleInterval = 0 + historyLoads := NewStoreHistoryLoads(DimLen) + loads := []float64{1.0, 2.0, 3.0} + rwTp := Read + kind := constant.LeaderKind + historyLoads.Add(1, rwTp, kind, loads) + re.Len(historyLoads.Get(1, rwTp, kind)[0], 10) + + expectLoads := make([][]float64, DimLen) + for i := 0; i < len(loads); i++ { + expectLoads[i] = make([]float64, 10) + } + for i := 0; i < 10; i++ { + historyLoads.Add(1, rwTp, kind, loads) + expectLoads[ByteDim][i] = 1.0 + expectLoads[KeyDim][i] = 2.0 + expectLoads[QueryDim][i] = 3.0 + } + re.EqualValues(expectLoads, historyLoads.Get(1, rwTp, kind)) +} diff --git a/pkg/storage/endpoint/gc_key_space.go b/pkg/storage/endpoint/gc_key_space.go deleted file mode 100644 index 66bf505583a..00000000000 --- a/pkg/storage/endpoint/gc_key_space.go +++ /dev/null @@ -1,198 +0,0 @@ -// Copyright 2022 TiKV Project Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package endpoint - -import ( - "encoding/json" - "math" - "strconv" - "strings" - "time" - - "github.com/pingcap/errors" - "github.com/pingcap/failpoint" - "github.com/pingcap/log" - "github.com/tikv/pd/pkg/errs" - "go.etcd.io/etcd/clientv3" - "go.uber.org/zap" -) - -// KeyspaceGCSafePoint is gcWorker's safepoint for specific key-space -type KeyspaceGCSafePoint struct { - SpaceID string `json:"space_id"` - SafePoint uint64 `json:"safe_point,omitempty"` -} - -// KeyspaceGCSafePointStorage defines the storage operations on Keyspaces' safe points -type KeyspaceGCSafePointStorage interface { - // Service safe point interfaces. - SaveServiceSafePoint(spaceID string, ssp *ServiceSafePoint) error - LoadServiceSafePoint(spaceID, serviceID string) (*ServiceSafePoint, error) - LoadMinServiceSafePoint(spaceID string, now time.Time) (*ServiceSafePoint, error) - RemoveServiceSafePoint(spaceID, serviceID string) error - // GC safe point interfaces. - SaveKeyspaceGCSafePoint(spaceID string, safePoint uint64) error - LoadKeyspaceGCSafePoint(spaceID string) (uint64, error) - LoadAllKeyspaceGCSafePoints(withGCSafePoint bool) ([]*KeyspaceGCSafePoint, error) -} - -var _ KeyspaceGCSafePointStorage = (*StorageEndpoint)(nil) - -// SaveServiceSafePoint saves service safe point under given key-space. -func (se *StorageEndpoint) SaveServiceSafePoint(spaceID string, ssp *ServiceSafePoint) error { - if ssp.ServiceID == "" { - return errors.New("service id of service safepoint cannot be empty") - } - key := KeyspaceServiceSafePointPath(spaceID, ssp.ServiceID) - value, err := json.Marshal(ssp) - if err != nil { - return err - } - return se.Save(key, string(value)) -} - -// LoadServiceSafePoint reads ServiceSafePoint for the given key-space ID and service name. -// Return nil if no safepoint exist for given service or just expired. -func (se *StorageEndpoint) LoadServiceSafePoint(spaceID, serviceID string) (*ServiceSafePoint, error) { - key := KeyspaceServiceSafePointPath(spaceID, serviceID) - value, err := se.Load(key) - if err != nil || value == "" { - return nil, err - } - ssp := &ServiceSafePoint{} - if err := json.Unmarshal([]byte(value), ssp); err != nil { - return nil, err - } - if ssp.ExpiredAt < time.Now().Unix() { - go func() { - if err = se.Remove(key); err != nil { - log.Error("remove expired key meet error", zap.String("key", key), errs.ZapError(err)) - } - }() - return nil, nil - } - return ssp, nil -} - -// LoadMinServiceSafePoint returns the minimum safepoint for the given key-space. -// Note that gc worker safe point are store separately. -// If no service safe point exist for the given key-space or all the service safe points just expired, return nil. -func (se *StorageEndpoint) LoadMinServiceSafePoint(spaceID string, now time.Time) (*ServiceSafePoint, error) { - prefix := KeyspaceServiceSafePointPrefix(spaceID) - prefixEnd := clientv3.GetPrefixRangeEnd(prefix) - keys, values, err := se.LoadRange(prefix, prefixEnd, 0) - if err != nil { - return nil, err - } - min := &ServiceSafePoint{SafePoint: math.MaxUint64} - expiredKeys := make([]string, 0) - for i, key := range keys { - ssp := &ServiceSafePoint{} - if err = json.Unmarshal([]byte(values[i]), ssp); err != nil { - return nil, err - } - - // gather expired keys - if ssp.ExpiredAt < now.Unix() { - expiredKeys = append(expiredKeys, key) - continue - } - if ssp.SafePoint < min.SafePoint { - min = ssp - } - } - // failpoint for immediate removal - failpoint.Inject("removeExpiredKeys", func() { - for _, key := range expiredKeys { - if err = se.Remove(key); err != nil { - log.Error("remove expired key meet error", zap.String("key", key), errs.ZapError(err)) - } - } - expiredKeys = []string{} - }) - // remove expired keys asynchronously - go func() { - for _, key := range expiredKeys { - if err = se.Remove(key); err != nil { - log.Error("remove expired key meet error", zap.String("key", key), errs.ZapError(err)) - } - } - }() - if min.SafePoint == math.MaxUint64 { - // no service safe point or all of them are expired. - return nil, nil - } - - // successfully found a valid min safe point. - return min, nil -} - -// RemoveServiceSafePoint removes target ServiceSafePoint -func (se *StorageEndpoint) RemoveServiceSafePoint(spaceID, serviceID string) error { - key := KeyspaceServiceSafePointPath(spaceID, serviceID) - return se.Remove(key) -} - -// SaveKeyspaceGCSafePoint saves GCSafePoint to the given key-space. -func (se *StorageEndpoint) SaveKeyspaceGCSafePoint(spaceID string, safePoint uint64) error { - value := strconv.FormatUint(safePoint, 16) - return se.Save(KeyspaceGCSafePointPath(spaceID), value) -} - -// LoadKeyspaceGCSafePoint reads GCSafePoint for the given key-space. -// Returns 0 if target safepoint not exist. -func (se *StorageEndpoint) LoadKeyspaceGCSafePoint(spaceID string) (uint64, error) { - value, err := se.Load(KeyspaceGCSafePointPath(spaceID)) - if err != nil || value == "" { - return 0, err - } - safePoint, err := strconv.ParseUint(value, 16, 64) - if err != nil { - return 0, err - } - return safePoint, nil -} - -// LoadAllKeyspaceGCSafePoints returns slice of KeyspaceGCSafePoint. -// If withGCSafePoint set to false, returned safePoints will be 0. -func (se *StorageEndpoint) LoadAllKeyspaceGCSafePoints(withGCSafePoint bool) ([]*KeyspaceGCSafePoint, error) { - prefix := KeyspaceSafePointPrefix() - prefixEnd := clientv3.GetPrefixRangeEnd(prefix) - suffix := KeyspaceGCSafePointSuffix() - keys, values, err := se.LoadRange(prefix, prefixEnd, 0) - if err != nil { - return nil, err - } - safePoints := make([]*KeyspaceGCSafePoint, 0, len(values)) - for i := range keys { - // skip non gc safe points - if !strings.HasSuffix(keys[i], suffix) { - continue - } - safePoint := &KeyspaceGCSafePoint{} - spaceID := strings.TrimPrefix(keys[i], prefix) - spaceID = strings.TrimSuffix(spaceID, suffix) - safePoint.SpaceID = spaceID - if withGCSafePoint { - value, err := strconv.ParseUint(values[i], 16, 64) - if err != nil { - return nil, err - } - safePoint.SafePoint = value - } - safePoints = append(safePoints, safePoint) - } - return safePoints, nil -} diff --git a/pkg/storage/endpoint/gc_safe_point.go b/pkg/storage/endpoint/gc_safe_point.go index e213eca4ed5..db5c58205c8 100644 --- a/pkg/storage/endpoint/gc_safe_point.go +++ b/pkg/storage/endpoint/gc_safe_point.go @@ -87,7 +87,7 @@ func (se *StorageEndpoint) LoadMinServiceGCSafePoint(now time.Time) (*ServiceSaf if err := json.Unmarshal([]byte(values[i]), ssp); err != nil { return nil, err } - if ssp.ServiceID == gcWorkerServiceSafePointID { + if ssp.ServiceID == GCWorkerServiceSafePointID { hasGCWorker = true // If gc_worker's expire time is incorrectly set, fix it. if ssp.ExpiredAt != math.MaxInt64 { @@ -125,7 +125,7 @@ func (se *StorageEndpoint) LoadMinServiceGCSafePoint(now time.Time) (*ServiceSaf func (se *StorageEndpoint) initServiceGCSafePointForGCWorker(initialValue uint64) (*ServiceSafePoint, error) { ssp := &ServiceSafePoint{ - ServiceID: gcWorkerServiceSafePointID, + ServiceID: GCWorkerServiceSafePointID, SafePoint: initialValue, ExpiredAt: math.MaxInt64, } @@ -165,7 +165,7 @@ func (se *StorageEndpoint) SaveServiceGCSafePoint(ssp *ServiceSafePoint) error { return errors.New("service id of service safepoint cannot be empty") } - if ssp.ServiceID == gcWorkerServiceSafePointID && ssp.ExpiredAt != math.MaxInt64 { + if ssp.ServiceID == GCWorkerServiceSafePointID && ssp.ExpiredAt != math.MaxInt64 { return errors.New("TTL of gc_worker's service safe point must be infinity") } @@ -180,7 +180,7 @@ func (se *StorageEndpoint) SaveServiceGCSafePoint(ssp *ServiceSafePoint) error { // RemoveServiceGCSafePoint removes a GC safepoint for the service func (se *StorageEndpoint) RemoveServiceGCSafePoint(serviceID string) error { - if serviceID == gcWorkerServiceSafePointID { + if serviceID == GCWorkerServiceSafePointID { return errors.New("cannot remove service safe point of gc_worker") } key := gcSafePointServicePath(serviceID) diff --git a/pkg/storage/endpoint/key_path.go b/pkg/storage/endpoint/key_path.go index 4b1555ad849..08aebb400d9 100644 --- a/pkg/storage/endpoint/key_path.go +++ b/pkg/storage/endpoint/key_path.go @@ -17,22 +17,27 @@ package endpoint import ( "fmt" "path" + "regexp" "strconv" "strings" + + "github.com/tikv/pd/pkg/mcs/utils" ) const ( - clusterPath = "raft" - configPath = "config" - serviceMiddlewarePath = "service_middleware" - schedulePath = "schedule" - gcPath = "gc" - rulesPath = "rules" - ruleGroupPath = "rule_group" - regionLabelPath = "region_label" - replicationPath = "replication_mode" - customScheduleConfigPath = "scheduler_config" - gcWorkerServiceSafePointID = "gc_worker" + pdRootPath = "/pd" + clusterPath = "raft" + configPath = "config" + serviceMiddlewarePath = "service_middleware" + schedulePath = "schedule" + gcPath = "gc" + rulesPath = "rules" + ruleGroupPath = "rule_group" + regionLabelPath = "region_label" + replicationPath = "replication_mode" + customScheduleConfigPath = "scheduler_config" + // GCWorkerServiceSafePointID is the service id of GC worker. + GCWorkerServiceSafePointID = "gc_worker" minResolvedTS = "min_resolved_ts" externalTimeStamp = "external_timestamp" keyspaceSafePointPrefix = "keyspaces/gc_safepoint" @@ -41,20 +46,32 @@ const ( keyspaceMetaInfix = "meta" keyspaceIDInfix = "id" keyspaceAllocID = "alloc_id" + gcSafePointInfix = "gc_safe_point" + serviceSafePointInfix = "service_safe_point" regionPathPrefix = "raft/r" // resource group storage endpoint has prefix `resource_group` resourceGroupSettingsPath = "settings" resourceGroupStatesPath = "states" - requestUnitConfigPath = "ru_config" + controllerConfigPath = "controller" // tso storage endpoint has prefix `tso` - microserviceKey = "microservice" - tsoServiceKey = "tso" - timestampKey = "timestamp" + tsoServiceKey = utils.TSOServiceName + globalTSOAllocatorEtcdPrefix = "gta" + // TimestampKey is the key of timestamp oracle used for the suffix. + TimestampKey = "timestamp" + + tsoKeyspaceGroupPrefix = tsoServiceKey + "/" + utils.KeyspaceGroupsKey + keyspaceGroupsMembershipKey = "membership" + keyspaceGroupsElectionKey = "election" // we use uint64 to represent ID, the max length of uint64 is 20. keyLen = 20 ) +// PDRootPath returns the PD root path. +func PDRootPath(clusterID uint64) string { + return path.Join(pdRootPath, strconv.FormatUint(clusterID, 10)) +} + // AppendToRootPath appends the given key to the rootPath. func AppendToRootPath(rootPath string, key string) string { return path.Join(rootPath, key) @@ -70,6 +87,11 @@ func ClusterBootstrapTimeKey() string { return path.Join(clusterPath, "status", "raft_bootstrap_time") } +// ConfigPath returns the path to save the PD config. +func ConfigPath(clusterID uint64) string { + return path.Join(PDRootPath(clusterID), configPath) +} + func scheduleConfigPath(scheduleName string) string { return path.Join(customScheduleConfigPath, scheduleName) } @@ -159,34 +181,29 @@ func ExternalTimestampPath() string { return path.Join(clusterPath, externalTimeStamp) } -// KeyspaceServiceSafePointPrefix returns the prefix of given service's service safe point. -// Prefix: /keyspaces/gc_safepoint/{space_id}/service/ -func KeyspaceServiceSafePointPrefix(spaceID string) string { - return path.Join(keyspaceSafePointPrefix, spaceID, "service") + "/" +// GCSafePointV2Path is the storage path of gc safe point v2. +// Path: keyspaces/gc_safe_point/{keyspaceID} +func GCSafePointV2Path(keyspaceID uint32) string { + return buildPath(false, keyspacePrefix, gcSafePointInfix, EncodeKeyspaceID(keyspaceID)) } -// KeyspaceGCSafePointPath returns the gc safe point's path of the given key-space. -// Path: /keyspaces/gc_safepoint/{space_id}/gc -func KeyspaceGCSafePointPath(spaceID string) string { - return path.Join(keyspaceSafePointPrefix, spaceID, keyspaceGCSafePointSuffix) +// GCSafePointV2Prefix is the path prefix to all gc safe point v2. +// Prefix: keyspaces/gc_safe_point/ +func GCSafePointV2Prefix() string { + return buildPath(true, keyspacePrefix, gcSafePointInfix) } -// KeyspaceServiceSafePointPath returns the path of given service's service safe point. -// Path: /keyspaces/gc_safepoint/{space_id}/service/{service_id} -func KeyspaceServiceSafePointPath(spaceID, serviceID string) string { - return path.Join(KeyspaceServiceSafePointPrefix(spaceID), serviceID) +// ServiceSafePointV2Path is the storage path of service safe point v2. +// Path: keyspaces/service_safe_point/{spaceID}/{serviceID} +func ServiceSafePointV2Path(keyspaceID uint32, serviceID string) string { + return buildPath(false, keyspacePrefix, serviceSafePointInfix, EncodeKeyspaceID(keyspaceID), serviceID) } -// KeyspaceSafePointPrefix returns prefix for all key-spaces' safe points. -// Path: /keyspaces/gc_safepoint/ -func KeyspaceSafePointPrefix() string { - return keyspaceSafePointPrefix + "/" -} - -// KeyspaceGCSafePointSuffix returns the suffix for any gc safepoint. -// Postfix: /gc -func KeyspaceGCSafePointSuffix() string { - return "/" + keyspaceGCSafePointSuffix +// ServiceSafePointV2Prefix is the path prefix of all service safe point that belongs to a specific keyspace. +// Can be used to retrieve keyspace's service safe point at once. +// Path: keyspaces/service_safe_point/{spaceID}/ +func ServiceSafePointV2Prefix(keyspaceID uint32) string { + return buildPath(true, keyspacePrefix, serviceSafePointInfix, EncodeKeyspaceID(keyspaceID)) } // KeyspaceMetaPrefix returns the prefix of keyspaces' metadata. @@ -198,7 +215,7 @@ func KeyspaceMetaPrefix() string { // KeyspaceMetaPath returns the path to the given keyspace's metadata. // Path: keyspaces/meta/{space_id} func KeyspaceMetaPath(spaceID uint32) string { - idStr := encodeKeyspaceID(spaceID) + idStr := EncodeKeyspaceID(spaceID) return path.Join(KeyspaceMetaPrefix(), idStr) } @@ -214,10 +231,132 @@ func KeyspaceIDAlloc() string { return path.Join(keyspacePrefix, keyspaceAllocID) } -// encodeKeyspaceID from uint32 to string. +// EncodeKeyspaceID from uint32 to string. // It adds extra padding to make encoded ID ordered. // Encoded ID can be decoded directly with strconv.ParseUint. // Width of the padded keyspaceID is 8 (decimal representation of uint24max is 16777215). -func encodeKeyspaceID(spaceID uint32) string { +func EncodeKeyspaceID(spaceID uint32) string { return fmt.Sprintf("%08d", spaceID) } + +// KeyspaceGroupIDPrefix returns the prefix of keyspace group id. +// Path: tso/keyspace_groups/membership +func KeyspaceGroupIDPrefix() string { + return path.Join(tsoKeyspaceGroupPrefix, keyspaceGroupsMembershipKey) +} + +// KeyspaceGroupIDPath returns the path to keyspace id from the given name. +// Path: tso/keyspace_groups/membership/{id} +func KeyspaceGroupIDPath(id uint32) string { + return path.Join(tsoKeyspaceGroupPrefix, keyspaceGroupsMembershipKey, encodeKeyspaceGroupID(id)) +} + +// GetCompiledKeyspaceGroupIDRegexp returns the compiled regular expression for matching keyspace group id. +func GetCompiledKeyspaceGroupIDRegexp() *regexp.Regexp { + pattern := strings.Join([]string{KeyspaceGroupIDPrefix(), `(\d{5})$`}, "/") + return regexp.MustCompile(pattern) +} + +// ResourceManagerSvcRootPath returns the root path of resource manager service. +// Path: /ms/{cluster_id}/resource_manager +func ResourceManagerSvcRootPath(clusterID uint64) string { + return svcRootPath(clusterID, utils.ResourceManagerServiceName) +} + +// SchedulingSvcRootPath returns the root path of scheduling service. +// Path: /ms/{cluster_id}/scheduling +func SchedulingSvcRootPath(clusterID uint64) string { + return svcRootPath(clusterID, utils.SchedulingServiceName) +} + +// TSOSvcRootPath returns the root path of tso service. +// Path: /ms/{cluster_id}/tso +func TSOSvcRootPath(clusterID uint64) string { + return svcRootPath(clusterID, utils.TSOServiceName) +} + +func svcRootPath(clusterID uint64, svcName string) string { + c := strconv.FormatUint(clusterID, 10) + return path.Join(utils.MicroserviceRootPath, c, svcName) +} + +// LegacyRootPath returns the root path of legacy pd service. +// Path: /pd/{cluster_id} +func LegacyRootPath(clusterID uint64) string { + return path.Join(pdRootPath, strconv.FormatUint(clusterID, 10)) +} + +// KeyspaceGroupPrimaryPath returns the path of keyspace group primary. +// default keyspace group: "/ms/{cluster_id}/tso/00000/primary". +// non-default keyspace group: "/ms/{cluster_id}/tso/keyspace_groups/election/{group}/primary". +func KeyspaceGroupPrimaryPath(rootPath string, keyspaceGroupID uint32) string { + electionPath := KeyspaceGroupsElectionPath(rootPath, keyspaceGroupID) + return path.Join(electionPath, utils.PrimaryKey) +} + +// KeyspaceGroupsElectionPath returns the path of keyspace groups election. +// default keyspace group: "/ms/{cluster_id}/tso/00000". +// non-default keyspace group: "/ms/{cluster_id}/tso/keyspace_groups/election/{group}". +func KeyspaceGroupsElectionPath(rootPath string, keyspaceGroupID uint32) string { + if keyspaceGroupID == utils.DefaultKeyspaceGroupID { + return path.Join(rootPath, "00000") + } + return path.Join(rootPath, utils.KeyspaceGroupsKey, keyspaceGroupsElectionKey, fmt.Sprintf("%05d", keyspaceGroupID)) +} + +// GetCompiledNonDefaultIDRegexp returns the compiled regular expression for matching non-default keyspace group id. +func GetCompiledNonDefaultIDRegexp(clusterID uint64) *regexp.Regexp { + rootPath := TSOSvcRootPath(clusterID) + pattern := strings.Join([]string{rootPath, utils.KeyspaceGroupsKey, keyspaceGroupsElectionKey, `(\d{5})`, utils.PrimaryKey + `$`}, "/") + return regexp.MustCompile(pattern) +} + +// encodeKeyspaceGroupID from uint32 to string. +func encodeKeyspaceGroupID(groupID uint32) string { + return fmt.Sprintf("%05d", groupID) +} + +func buildPath(withSuffix bool, str ...string) string { + var sb strings.Builder + for i := 0; i < len(str); i++ { + if i != 0 { + sb.WriteString("/") + } + sb.WriteString(str[i]) + } + if withSuffix { + sb.WriteString("/") + } + return sb.String() +} + +// KeyspaceGroupTSPath constructs the timestampOracle path prefix, which is: +// 1. for the default keyspace group: +// "" in /pd/{cluster_id}/timestamp +// 2. for the non-default keyspace groups: +// {group}/gta in /ms/{cluster_id}/tso/{group}/gta/timestamp +func KeyspaceGroupTSPath(groupID uint32) string { + if groupID == utils.DefaultKeyspaceGroupID { + return "" + } + return path.Join(fmt.Sprintf("%05d", groupID), globalTSOAllocatorEtcdPrefix) +} + +// TimestampPath returns the timestamp path for the given timestamp oracle path prefix. +func TimestampPath(tsPath string) string { + return path.Join(tsPath, TimestampKey) +} + +// FullTimestampPath returns the full timestamp path. +// 1. for the default keyspace group: +// /pd/{cluster_id}/timestamp +// 2. for the non-default keyspace groups: +// /ms/{cluster_id}/tso/{group}/gta/timestamp +func FullTimestampPath(clusterID uint64, groupID uint32) string { + rootPath := TSOSvcRootPath(clusterID) + tsPath := TimestampPath(KeyspaceGroupTSPath(groupID)) + if groupID == utils.DefaultKeyspaceGroupID { + rootPath = LegacyRootPath(clusterID) + } + return path.Join(rootPath, tsPath) +} diff --git a/pkg/storage/endpoint/keyspace.go b/pkg/storage/endpoint/keyspace.go index 7aa82e8985b..09733ad59c1 100644 --- a/pkg/storage/endpoint/keyspace.go +++ b/pkg/storage/endpoint/keyspace.go @@ -41,7 +41,7 @@ type KeyspaceStorage interface { SaveKeyspaceID(txn kv.Txn, id uint32, name string) error LoadKeyspaceID(txn kv.Txn, name string) (bool, uint32, error) // LoadRangeKeyspace loads no more than limit keyspaces starting at startID. - LoadRangeKeyspace(startID uint32, limit int) ([]*keyspacepb.KeyspaceMeta, error) + LoadRangeKeyspace(txn kv.Txn, startID uint32, limit int) ([]*keyspacepb.KeyspaceMeta, error) RunInTxn(ctx context.Context, f func(txn kv.Txn) error) error } @@ -104,10 +104,10 @@ func (se *StorageEndpoint) RunInTxn(ctx context.Context, f func(txn kv.Txn) erro // LoadRangeKeyspace loads keyspaces starting at startID. // limit specifies the limit of loaded keyspaces. -func (se *StorageEndpoint) LoadRangeKeyspace(startID uint32, limit int) ([]*keyspacepb.KeyspaceMeta, error) { +func (se *StorageEndpoint) LoadRangeKeyspace(txn kv.Txn, startID uint32, limit int) ([]*keyspacepb.KeyspaceMeta, error) { startKey := KeyspaceMetaPath(startID) endKey := clientv3.GetPrefixRangeEnd(KeyspaceMetaPrefix()) - keys, values, err := se.LoadRange(startKey, endKey, limit) + keys, values, err := txn.LoadRange(startKey, endKey, limit) if err != nil { return nil, err } diff --git a/pkg/storage/endpoint/meta.go b/pkg/storage/endpoint/meta.go index ed6a9d9ebd6..4ba9eb42c5c 100644 --- a/pkg/storage/endpoint/meta.go +++ b/pkg/storage/endpoint/meta.go @@ -32,11 +32,11 @@ import ( type MetaStorage interface { LoadMeta(meta *metapb.Cluster) (bool, error) SaveMeta(meta *metapb.Cluster) error - LoadStore(storeID uint64, store *metapb.Store) (bool, error) - SaveStore(store *metapb.Store) error + LoadStoreMeta(storeID uint64, store *metapb.Store) (bool, error) + SaveStoreMeta(store *metapb.Store) error SaveStoreWeight(storeID uint64, leader, region float64) error LoadStores(f func(store *core.StoreInfo)) error - DeleteStore(store *metapb.Store) error + DeleteStoreMeta(store *metapb.Store) error RegionStorage } @@ -71,13 +71,13 @@ func (se *StorageEndpoint) SaveMeta(meta *metapb.Cluster) error { return se.saveProto(clusterPath, meta) } -// LoadStore loads one store from storage. -func (se *StorageEndpoint) LoadStore(storeID uint64, store *metapb.Store) (bool, error) { +// LoadStoreMeta loads one store from storage. +func (se *StorageEndpoint) LoadStoreMeta(storeID uint64, store *metapb.Store) (bool, error) { return se.loadProto(StorePath(storeID), store) } -// SaveStore saves one store to storage. -func (se *StorageEndpoint) SaveStore(store *metapb.Store) error { +// SaveStoreMeta saves one store to storage. +func (se *StorageEndpoint) SaveStoreMeta(store *metapb.Store) error { return se.saveProto(StorePath(store.GetId()), store) } @@ -146,8 +146,8 @@ func (se *StorageEndpoint) loadFloatWithDefaultValue(path string, def float64) ( return val, nil } -// DeleteStore deletes one store from storage. -func (se *StorageEndpoint) DeleteStore(store *metapb.Store) error { +// DeleteStoreMeta deletes one store from storage. +func (se *StorageEndpoint) DeleteStoreMeta(store *metapb.Store) error { return se.Remove(StorePath(store.GetId())) } diff --git a/pkg/storage/endpoint/resource_group.go b/pkg/storage/endpoint/resource_group.go index 5527d6d4fd2..f1b3feb36aa 100644 --- a/pkg/storage/endpoint/resource_group.go +++ b/pkg/storage/endpoint/resource_group.go @@ -26,7 +26,7 @@ type ResourceGroupStorage interface { LoadResourceGroupStates(f func(k, v string)) error SaveResourceGroupStates(name string, obj interface{}) error DeleteResourceGroupStates(name string) error - SaveRequestUnitConfig(config interface{}) error + SaveControllerConfig(config interface{}) error } var _ ResourceGroupStorage = (*StorageEndpoint)(nil) @@ -61,7 +61,7 @@ func (se *StorageEndpoint) LoadResourceGroupStates(f func(k, v string)) error { return se.loadRangeByPrefix(resourceGroupStatesPath+"/", f) } -// SaveRequestUnitConfig stores the request unit config to storage. -func (se *StorageEndpoint) SaveRequestUnitConfig(config interface{}) error { - return se.saveJSON(requestUnitConfigPath, config) +// SaveControllerConfig stores the resource controller config to storage. +func (se *StorageEndpoint) SaveControllerConfig(config interface{}) error { + return se.saveJSON(controllerConfigPath, config) } diff --git a/pkg/storage/endpoint/safepoint_v2.go b/pkg/storage/endpoint/safepoint_v2.go new file mode 100644 index 00000000000..cac2606a470 --- /dev/null +++ b/pkg/storage/endpoint/safepoint_v2.go @@ -0,0 +1,220 @@ +// Copyright 2023 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package endpoint + +import ( + "encoding/json" + "math" + "time" + + "github.com/pingcap/errors" + "github.com/pingcap/log" + "github.com/tikv/pd/pkg/errs" + "go.etcd.io/etcd/clientv3" + "go.uber.org/zap" +) + +// GCSafePointV2 represents the overall safe point for a specific keyspace. +type GCSafePointV2 struct { + KeyspaceID uint32 `json:"keyspace_id"` + SafePoint uint64 `json:"safe_point"` +} + +// ServiceSafePointV2 represents a service's safepoint under a specific keyspace. +// Services can post service safe point to prevent gc safe point from incrementing. +type ServiceSafePointV2 struct { + KeyspaceID uint32 `json:"keyspace_id"` + ServiceID string `json:"service_id"` + ExpiredAt int64 `json:"expired_at"` + SafePoint uint64 `json:"safe_point"` +} + +// SafePointV2Storage defines the storage operations on safe point v2. +type SafePointV2Storage interface { + LoadGCSafePointV2(keyspaceID uint32) (*GCSafePointV2, error) + SaveGCSafePointV2(gcSafePoint *GCSafePointV2) error + LoadAllGCSafePoints() ([]*GCSafePointV2, error) + + LoadMinServiceSafePointV2(keyspaceID uint32, now time.Time) (*ServiceSafePointV2, error) + LoadServiceSafePointV2(keyspaceID uint32, serviceID string) (*ServiceSafePointV2, error) + + SaveServiceSafePointV2(serviceSafePoint *ServiceSafePointV2) error + RemoveServiceSafePointV2(keyspaceID uint32, serviceID string) error +} + +var _ SafePointV2Storage = (*StorageEndpoint)(nil) + +// LoadGCSafePointV2 loads gc safe point for the given keyspace. +func (se *StorageEndpoint) LoadGCSafePointV2(keyspaceID uint32) (*GCSafePointV2, error) { + key := GCSafePointV2Path(keyspaceID) + value, err := se.Load(key) + if err != nil { + return nil, err + } + // GC Safe Point does not exist for the given keyspace + if value == "" { + return &GCSafePointV2{ + KeyspaceID: keyspaceID, + SafePoint: 0, + }, nil + } + gcSafePoint := &GCSafePointV2{} + if err = json.Unmarshal([]byte(value), gcSafePoint); err != nil { + return nil, errs.ErrJSONUnmarshal.Wrap(err).GenWithStackByCause() + } + return gcSafePoint, nil +} + +// SaveGCSafePointV2 saves gc safe point for the given keyspace. +func (se *StorageEndpoint) SaveGCSafePointV2(gcSafePoint *GCSafePointV2) error { + key := GCSafePointV2Path(gcSafePoint.KeyspaceID) + value, err := json.Marshal(gcSafePoint) + if err != nil { + return errs.ErrJSONMarshal.Wrap(err).GenWithStackByCause() + } + return se.Save(key, string(value)) +} + +// LoadAllGCSafePoints returns gc safe point for all keyspaces +func (se *StorageEndpoint) LoadAllGCSafePoints() ([]*GCSafePointV2, error) { + prefix := GCSafePointV2Prefix() + prefixEnd := clientv3.GetPrefixRangeEnd(prefix) + _, values, err := se.LoadRange(prefix, prefixEnd, 0) + if err != nil { + return nil, err + } + gcSafePoints := make([]*GCSafePointV2, 0, len(values)) + for _, value := range values { + gcSafePoint := &GCSafePointV2{} + if err = json.Unmarshal([]byte(value), gcSafePoint); err != nil { + return nil, errs.ErrJSONUnmarshal.Wrap(err).GenWithStackByCause() + } + gcSafePoints = append(gcSafePoints, gcSafePoint) + } + return gcSafePoints, nil +} + +// LoadMinServiceSafePointV2 returns the minimum safepoint for the given keyspace. +// If no service safe point exist for the given key space or all the service safe points just expired, return nil. +// This also attempt to remove expired service safe point. +func (se *StorageEndpoint) LoadMinServiceSafePointV2(keyspaceID uint32, now time.Time) (*ServiceSafePointV2, error) { + prefix := ServiceSafePointV2Prefix(keyspaceID) + prefixEnd := clientv3.GetPrefixRangeEnd(prefix) + keys, values, err := se.LoadRange(prefix, prefixEnd, 0) + if err != nil { + return nil, err + } + if len(keys) == 0 { + return se.initServiceSafePointV2ForGCWorker(keyspaceID, 0) + } + + hasGCWorker := false + min := &ServiceSafePointV2{KeyspaceID: keyspaceID, SafePoint: math.MaxUint64} + for i, key := range keys { + serviceSafePoint := &ServiceSafePointV2{} + if err = json.Unmarshal([]byte(values[i]), serviceSafePoint); err != nil { + return nil, err + } + if serviceSafePoint.ServiceID == GCWorkerServiceSafePointID { + hasGCWorker = true + // If gc_worker's expire time is incorrectly set, fix it. + if serviceSafePoint.ExpiredAt != math.MaxInt64 { + serviceSafePoint.ExpiredAt = math.MaxInt64 + err = se.SaveServiceSafePointV2(serviceSafePoint) + if err != nil { + return nil, errors.Trace(err) + } + } + } + if serviceSafePoint.ExpiredAt < now.Unix() { + if err = se.Remove(key); err != nil { + log.Warn("failed to remove expired service safe point", zap.Error(err)) + } + continue + } + if serviceSafePoint.SafePoint < min.SafePoint { + min = serviceSafePoint + } + } + if min.SafePoint == math.MaxUint64 { + // No service safe point or all of them are expired, set min service safe point to 0 to allow any update + log.Info("there are no valid service safepoints. init gc_worker's service safepoint to 0") + return se.initServiceSafePointV2ForGCWorker(keyspaceID, 0) + } + if !hasGCWorker { + // If there exists some service safepoints but gc_worker is missing, init it with the min value among all + // safepoints (including expired ones) + return se.initServiceSafePointV2ForGCWorker(keyspaceID, min.SafePoint) + } + return min, nil +} + +// LoadServiceSafePointV2 returns ServiceSafePointV2 for given keyspaceID and serviceID. +func (se *StorageEndpoint) LoadServiceSafePointV2(keyspaceID uint32, serviceID string) (*ServiceSafePointV2, error) { + key := ServiceSafePointV2Path(keyspaceID, serviceID) + value, err := se.Load(key) + if err != nil { + return nil, err + } + // Service Safe Point does not exist for the given keyspaceID and serviceID + if value == "" { + return nil, nil + } + serviceSafePoint := &ServiceSafePointV2{} + if err = json.Unmarshal([]byte(value), serviceSafePoint); err != nil { + return nil, err + } + return serviceSafePoint, nil +} + +func (se *StorageEndpoint) initServiceSafePointV2ForGCWorker(keyspaceID uint32, initialValue uint64) (*ServiceSafePointV2, error) { + ssp := &ServiceSafePointV2{ + KeyspaceID: keyspaceID, + ServiceID: GCWorkerServiceSafePointID, + SafePoint: initialValue, + ExpiredAt: math.MaxInt64, + } + if err := se.SaveServiceSafePointV2(ssp); err != nil { + return nil, err + } + return ssp, nil +} + +// SaveServiceSafePointV2 stores service safe point to etcd. +func (se *StorageEndpoint) SaveServiceSafePointV2(serviceSafePoint *ServiceSafePointV2) error { + if serviceSafePoint.ServiceID == "" { + return errors.New("service id of service safepoint cannot be empty") + } + + if serviceSafePoint.ServiceID == GCWorkerServiceSafePointID && serviceSafePoint.ExpiredAt != math.MaxInt64 { + return errors.New("TTL of gc_worker's service safe point must be infinity") + } + + key := ServiceSafePointV2Path(serviceSafePoint.KeyspaceID, serviceSafePoint.ServiceID) + value, err := json.Marshal(serviceSafePoint) + if err != nil { + return errs.ErrJSONMarshal.Wrap(err).GenWithStackByCause() + } + return se.Save(key, string(value)) +} + +// RemoveServiceSafePointV2 removes a service safe point. +func (se *StorageEndpoint) RemoveServiceSafePointV2(keyspaceID uint32, serviceID string) error { + if serviceID == GCWorkerServiceSafePointID { + return errors.New("cannot remove service safe point of gc_worker") + } + key := ServiceSafePointV2Path(keyspaceID, serviceID) + return se.Remove(key) +} diff --git a/pkg/storage/endpoint/tso.go b/pkg/storage/endpoint/tso.go index 81f2a2f6fd0..f0aa9d8cde0 100644 --- a/pkg/storage/endpoint/tso.go +++ b/pkg/storage/endpoint/tso.go @@ -15,10 +15,13 @@ package endpoint import ( + "context" "strings" "time" + "github.com/pingcap/errors" "github.com/pingcap/log" + "github.com/tikv/pd/pkg/storage/kv" "github.com/tikv/pd/pkg/utils/typeutil" "go.etcd.io/etcd/clientv3" "go.uber.org/zap" @@ -28,6 +31,7 @@ import ( type TSOStorage interface { LoadTimestamp(prefix string) (time.Time, error) SaveTimestamp(key string, ts time.Time) error + DeleteTimestamp(key string) error } var _ TSOStorage = (*StorageEndpoint)(nil) @@ -48,7 +52,7 @@ func (se *StorageEndpoint) LoadTimestamp(prefix string) (time.Time, error) { maxTSWindow := typeutil.ZeroTime for i, key := range keys { key := strings.TrimSpace(key) - if !strings.HasSuffix(key, timestampKey) { + if !strings.HasSuffix(key, TimestampKey) { continue } tsWindow, err := typeutil.ParseTimestamp([]byte(values[i])) @@ -65,6 +69,31 @@ func (se *StorageEndpoint) LoadTimestamp(prefix string) (time.Time, error) { // SaveTimestamp saves the timestamp to the storage. func (se *StorageEndpoint) SaveTimestamp(key string, ts time.Time) error { - data := typeutil.Uint64ToBytes(uint64(ts.UnixNano())) - return se.Save(key, string(data)) + return se.RunInTxn(context.Background(), func(txn kv.Txn) error { + value, err := txn.Load(key) + if err != nil { + return err + } + + previousTS := typeutil.ZeroTime + if value != "" { + previousTS, err = typeutil.ParseTimestamp([]byte(value)) + if err != nil { + log.Error("parse timestamp failed", zap.String("key", key), zap.String("value", value), zap.Error(err)) + return err + } + } + if previousTS != typeutil.ZeroTime && typeutil.SubRealTimeByWallClock(ts, previousTS) <= 0 { + return errors.Errorf("saving timestamp %d is less than or equal to the previous one %d", ts.UnixNano(), previousTS.UnixNano()) + } + data := typeutil.Uint64ToBytes(uint64(ts.UnixNano())) + return txn.Save(key, string(data)) + }) +} + +// DeleteTimestamp deletes the timestamp from the storage. +func (se *StorageEndpoint) DeleteTimestamp(key string) error { + return se.RunInTxn(context.Background(), func(txn kv.Txn) error { + return txn.Remove(key) + }) } diff --git a/pkg/storage/endpoint/tso_keyspace_group.go b/pkg/storage/endpoint/tso_keyspace_group.go new file mode 100644 index 00000000000..498cd878887 --- /dev/null +++ b/pkg/storage/endpoint/tso_keyspace_group.go @@ -0,0 +1,214 @@ +// Copyright 2023 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package endpoint + +import ( + "context" + "encoding/json" + + "github.com/tikv/pd/pkg/slice" + "github.com/tikv/pd/pkg/storage/kv" + "go.etcd.io/etcd/clientv3" +) + +// UserKind represents the user kind. +type UserKind int + +// Different user kinds. +const ( + Basic UserKind = iota + Standard + Enterprise + + UserKindCount +) + +// StringUserKind creates a UserKind with string. +func StringUserKind(input string) UserKind { + switch input { + case Basic.String(): + return Basic + case Standard.String(): + return Standard + case Enterprise.String(): + return Enterprise + default: + return Basic + } +} + +func (k UserKind) String() string { + switch k { + case Basic: + return "basic" + case Standard: + return "standard" + case Enterprise: + return "enterprise" + } + return "unknown UserKind" +} + +// IsUserKindValid checks if the user kind is valid. +func IsUserKindValid(kind string) bool { + switch kind { + case Basic.String(), Standard.String(), Enterprise.String(): + return true + default: + return false + } +} + +// KeyspaceGroupMember defines an election member which campaigns for the primary of the keyspace group. +// Its `Priority` is used in keyspace group primary weighted-election to balance primaries' distribution. +// Among multiple replicas of a keyspace group, the higher the priority, the more likely +// the replica is to be elected as primary. +type KeyspaceGroupMember struct { + Address string `json:"address"` + Priority int `json:"priority"` +} + +// SplitState defines the split state of a keyspace group. +type SplitState struct { + // SplitSource is the current keyspace group ID from which the keyspace group is split. + // When the keyspace group is being split to another keyspace group, the split-source will + // be set to its own ID. + SplitSource uint32 `json:"split-source"` +} + +// MergeState defines the merging state of a keyspace group. +type MergeState struct { + // MergeList is the list of keyspace group IDs which are merging to this target keyspace group. + MergeList []uint32 `json:"merge-list"` +} + +// KeyspaceGroup is the keyspace group. +type KeyspaceGroup struct { + ID uint32 `json:"id"` + UserKind string `json:"user-kind"` + // SplitState is the current split state of the keyspace group. + SplitState *SplitState `json:"split-state,omitempty"` + // MergeState is the current merging state of the keyspace group. + MergeState *MergeState `json:"merge-state,omitempty"` + // Members are the election members which campaign for the primary of the keyspace group. + Members []KeyspaceGroupMember `json:"members"` + // Keyspaces are the keyspace IDs which belong to the keyspace group. + Keyspaces []uint32 `json:"keyspaces"` + // KeyspaceLookupTable is for fast lookup if a given keyspace belongs to this keyspace group. + // It's not persisted and will be built when loading from storage. + KeyspaceLookupTable map[uint32]struct{} `json:"-"` +} + +// IsSplitting checks if the keyspace group is in split state. +func (kg *KeyspaceGroup) IsSplitting() bool { + return kg != nil && kg.SplitState != nil +} + +// IsSplitTarget checks if the keyspace group is in split state and is the split target. +func (kg *KeyspaceGroup) IsSplitTarget() bool { + return kg.IsSplitting() && kg.SplitState.SplitSource != kg.ID +} + +// IsSplitSource checks if the keyspace group is in split state and is the split source. +func (kg *KeyspaceGroup) IsSplitSource() bool { + return kg.IsSplitting() && kg.SplitState.SplitSource == kg.ID +} + +// SplitSource returns the keyspace group split source ID. When the keyspace group is the split source +// itself, it will return its own ID. +func (kg *KeyspaceGroup) SplitSource() uint32 { + if kg.IsSplitting() { + return kg.SplitState.SplitSource + } + return 0 +} + +// IsMerging checks if the keyspace group is in merging state. +func (kg *KeyspaceGroup) IsMerging() bool { + return kg != nil && kg.MergeState != nil +} + +// IsMergeTarget checks if the keyspace group is in merging state and is the merge target. +func (kg *KeyspaceGroup) IsMergeTarget() bool { + return kg.IsMerging() && !slice.Contains(kg.MergeState.MergeList, kg.ID) +} + +// IsMergeSource checks if the keyspace group is in merging state and is the merge source. +func (kg *KeyspaceGroup) IsMergeSource() bool { + return kg.IsMerging() && slice.Contains(kg.MergeState.MergeList, kg.ID) +} + +// KeyspaceGroupStorage is the interface for keyspace group storage. +type KeyspaceGroupStorage interface { + LoadKeyspaceGroups(startID uint32, limit int) ([]*KeyspaceGroup, error) + LoadKeyspaceGroup(txn kv.Txn, id uint32) (*KeyspaceGroup, error) + SaveKeyspaceGroup(txn kv.Txn, kg *KeyspaceGroup) error + DeleteKeyspaceGroup(txn kv.Txn, id uint32) error + // TODO: add more interfaces. + RunInTxn(ctx context.Context, f func(txn kv.Txn) error) error +} + +var _ KeyspaceGroupStorage = (*StorageEndpoint)(nil) + +// LoadKeyspaceGroup loads the keyspace group by ID. +func (se *StorageEndpoint) LoadKeyspaceGroup(txn kv.Txn, id uint32) (*KeyspaceGroup, error) { + value, err := txn.Load(KeyspaceGroupIDPath(id)) + if err != nil || value == "" { + return nil, err + } + kg := &KeyspaceGroup{} + if err := json.Unmarshal([]byte(value), kg); err != nil { + return nil, err + } + return kg, nil +} + +// SaveKeyspaceGroup saves the keyspace group. +func (se *StorageEndpoint) SaveKeyspaceGroup(txn kv.Txn, kg *KeyspaceGroup) error { + key := KeyspaceGroupIDPath(kg.ID) + value, err := json.Marshal(kg) + if err != nil { + return err + } + return txn.Save(key, string(value)) +} + +// DeleteKeyspaceGroup deletes the keyspace group. +func (se *StorageEndpoint) DeleteKeyspaceGroup(txn kv.Txn, id uint32) error { + return txn.Remove(KeyspaceGroupIDPath(id)) +} + +// LoadKeyspaceGroups loads keyspace groups from the start ID with limit. +// If limit is 0, it will load all keyspace groups from the start ID. +func (se *StorageEndpoint) LoadKeyspaceGroups(startID uint32, limit int) ([]*KeyspaceGroup, error) { + prefix := KeyspaceGroupIDPath(startID) + prefixEnd := clientv3.GetPrefixRangeEnd(KeyspaceGroupIDPrefix()) + keys, values, err := se.LoadRange(prefix, prefixEnd, limit) + if err != nil { + return nil, err + } + if len(keys) == 0 { + return []*KeyspaceGroup{}, nil + } + kgs := make([]*KeyspaceGroup, 0, len(keys)) + for _, value := range values { + kg := &KeyspaceGroup{} + if err = json.Unmarshal([]byte(value), kg); err != nil { + return nil, err + } + kgs = append(kgs, kg) + } + return kgs, nil +} diff --git a/pkg/storage/hot_region_storage.go b/pkg/storage/hot_region_storage.go index 8ad09bbe853..62f8f5bb81f 100644 --- a/pkg/storage/hot_region_storage.go +++ b/pkg/storage/hot_region_storage.go @@ -34,6 +34,7 @@ import ( "github.com/tikv/pd/pkg/encryption" "github.com/tikv/pd/pkg/errs" "github.com/tikv/pd/pkg/storage/kv" + "github.com/tikv/pd/pkg/utils/logutil" "github.com/tikv/pd/pkg/utils/syncutil" "go.uber.org/zap" ) @@ -162,6 +163,8 @@ func NewHotRegionsStorage( // Delete hot region whose update_time is smaller than time.Now() minus remain day in the background. func (h *HotRegionStorage) backgroundDelete() { + defer logutil.LogPanic() + // make delete happened in defaultDeleteTime clock. now := time.Now() next := time.Date(now.Year(), now.Month(), now.Day(), defaultDeleteTime, 0, 0, 0, now.Location()) @@ -198,6 +201,8 @@ func (h *HotRegionStorage) backgroundDelete() { // Write hot_region info into db in the background. func (h *HotRegionStorage) backgroundFlush() { + defer logutil.LogPanic() + interval := h.getCurInterval() ticker := time.NewTicker(interval) defer func() { diff --git a/pkg/storage/keyspace_test.go b/pkg/storage/keyspace_test.go index 152d71afb5c..1236d1b34bd 100644 --- a/pkg/storage/keyspace_test.go +++ b/pkg/storage/keyspace_test.go @@ -72,9 +72,8 @@ func TestSaveLoadKeyspace(t *testing.T) { func TestLoadRangeKeyspaces(t *testing.T) { re := require.New(t) storage := NewStorageWithMemoryBackend() - - // Store test keyspace meta. keyspaces := makeTestKeyspaces() + // Store test keyspace meta. err := storage.RunInTxn(context.TODO(), func(txn kv.Txn) error { for _, keyspace := range keyspaces { re.NoError(storage.SaveKeyspaceMeta(txn, keyspace)) @@ -82,21 +81,26 @@ func TestLoadRangeKeyspaces(t *testing.T) { return nil }) re.NoError(err) + // Test load range keyspaces. + err = storage.RunInTxn(context.TODO(), func(txn kv.Txn) error { + // Load all keyspaces. + loadedKeyspaces, err := storage.LoadRangeKeyspace(txn, keyspaces[0].GetId(), 0) + re.NoError(err) + re.ElementsMatch(keyspaces, loadedKeyspaces) - // Load all keyspaces. - loadedKeyspaces, err := storage.LoadRangeKeyspace(keyspaces[0].GetId(), 0) - re.NoError(err) - re.ElementsMatch(keyspaces, loadedKeyspaces) + // Load keyspaces with id >= second test keyspace's id. + loadedKeyspaces2, err := storage.LoadRangeKeyspace(txn, keyspaces[1].GetId(), 0) + re.NoError(err) + re.ElementsMatch(keyspaces[1:], loadedKeyspaces2) - // Load keyspaces with id >= second test keyspace's id. - loadedKeyspaces2, err := storage.LoadRangeKeyspace(keyspaces[1].GetId(), 0) - re.NoError(err) - re.ElementsMatch(keyspaces[1:], loadedKeyspaces2) + // Load keyspace with the smallest id. + loadedKeyspace3, err := storage.LoadRangeKeyspace(txn, 1, 1) + re.NoError(err) + re.ElementsMatch(keyspaces[:1], loadedKeyspace3) - // Load keyspace with the smallest id. - loadedKeyspace3, err := storage.LoadRangeKeyspace(1, 1) + return nil + }) re.NoError(err) - re.ElementsMatch(keyspaces[:1], loadedKeyspace3) } func makeTestKeyspaces() []*keyspacepb.KeyspaceMeta { diff --git a/pkg/storage/kv/etcd_kv.go b/pkg/storage/kv/etcd_kv.go index 7018d8a44aa..767aeff77a6 100644 --- a/pkg/storage/kv/etcd_kv.go +++ b/pkg/storage/kv/etcd_kv.go @@ -25,7 +25,6 @@ import ( "github.com/pingcap/log" "github.com/tikv/pd/pkg/errs" "github.com/tikv/pd/pkg/utils/etcdutil" - "github.com/tikv/pd/pkg/utils/syncutil" "go.etcd.io/etcd/clientv3" "go.uber.org/zap" ) @@ -195,10 +194,8 @@ func (t *SlowLogTxn) Commit() (*clientv3.TxnResponse, error) { // Transaction commit will be successful only if all conditions are met, // aka, no other transaction has modified values loaded during current transaction. type etcdTxn struct { - kv *etcdKVBase - ctx context.Context - // mu protects conditions and operations. - mu syncutil.Mutex + kv *etcdKVBase + ctx context.Context conditions []clientv3.Cmp operations []clientv3.Op } @@ -221,8 +218,6 @@ func (kv *etcdKVBase) RunInTxn(ctx context.Context, f func(txn Txn) error) error func (txn *etcdTxn) Save(key, value string) error { key = path.Join(txn.kv.rootPath, key) operation := clientv3.OpPut(key, value) - txn.mu.Lock() - defer txn.mu.Unlock() txn.operations = append(txn.operations, operation) return nil } @@ -231,8 +226,6 @@ func (txn *etcdTxn) Save(key, value string) error { func (txn *etcdTxn) Remove(key string) error { key = path.Join(txn.kv.rootPath, key) operation := clientv3.OpDelete(key) - txn.mu.Lock() - defer txn.mu.Unlock() txn.operations = append(txn.operations, operation) return nil } @@ -261,8 +254,6 @@ func (txn *etcdTxn) Load(key string) (string, error) { return "", errs.ErrEtcdKVGetResponse.GenWithStackByArgs(resp.Kvs) } // Append the check condition to transaction. - txn.mu.Lock() - defer txn.mu.Unlock() txn.conditions = append(txn.conditions, condition) return value, nil } @@ -276,8 +267,6 @@ func (txn *etcdTxn) LoadRange(key, endKey string, limit int) (keys []string, val return keys, values, err } // If LoadRange successful, must make sure values stay the same before commit. - txn.mu.Lock() - defer txn.mu.Unlock() for i := range keys { fullKey := path.Join(txn.kv.rootPath, keys[i]) condition := clientv3.Compare(clientv3.Value(fullKey), "=", values[i]) @@ -286,12 +275,13 @@ func (txn *etcdTxn) LoadRange(key, endKey string, limit int) (keys []string, val return keys, values, err } -// commit perform the operations on etcd, with pre-condition that values observed by user has not been changed. +// commit perform the operations on etcd, with pre-condition that values observed by user have not been changed. func (txn *etcdTxn) commit() error { - baseTxn := txn.kv.client.Txn(txn.ctx) - baseTxn.If(txn.conditions...) - baseTxn.Then(txn.operations...) - resp, err := baseTxn.Commit() + // Using slowLogTxn to commit transaction. + slowLogTxn := NewSlowLogTxn(txn.kv.client) + slowLogTxn.If(txn.conditions...) + slowLogTxn.Then(txn.operations...) + resp, err := slowLogTxn.Commit() if err != nil { return err } diff --git a/pkg/storage/leveldb_backend.go b/pkg/storage/leveldb_backend.go index 25044008f6e..d25044e9c20 100644 --- a/pkg/storage/leveldb_backend.go +++ b/pkg/storage/leveldb_backend.go @@ -27,6 +27,7 @@ import ( "github.com/tikv/pd/pkg/errs" "github.com/tikv/pd/pkg/storage/endpoint" "github.com/tikv/pd/pkg/storage/kv" + "github.com/tikv/pd/pkg/utils/logutil" "github.com/tikv/pd/pkg/utils/syncutil" ) @@ -80,6 +81,8 @@ func newLevelDBBackend( var dirtyFlushTick = time.Second func (lb *levelDBBackend) backgroundFlush() { + defer logutil.LogPanic() + var ( isFlush bool err error diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go index f2bfe1f74fb..aba01dfa806 100644 --- a/pkg/storage/storage.go +++ b/pkg/storage/storage.go @@ -40,10 +40,11 @@ type Storage interface { endpoint.GCSafePointStorage endpoint.MinResolvedTSStorage endpoint.ExternalTSStorage - endpoint.KeyspaceGCSafePointStorage + endpoint.SafePointV2Storage endpoint.KeyspaceStorage endpoint.ResourceGroupStorage endpoint.TSOStorage + endpoint.KeyspaceGroupStorage } // NewStorageWithMemoryBackend creates a new storage with memory backend. diff --git a/pkg/storage/storage_gc_test.go b/pkg/storage/storage_gc_test.go index 0825885e8c1..141777d441e 100644 --- a/pkg/storage/storage_gc_test.go +++ b/pkg/storage/storage_gc_test.go @@ -24,62 +24,45 @@ import ( "github.com/tikv/pd/pkg/storage/endpoint" ) -func testGCSafePoints() ([]string, []uint64) { - spaceIDs := []string{ - "keySpace1", - "keySpace2", - "keySpace3", - "keySpace4", - "keySpace5", +func testGCSafePoints() []*endpoint.GCSafePointV2 { + gcSafePoint := []*endpoint.GCSafePointV2{ + {KeyspaceID: uint32(1), SafePoint: 0}, + {KeyspaceID: uint32(2), SafePoint: 1}, + {KeyspaceID: uint32(3), SafePoint: 4396}, + {KeyspaceID: uint32(4), SafePoint: 23333333333}, + {KeyspaceID: uint32(5), SafePoint: math.MaxUint64}, } - safePoints := []uint64{ - 0, - 1, - 4396, - 23333333333, - math.MaxUint64, - } - return spaceIDs, safePoints + + return gcSafePoint } -func testServiceSafePoints() ([]string, []*endpoint.ServiceSafePoint) { - spaceIDs := []string{ - "keySpace1", - "keySpace1", - "keySpace1", - "keySpace2", - "keySpace2", - "keySpace2", - "keySpace3", - "keySpace3", - "keySpace3", - } +func testServiceSafePoints() []*endpoint.ServiceSafePointV2 { expireAt := time.Now().Add(100 * time.Second).Unix() - serviceSafePoints := []*endpoint.ServiceSafePoint{ - {ServiceID: "service1", ExpiredAt: expireAt, SafePoint: 1}, - {ServiceID: "service2", ExpiredAt: expireAt, SafePoint: 2}, - {ServiceID: "service3", ExpiredAt: expireAt, SafePoint: 3}, - {ServiceID: "service1", ExpiredAt: expireAt, SafePoint: 1}, - {ServiceID: "service2", ExpiredAt: expireAt, SafePoint: 2}, - {ServiceID: "service3", ExpiredAt: expireAt, SafePoint: 3}, - {ServiceID: "service1", ExpiredAt: expireAt, SafePoint: 1}, - {ServiceID: "service2", ExpiredAt: expireAt, SafePoint: 2}, - {ServiceID: "service3", ExpiredAt: expireAt, SafePoint: 3}, - } - return spaceIDs, serviceSafePoints + serviceSafePoints := []*endpoint.ServiceSafePointV2{ + {KeyspaceID: uint32(1), ServiceID: "service1", ExpiredAt: expireAt, SafePoint: 1}, + {KeyspaceID: uint32(1), ServiceID: "service2", ExpiredAt: expireAt, SafePoint: 2}, + {KeyspaceID: uint32(1), ServiceID: "service3", ExpiredAt: expireAt, SafePoint: 3}, + {KeyspaceID: uint32(2), ServiceID: "service1", ExpiredAt: expireAt, SafePoint: 1}, + {KeyspaceID: uint32(2), ServiceID: "service2", ExpiredAt: expireAt, SafePoint: 2}, + {KeyspaceID: uint32(2), ServiceID: "service3", ExpiredAt: expireAt, SafePoint: 3}, + {KeyspaceID: uint32(3), ServiceID: "service1", ExpiredAt: expireAt, SafePoint: 1}, + {KeyspaceID: uint32(3), ServiceID: "service2", ExpiredAt: expireAt, SafePoint: 2}, + {KeyspaceID: uint32(3), ServiceID: "service3", ExpiredAt: expireAt, SafePoint: 3}, + } + return serviceSafePoints } func TestSaveLoadServiceSafePoint(t *testing.T) { re := require.New(t) storage := NewStorageWithMemoryBackend() - testSpaceID, testSafePoints := testServiceSafePoints() - for i := range testSpaceID { - re.NoError(storage.SaveServiceSafePoint(testSpaceID[i], testSafePoints[i])) + testServiceSafepoints := testServiceSafePoints() + for i := range testServiceSafepoints { + re.NoError(storage.SaveServiceSafePointV2(testServiceSafepoints[i])) } - for i := range testSpaceID { - loadedSafePoint, err := storage.LoadServiceSafePoint(testSpaceID[i], testSafePoints[i].ServiceID) + for i := range testServiceSafepoints { + loadedServiceSafePoint, err := storage.LoadServiceSafePointV2(testServiceSafepoints[i].KeyspaceID, testServiceSafepoints[i].ServiceID) re.NoError(err) - re.Equal(testSafePoints[i], loadedSafePoint) + re.Equal(testServiceSafepoints[i].SafePoint, loadedServiceSafePoint.SafePoint) } } @@ -87,58 +70,48 @@ func TestLoadMinServiceSafePoint(t *testing.T) { re := require.New(t) storage := NewStorageWithMemoryBackend() currentTime := time.Now() - expireAt1 := currentTime.Add(100 * time.Second).Unix() - expireAt2 := currentTime.Add(200 * time.Second).Unix() - expireAt3 := currentTime.Add(300 * time.Second).Unix() - - serviceSafePoints := []*endpoint.ServiceSafePoint{ - {ServiceID: "0", ExpiredAt: expireAt1, SafePoint: 100}, - {ServiceID: "1", ExpiredAt: expireAt2, SafePoint: 200}, - {ServiceID: "2", ExpiredAt: expireAt3, SafePoint: 300}, + expireAt1 := currentTime.Add(1000 * time.Second).Unix() + expireAt2 := currentTime.Add(2000 * time.Second).Unix() + expireAt3 := currentTime.Add(3000 * time.Second).Unix() + + testKeyspaceID := uint32(1) + serviceSafePoints := []*endpoint.ServiceSafePointV2{ + {KeyspaceID: testKeyspaceID, ServiceID: "0", ExpiredAt: expireAt1, SafePoint: 300}, + {KeyspaceID: testKeyspaceID, ServiceID: "1", ExpiredAt: expireAt2, SafePoint: 400}, + {KeyspaceID: testKeyspaceID, ServiceID: "2", ExpiredAt: expireAt3, SafePoint: 500}, } - testKeyspace := "test" for _, serviceSafePoint := range serviceSafePoints { - re.NoError(storage.SaveServiceSafePoint(testKeyspace, serviceSafePoint)) + re.NoError(storage.SaveServiceSafePointV2(serviceSafePoint)) } // enabling failpoint to make expired key removal immediately observable re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/storage/endpoint/removeExpiredKeys", "return(true)")) - minSafePoint, err := storage.LoadMinServiceSafePoint(testKeyspace, currentTime) - re.NoError(err) - re.Equal(serviceSafePoints[0], minSafePoint) - - // the safePoint with ServiceID 0 should be removed due to expiration - minSafePoint2, err := storage.LoadMinServiceSafePoint(testKeyspace, currentTime.Add(150*time.Second)) + minSafePoint, err := storage.LoadMinServiceSafePointV2(testKeyspaceID, currentTime) re.NoError(err) - re.Equal(serviceSafePoints[1], minSafePoint2) + re.Equal(serviceSafePoints[0].SafePoint, minSafePoint.SafePoint) - // verify that service safe point with ServiceID 0 has been removed - ssp, err := storage.LoadServiceSafePoint(testKeyspace, "0") + // gc_worker service safepoint will not be removed. + ssp, err := storage.LoadMinServiceSafePointV2(testKeyspaceID, currentTime.Add(5000*time.Second)) re.NoError(err) - re.Nil(ssp) - - // all remaining service safePoints should be removed due to expiration - ssp, err = storage.LoadMinServiceSafePoint(testKeyspace, currentTime.Add(500*time.Second)) - re.NoError(err) - re.Nil(ssp) + re.Equal(ssp.ServiceID, endpoint.GCWorkerServiceSafePointID) re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/storage/endpoint/removeExpiredKeys")) } func TestRemoveServiceSafePoint(t *testing.T) { re := require.New(t) storage := NewStorageWithMemoryBackend() - testSpaceID, testSafePoints := testServiceSafePoints() + testServiceSafepoint := testServiceSafePoints() // save service safe points - for i := range testSpaceID { - re.NoError(storage.SaveServiceSafePoint(testSpaceID[i], testSafePoints[i])) + for _, serviceSafePoint := range testServiceSafepoint { + re.NoError(storage.SaveServiceSafePointV2(serviceSafePoint)) } // remove saved service safe points - for i := range testSpaceID { - re.NoError(storage.RemoveServiceSafePoint(testSpaceID[i], testSafePoints[i].ServiceID)) + for _, serviceSafePoint := range testServiceSafepoint { + re.NoError(storage.RemoveServiceSafePointV2(serviceSafePoint.KeyspaceID, serviceSafePoint.ServiceID)) } // check that service safe points are empty - for i := range testSpaceID { - loadedSafePoint, err := storage.LoadServiceSafePoint(testSpaceID[i], testSafePoints[i].ServiceID) + for i := range testServiceSafepoint { + loadedSafePoint, err := storage.LoadServiceSafePointV2(testServiceSafepoint[i].KeyspaceID, testServiceSafepoint[i].ServiceID) re.NoError(err) re.Nil(loadedSafePoint) } @@ -147,54 +120,19 @@ func TestRemoveServiceSafePoint(t *testing.T) { func TestSaveLoadGCSafePoint(t *testing.T) { re := require.New(t) storage := NewStorageWithMemoryBackend() - testSpaceIDs, testSafePoints := testGCSafePoints() - for i := range testSpaceIDs { - testSpaceID := testSpaceIDs[i] - testSafePoint := testSafePoints[i] - err := storage.SaveKeyspaceGCSafePoint(testSpaceID, testSafePoint) - re.NoError(err) - loaded, err := storage.LoadKeyspaceGCSafePoint(testSpaceID) + testGCSafePoints := testGCSafePoints() + for _, testGCSafePoint := range testGCSafePoints { + testSpaceID := testGCSafePoint.KeyspaceID + testSafePoint := testGCSafePoint.SafePoint + err := storage.SaveGCSafePointV2(testGCSafePoint) re.NoError(err) - re.Equal(testSafePoint, loaded) - } -} - -func TestLoadAllKeyspaceGCSafePoints(t *testing.T) { - re := require.New(t) - storage := NewStorageWithMemoryBackend() - testSpaceIDs, testSafePoints := testGCSafePoints() - for i := range testSpaceIDs { - err := storage.SaveKeyspaceGCSafePoint(testSpaceIDs[i], testSafePoints[i]) + loadGCSafePoint, err := storage.LoadGCSafePointV2(testSpaceID) re.NoError(err) - } - loadedSafePoints, err := storage.LoadAllKeyspaceGCSafePoints(true) - re.NoError(err) - for i := range loadedSafePoints { - re.Equal(testSpaceIDs[i], loadedSafePoints[i].SpaceID) - re.Equal(testSafePoints[i], loadedSafePoints[i].SafePoint) + re.Equal(testSafePoint, loadGCSafePoint.SafePoint) } - // saving some service safe points. - spaceIDs, safePoints := testServiceSafePoints() - for i := range spaceIDs { - re.NoError(storage.SaveServiceSafePoint(spaceIDs[i], safePoints[i])) - } - - // verify that service safe points do not interfere with gc safe points. - loadedSafePoints, err = storage.LoadAllKeyspaceGCSafePoints(true) - re.NoError(err) - for i := range loadedSafePoints { - re.Equal(testSpaceIDs[i], loadedSafePoints[i].SpaceID) - re.Equal(testSafePoints[i], loadedSafePoints[i].SafePoint) - } - - // verify that when withGCSafePoint set to false, returned safePoints is 0 - loadedSafePoints, err = storage.LoadAllKeyspaceGCSafePoints(false) - re.NoError(err) - for i := range loadedSafePoints { - re.Equal(testSpaceIDs[i], loadedSafePoints[i].SpaceID) - re.Equal(uint64(0), loadedSafePoints[i].SafePoint) - } + _, err2 := storage.LoadGCSafePointV2(999) + re.NoError(err2) } func TestLoadEmpty(t *testing.T) { @@ -202,17 +140,12 @@ func TestLoadEmpty(t *testing.T) { storage := NewStorageWithMemoryBackend() // loading non-existing GC safepoint should return 0 - gcSafePoint, err := storage.LoadKeyspaceGCSafePoint("testKeyspace") + gcSafePoint, err := storage.LoadGCSafePointV2(1) re.NoError(err) - re.Equal(uint64(0), gcSafePoint) + re.Equal(uint64(0), gcSafePoint.SafePoint) // loading non-existing service safepoint should return nil - serviceSafePoint, err := storage.LoadServiceSafePoint("testKeyspace", "testService") + serviceSafePoint, err := storage.LoadServiceSafePointV2(1, "testService") re.NoError(err) re.Nil(serviceSafePoint) - - // loading empty key spaces should return empty slices - safePoints, err := storage.LoadAllKeyspaceGCSafePoints(true) - re.NoError(err) - re.Empty(safePoints) } diff --git a/pkg/storage/storage_test.go b/pkg/storage/storage_test.go index 9213aeb6026..77f94183403 100644 --- a/pkg/storage/storage_test.go +++ b/pkg/storage/storage_test.go @@ -52,12 +52,12 @@ func TestBasic(t *testing.T) { re.Equal(meta, newMeta) store := &metapb.Store{Id: 123} - ok, err = storage.LoadStore(123, store) + ok, err = storage.LoadStoreMeta(123, store) re.False(ok) re.NoError(err) - re.NoError(storage.SaveStore(store)) + re.NoError(storage.SaveStoreMeta(store)) newStore := &metapb.Store{} - ok, err = storage.LoadStore(123, newStore) + ok, err = storage.LoadStoreMeta(123, newStore) re.True(ok) re.NoError(err) re.Equal(store, newStore) @@ -87,7 +87,7 @@ func mustSaveStores(re *require.Assertions, s Storage, n int) []*metapb.Store { } for _, store := range stores { - re.NoError(s.SaveStore(store)) + re.NoError(s.SaveStoreMeta(store)) } return stores @@ -218,7 +218,7 @@ func TestLoadRegions(t *testing.T) { regions := mustSaveRegions(re, storage, n) re.NoError(storage.LoadRegions(context.Background(), cache.CheckAndPutRegion)) - re.Equal(n, cache.GetRegionCount()) + re.Equal(n, cache.GetTotalRegionCount()) for _, region := range cache.GetMetaRegions() { re.Equal(regions[region.GetId()], region) } @@ -255,7 +255,7 @@ func TestLoadRegionsToCache(t *testing.T) { regions := mustSaveRegions(re, storage, n) re.NoError(TryLoadRegionsOnce(context.Background(), storage, cache.CheckAndPutRegion)) - re.Equal(n, cache.GetRegionCount()) + re.Equal(n, cache.GetTotalRegionCount()) for _, region := range cache.GetMetaRegions() { re.Equal(regions[region.GetId()], region) } @@ -263,7 +263,7 @@ func TestLoadRegionsToCache(t *testing.T) { n = 20 mustSaveRegions(re, storage, n) re.NoError(TryLoadRegionsOnce(context.Background(), storage, cache.CheckAndPutRegion)) - re.Equal(n, cache.GetRegionCount()) + re.Equal(n, cache.GetTotalRegionCount()) } func TestLoadRegionsExceedRangeLimit(t *testing.T) { @@ -275,7 +275,7 @@ func TestLoadRegionsExceedRangeLimit(t *testing.T) { n := 1000 regions := mustSaveRegions(re, storage, n) re.NoError(storage.LoadRegions(context.Background(), cache.CheckAndPutRegion)) - re.Equal(n, cache.GetRegionCount()) + re.Equal(n, cache.GetTotalRegionCount()) for _, region := range cache.GetMetaRegions() { re.Equal(regions[region.GetId()], region) } diff --git a/pkg/storage/storage_tso_test.go b/pkg/storage/storage_tso_test.go index 22f6a718022..1dbba289512 100644 --- a/pkg/storage/storage_tso_test.go +++ b/pkg/storage/storage_tso_test.go @@ -21,6 +21,7 @@ import ( "time" "github.com/stretchr/testify/require" + "github.com/tikv/pd/pkg/storage/endpoint" "github.com/tikv/pd/pkg/utils/etcdutil" "go.etcd.io/etcd/clientv3" "go.etcd.io/etcd/embed" @@ -42,9 +43,8 @@ func TestSaveLoadTimestamp(t *testing.T) { rootPath := path.Join("/pd", strconv.FormatUint(100, 10)) storage := NewStorageWithEtcdBackend(client, rootPath) - key := "timestamp" expectedTS := time.Now().Round(0) - err = storage.SaveTimestamp(key, expectedTS) + err = storage.SaveTimestamp(endpoint.TimestampKey, expectedTS) re.NoError(err) ts, err := storage.LoadTimestamp("") re.NoError(err) @@ -68,16 +68,15 @@ func TestGlobalLocalTimestamp(t *testing.T) { storage := NewStorageWithEtcdBackend(client, rootPath) ltaKey := "lta" - timestampKey := "timestamp" dc1LocationKey, dc2LocationKey := "dc1", "dc2" localTS1 := time.Now().Round(0) - l1 := path.Join(ltaKey, dc1LocationKey, timestampKey) - l2 := path.Join(ltaKey, dc2LocationKey, timestampKey) + l1 := path.Join(ltaKey, dc1LocationKey, endpoint.TimestampKey) + l2 := path.Join(ltaKey, dc2LocationKey, endpoint.TimestampKey) err = storage.SaveTimestamp(l1, localTS1) re.NoError(err) globalTS := time.Now().Round(0) - err = storage.SaveTimestamp(timestampKey, globalTS) + err = storage.SaveTimestamp(endpoint.TimestampKey, globalTS) re.NoError(err) localTS2 := time.Now().Round(0) err = storage.SaveTimestamp(l2, localTS2) @@ -91,3 +90,32 @@ func TestGlobalLocalTimestamp(t *testing.T) { re.NoError(err) re.Equal(localTS1, ts) } + +func TestTimestampTxn(t *testing.T) { + re := require.New(t) + + cfg := etcdutil.NewTestSingleConfig(t) + etcd, err := embed.StartEtcd(cfg) + re.NoError(err) + defer etcd.Close() + + ep := cfg.LCUrls[0].String() + client, err := clientv3.New(clientv3.Config{ + Endpoints: []string{ep}, + }) + re.NoError(err) + rootPath := path.Join("/pd", strconv.FormatUint(100, 10)) + storage := NewStorageWithEtcdBackend(client, rootPath) + + globalTS1 := time.Now().Round(0) + err = storage.SaveTimestamp(endpoint.TimestampKey, globalTS1) + re.NoError(err) + + globalTS2 := globalTS1.Add(-time.Millisecond).Round(0) + err = storage.SaveTimestamp(endpoint.TimestampKey, globalTS2) + re.Error(err) + + ts, err := storage.LoadTimestamp("") + re.NoError(err) + re.Equal(globalTS1, ts) +} diff --git a/pkg/swaggerserver/swaggerserver.go b/pkg/swaggerserver/swaggerserver.go index 778844adbde..d68bab06eb2 100644 --- a/pkg/swaggerserver/swaggerserver.go +++ b/pkg/swaggerserver/swaggerserver.go @@ -12,12 +12,17 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build swagger_server +// +build swagger_server + package swaggerserver import ( "context" "net/http" + httpSwagger "github.com/swaggo/http-swagger" + _ "github.com/tikv/pd/docs/swagger" "github.com/tikv/pd/pkg/utils/apiutil" "github.com/tikv/pd/server" ) @@ -33,9 +38,14 @@ var ( } ) +// Enabled return true if swagger server is disabled. +func Enabled() bool { + return true +} + // NewHandler creates a HTTP handler for Swagger. func NewHandler(context.Context, *server.Server) (http.Handler, apiutil.APIServiceGroup, error) { swaggerHandler := http.NewServeMux() - swaggerHandler.Handle(swaggerPrefix, handler()) + swaggerHandler.Handle(swaggerPrefix, httpSwagger.Handler()) return swaggerHandler, swaggerServiceGroup, nil } diff --git a/pkg/swaggerserver/empty_handler.go b/pkg/swaggerserver/swaggerserver_disable.go similarity index 61% rename from pkg/swaggerserver/empty_handler.go rename to pkg/swaggerserver/swaggerserver_disable.go index 79f33a9af6b..c3b861b3b6c 100644 --- a/pkg/swaggerserver/empty_handler.go +++ b/pkg/swaggerserver/swaggerserver_disable.go @@ -1,4 +1,4 @@ -// Copyright 2020 TiKV Project Authors. +// Copyright 2023 TiKV Project Authors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,12 +18,19 @@ package swaggerserver import ( - "io" + "context" "net/http" + + "github.com/tikv/pd/pkg/utils/apiutil" + "github.com/tikv/pd/server" ) -func handler() http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - _, _ = io.WriteString(w, "Swagger UI is not built. Try `make` with `SWAGGER=1`.\n") - }) +// Enabled return false if swagger server is disabled. +func Enabled() bool { + return false +} + +// NewHandler creates a HTTP handler for Swagger. +func NewHandler(context.Context, *server.Server) (http.Handler, apiutil.APIServiceGroup, error) { + return nil, apiutil.APIServiceGroup{}, nil } diff --git a/server/region_syncer/client.go b/pkg/syncer/client.go similarity index 81% rename from server/region_syncer/client.go rename to pkg/syncer/client.go index 2c1dc71e772..ac409f90115 100644 --- a/server/region_syncer/client.go +++ b/pkg/syncer/client.go @@ -18,7 +18,7 @@ import ( "context" "time" - "github.com/pingcap/errors" + "github.com/docker/go-units" "github.com/pingcap/kvproto/pkg/metapb" "github.com/pingcap/kvproto/pkg/pdpb" "github.com/pingcap/log" @@ -26,6 +26,7 @@ import ( "github.com/tikv/pd/pkg/errs" "github.com/tikv/pd/pkg/storage" "github.com/tikv/pd/pkg/utils/grpcutil" + "github.com/tikv/pd/pkg/utils/logutil" "go.uber.org/zap" "google.golang.org/grpc" "google.golang.org/grpc/backoff" @@ -37,6 +38,7 @@ import ( const ( keepaliveTime = 10 * time.Second keepaliveTimeout = 3 * time.Second + msgSize = 8 * units.MiB ) // StopSyncWithLeader stop to sync the region with leader. @@ -55,38 +57,6 @@ func (s *RegionSyncer) reset() { s.mu.clientCancel, s.mu.clientCtx = nil, nil } -func (s *RegionSyncer) establish(ctx context.Context, addr string) (*grpc.ClientConn, error) { - tlsCfg, err := s.tlsConfig.ToTLSConfig() - if err != nil { - return nil, err - } - cc, err := grpcutil.GetClientConn( - ctx, - addr, - tlsCfg, - grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(msgSize)), - grpc.WithKeepaliveParams(keepalive.ClientParameters{ - Time: keepaliveTime, - Timeout: keepaliveTimeout, - }), - grpc.WithConnectParams(grpc.ConnectParams{ - Backoff: backoff.Config{ - BaseDelay: time.Second, // Default was 1s. - Multiplier: 1.6, // Default - Jitter: 0.2, // Default - MaxDelay: 3 * time.Second, // Default was 120s. - }, - MinConnectTimeout: 5 * time.Second, - }), - // WithBlock will block the dial step until success or cancel the context. - grpc.WithBlock(), - ) - if err != nil { - return nil, errors.WithStack(err) - } - return cc, nil -} - func (s *RegionSyncer) syncRegion(ctx context.Context, conn *grpc.ClientConn) (ClientStream, error) { cli := pdpb.NewPDClient(conn) syncStream, err := cli.SyncRegions(ctx) @@ -117,6 +87,7 @@ func (s *RegionSyncer) StartSyncWithLeader(addr string) { ctx := s.mu.clientCtx go func() { + defer logutil.LogPanic() defer s.wg.Done() // used to load region from kv storage to cache storage. bc := s.server.GetBasicCluster() @@ -129,19 +100,26 @@ func (s *RegionSyncer) StartSyncWithLeader(addr string) { log.Warn("failed to load regions", errs.ZapError(err)) } // establish client. - var conn *grpc.ClientConn - for { - select { - case <-ctx.Done(): - return - default: - } - conn, err = s.establish(ctx, addr) - if err != nil { - log.Error("cannot establish connection with leader", zap.String("server", s.server.Name()), zap.String("leader", s.server.GetLeader().GetName()), errs.ZapError(err)) - continue - } - break + conn := grpcutil.CreateClientConn(ctx, addr, s.tlsConfig, + grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(msgSize)), + grpc.WithKeepaliveParams(keepalive.ClientParameters{ + Time: keepaliveTime, + Timeout: keepaliveTimeout, + }), + grpc.WithConnectParams(grpc.ConnectParams{ + Backoff: backoff.Config{ + BaseDelay: time.Second, // Default was 1s. + Multiplier: 1.6, // Default + Jitter: 0.2, // Default + MaxDelay: 3 * time.Second, // Default was 120s. + }, + MinConnectTimeout: 5 * time.Second, + }), + // WithBlock will block the dial step until success or cancel the context. + grpc.WithBlock()) + // it means the context is canceled. + if conn == nil { + return } defer conn.Close() diff --git a/server/region_syncer/client_test.go b/pkg/syncer/client_test.go similarity index 94% rename from server/region_syncer/client_test.go rename to pkg/syncer/client_test.go index d0e1148bef2..ba389b5de6d 100644 --- a/server/region_syncer/client_test.go +++ b/pkg/syncer/client_test.go @@ -45,9 +45,9 @@ func TestLoadRegion(t *testing.T) { for i := 0; i < 30; i++ { rs.SaveRegion(&metapb.Region{Id: uint64(i) + 1}) } - re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/storage/base_backend/slowLoadRegion", "return(true)")) + re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/storage/endpoint/slowLoadRegion", "return(true)")) defer func() { - re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/storage/base_backend/slowLoadRegion")) + re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/storage/endpoint/slowLoadRegion")) }() rc := NewRegionSyncer(server) diff --git a/server/region_syncer/history_buffer.go b/pkg/syncer/history_buffer.go similarity index 100% rename from server/region_syncer/history_buffer.go rename to pkg/syncer/history_buffer.go diff --git a/server/region_syncer/history_buffer_test.go b/pkg/syncer/history_buffer_test.go similarity index 100% rename from server/region_syncer/history_buffer_test.go rename to pkg/syncer/history_buffer_test.go diff --git a/server/region_syncer/metrics.go b/pkg/syncer/metrics.go similarity index 100% rename from server/region_syncer/metrics.go rename to pkg/syncer/metrics.go diff --git a/server/region_syncer/server.go b/pkg/syncer/server.go similarity index 99% rename from server/region_syncer/server.go rename to pkg/syncer/server.go index fe28434ddb1..7d339e75dbe 100644 --- a/server/region_syncer/server.go +++ b/pkg/syncer/server.go @@ -39,7 +39,6 @@ import ( ) const ( - msgSize = 8 * units.MiB defaultBucketRate = 20 * units.MiB // 20MB/s defaultBucketCapacity = 20 * units.MiB // 20MB maxSyncRegionBatchSize = 100 diff --git a/pkg/systimemon/systimemon.go b/pkg/systimemon/systimemon.go index a3124312fa2..75fc5e68d8b 100644 --- a/pkg/systimemon/systimemon.go +++ b/pkg/systimemon/systimemon.go @@ -20,11 +20,13 @@ import ( "github.com/pingcap/log" "github.com/tikv/pd/pkg/errs" + "github.com/tikv/pd/pkg/utils/logutil" "go.uber.org/zap" ) // StartMonitor calls systimeErrHandler if system time jump backward. func StartMonitor(ctx context.Context, now func() time.Time, systimeErrHandler func()) { + defer logutil.LogPanic() log.Info("start system time monitor") tick := time.NewTicker(100 * time.Millisecond) defer tick.Stop() diff --git a/pkg/timerpool/pool.go b/pkg/timerpool/pool.go new file mode 100644 index 00000000000..28ffacfc629 --- /dev/null +++ b/pkg/timerpool/pool.go @@ -0,0 +1,43 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Note: This file is copied from https://go-review.googlesource.com/c/go/+/276133 + +package timerpool + +import ( + "sync" + "time" +) + +// GlobalTimerPool is a global pool for reusing *time.Timer. +var GlobalTimerPool TimerPool + +// TimerPool is a wrapper of sync.Pool which caches *time.Timer for reuse. +type TimerPool struct { + pool sync.Pool +} + +// Get returns a timer with a given duration. +func (tp *TimerPool) Get(d time.Duration) *time.Timer { + if v := tp.pool.Get(); v != nil { + timer := v.(*time.Timer) + timer.Reset(d) + return timer + } + return time.NewTimer(d) +} + +// Put tries to call timer.Stop() before putting it back into pool, +// if the timer.Stop() returns false (it has either already expired or been stopped), +// have a shot at draining the channel with residual time if there is one. +func (tp *TimerPool) Put(timer *time.Timer) { + if !timer.Stop() { + select { + case <-timer.C: + default: + } + } + tp.pool.Put(timer) +} diff --git a/pkg/timerpool/pool_test.go b/pkg/timerpool/pool_test.go new file mode 100644 index 00000000000..d6dffc723a9 --- /dev/null +++ b/pkg/timerpool/pool_test.go @@ -0,0 +1,70 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Note: This file is copied from https://go-review.googlesource.com/c/go/+/276133 + +package timerpool + +import ( + "testing" + "time" +) + +func TestTimerPool(t *testing.T) { + var tp TimerPool + + for i := 0; i < 100; i++ { + timer := tp.Get(20 * time.Millisecond) + + select { + case <-timer.C: + t.Errorf("timer expired too early") + continue + default: + } + + select { + case <-time.After(100 * time.Millisecond): + t.Errorf("timer didn't expire on time") + case <-timer.C: + } + + tp.Put(timer) + } +} + +const timeout = 10 * time.Millisecond + +func BenchmarkTimerUtilization(b *testing.B) { + b.Run("TimerWithPool", func(b *testing.B) { + for i := 0; i < b.N; i++ { + t := GlobalTimerPool.Get(timeout) + GlobalTimerPool.Put(t) + } + }) + b.Run("TimerWithoutPool", func(b *testing.B) { + for i := 0; i < b.N; i++ { + t := time.NewTimer(timeout) + t.Stop() + } + }) +} + +func BenchmarkTimerPoolParallel(b *testing.B) { + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + t := GlobalTimerPool.Get(timeout) + GlobalTimerPool.Put(t) + } + }) +} + +func BenchmarkTimerNativeParallel(b *testing.B) { + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + t := time.NewTimer(timeout) + t.Stop() + } + }) +} diff --git a/pkg/tso/admin.go b/pkg/tso/admin.go index 337d09e677a..7d510cdef65 100644 --- a/pkg/tso/admin.go +++ b/pkg/tso/admin.go @@ -25,7 +25,7 @@ import ( // Handler defines the common behaviors of a basic tso handler. type Handler interface { - ResetTS(ts uint64, ignoreSmaller, skipUpperBoundCheck bool) error + ResetTS(ts uint64, ignoreSmaller, skipUpperBoundCheck bool, keyspaceGroupID uint32) error } // AdminHandler wrap the basic tso handler to provide http service. @@ -93,7 +93,7 @@ func (h *AdminHandler) ResetTS(w http.ResponseWriter, r *http.Request) { ignoreSmaller, skipUpperBoundCheck = true, true } - if err = handler.ResetTS(ts, ignoreSmaller, skipUpperBoundCheck); err != nil { + if err = handler.ResetTS(ts, ignoreSmaller, skipUpperBoundCheck, 0); err != nil { if err == errs.ErrServerNotStarted { h.rd.JSON(w, http.StatusInternalServerError, err.Error()) } else { diff --git a/pkg/tso/allocator_manager.go b/pkg/tso/allocator_manager.go index 45fb771b9d6..d7a8a9eb81d 100644 --- a/pkg/tso/allocator_manager.go +++ b/pkg/tso/allocator_manager.go @@ -30,11 +30,14 @@ import ( "github.com/pingcap/log" "github.com/tikv/pd/pkg/election" "github.com/tikv/pd/pkg/errs" + mcsutils "github.com/tikv/pd/pkg/mcs/utils" + "github.com/tikv/pd/pkg/member" "github.com/tikv/pd/pkg/slice" "github.com/tikv/pd/pkg/storage/endpoint" "github.com/tikv/pd/pkg/storage/kv" "github.com/tikv/pd/pkg/utils/etcdutil" "github.com/tikv/pd/pkg/utils/grpcutil" + "github.com/tikv/pd/pkg/utils/logutil" "github.com/tikv/pd/pkg/utils/syncutil" "go.etcd.io/etcd/clientv3" "go.uber.org/zap" @@ -47,7 +50,6 @@ const ( checkStep = time.Minute patrolStep = time.Second defaultAllocatorLeaderLease = 3 - leaderTickInterval = 50 * time.Millisecond localTSOAllocatorEtcdPrefix = "lta" localTSOSuffixEtcdPrefix = "lts" ) @@ -94,61 +96,61 @@ func (info *DCLocationInfo) clone() DCLocationInfo { return copiedInfo } -// Member defines the interface for the election related logic. -type Member interface { - // ID returns the unique ID for this participant in the group. For example, it can be - // unique server id of a cluster or the unique keyspace group replica id of the election +// ElectionMember defines the interface for the election related logic. +type ElectionMember interface { + // ID returns the unique ID in the election group. For example, it can be unique + // server id of a cluster or the unique keyspace group replica id of the election // group comprised of the replicas of a keyspace group. ID() uint64 + // ID returns the unique Name in the election group. + Name() string // MemberValue returns the member value. MemberValue() string - // Member returns the member. - Member() *pdpb.Member + // GetMember() returns the current member + GetMember() interface{} // Client returns the etcd client. Client() *clientv3.Client // IsLeader returns whether the participant is the leader or not by checking its // leadership's lease and leader info. IsLeader() bool - // GetLeaderID returns current leader's member ID. - GetLeaderID() uint64 - // GetLeader returns current leader of the election group. - GetLeader() *pdpb.Member + // IsLeaderElected returns true if the leader exists; otherwise false. + IsLeaderElected() bool + // CheckLeader checks if someone else is taking the leadership. If yes, returns the leader; + // otherwise returns a bool which indicates if it is needed to check later. + CheckLeader() (leader member.ElectionLeader, checkAgain bool) // EnableLeader declares the member itself to be the leader. EnableLeader() - // GetLeaderPath returns the path of the leader. - GetLeaderPath() string - // GetLeadership returns the leadership of the PD member. - GetLeadership() *election.Leadership - // CampaignLeader is used to campaign a PD member's leadership - // and make it become a PD leader. - CampaignLeader(leaseTimeout int64) error - // KeepLeader is used to keep the PD leader's leadership. + // KeepLeader is used to keep the leader's leadership. KeepLeader(ctx context.Context) - // CheckLeader checks returns true if it is needed to check later. - CheckLeader() (*pdpb.Member, int64, bool) - // WatchLeader is used to watch the changes of the leader. - WatchLeader(serverCtx context.Context, leader *pdpb.Member, revision int64) - // ResetLeader is used to reset the PD member's current leadership. + // CampaignLeader is used to campaign the leadership and make it become a leader in an election group. + CampaignLeader(leaseTimeout int64) error + // ResetLeader is used to reset the member's current leadership. // Basically it will reset the leader lease and unset leader info. ResetLeader() - // IsSameLeader checks whether a server is the leader itself. - IsSameLeader(leader *pdpb.Member) bool - // CheckPriority checks whether there is another participant has higher priority and resign it as the leader if so. - CheckPriority(ctx context.Context) + // GetLeaderListenUrls returns current leader's listen urls + // The first element is the leader/primary url + GetLeaderListenUrls() []string + // GetLeaderID returns current leader's member ID. + GetLeaderID() uint64 + // GetLeaderPath returns the path of the leader. + GetLeaderPath() string + // GetLeadership returns the leadership of the election member. + GetLeadership() *election.Leadership + // GetLastLeaderUpdatedTime returns the last time when the leader is updated. + GetLastLeaderUpdatedTime() time.Time // GetDCLocationPathPrefix returns the dc-location path prefix of the cluster. GetDCLocationPathPrefix() string // GetDCLocationPath returns the dc-location path of a member with the given member ID. GetDCLocationPath(id uint64) string - // PrecheckLeader does some pre-check before checking whether it's the leader. - PrecheckLeader() error + // PreCheckLeader does some pre-check before checking whether it's the leader. + PreCheckLeader() error } // AllocatorManager is used to manage the TSO Allocators a PD server holds. // It is in charge of maintaining TSO allocators' leadership, checking election // priority, and forwarding TSO allocation requests to correct TSO Allocators. type AllocatorManager struct { - enableLocalTSO bool - mu struct { + mu struct { syncutil.RWMutex // There are two kinds of TSO Allocators: // 1. Global TSO Allocator, as a global single point to allocate @@ -161,16 +163,29 @@ type AllocatorManager struct { // the number of suffix bits we need in the TSO logical part. maxSuffix int32 } + // for the synchronization purpose of the allocator update checks wg sync.WaitGroup - // for election use - member Member + // for the synchronization purpose of the service loops + svcLoopWG sync.WaitGroup + + ctx context.Context + cancel context.CancelFunc + // kgID is the keyspace group ID + kgID uint32 + // member is for election use + member ElectionMember // TSO config rootPath string storage endpoint.TSOStorage + enableLocalTSO bool saveInterval time.Duration updatePhysicalInterval time.Duration - maxResetTSGap func() time.Duration - securityConfig *grpcutil.TLSConfig + // leaderLease defines the time within which a TSO primary/leader must update its TTL + // in etcd, otherwise etcd will expire the leader key and other servers can campaign + // the primary/leader again. Etcd only supports seconds TTL, so here is second too. + leaderLease int64 + maxResetTSGap func() time.Duration + securityConfig *grpcutil.TLSConfig // for gRPC use localAllocatorConn struct { syncutil.RWMutex @@ -180,38 +195,146 @@ type AllocatorManager struct { // NewAllocatorManager creates a new TSO Allocator Manager. func NewAllocatorManager( - m Member, + ctx context.Context, + keyspaceGroupID uint32, + member ElectionMember, rootPath string, storage endpoint.TSOStorage, - enableLocalTSO bool, - saveInterval time.Duration, - updatePhysicalInterval time.Duration, - tlsConfig *grpcutil.TLSConfig, - maxResetTSGap func() time.Duration, + cfg Config, + startGlobalLeaderLoop bool, ) *AllocatorManager { - allocatorManager := &AllocatorManager{ - enableLocalTSO: enableLocalTSO, - member: m, + ctx, cancel := context.WithCancel(ctx) + am := &AllocatorManager{ + ctx: ctx, + cancel: cancel, + kgID: keyspaceGroupID, + member: member, rootPath: rootPath, storage: storage, - saveInterval: saveInterval, - updatePhysicalInterval: updatePhysicalInterval, - maxResetTSGap: maxResetTSGap, - securityConfig: tlsConfig, - } - allocatorManager.mu.allocatorGroups = make(map[string]*allocatorGroup) - allocatorManager.mu.clusterDCLocations = make(map[string]*DCLocationInfo) - allocatorManager.localAllocatorConn.clientConns = make(map[string]*grpc.ClientConn) - return allocatorManager + enableLocalTSO: cfg.IsLocalTSOEnabled(), + saveInterval: cfg.GetTSOSaveInterval(), + updatePhysicalInterval: cfg.GetTSOUpdatePhysicalInterval(), + leaderLease: cfg.GetLeaderLease(), + maxResetTSGap: cfg.GetMaxResetTSGap, + securityConfig: cfg.GetTLSConfig(), + } + am.mu.allocatorGroups = make(map[string]*allocatorGroup) + am.mu.clusterDCLocations = make(map[string]*DCLocationInfo) + am.localAllocatorConn.clientConns = make(map[string]*grpc.ClientConn) + + // Set up the Global TSO Allocator here, it will be initialized once the member campaigns leader successfully. + am.SetUpGlobalAllocator(am.ctx, am.member.GetLeadership(), startGlobalLeaderLoop) + am.svcLoopWG.Add(1) + go am.tsoAllocatorLoop() + + return am +} + +// SetUpGlobalAllocator is used to set up the global allocator, which will initialize the allocator and put it into +// an allocator daemon. An TSO Allocator should only be set once, and may be initialized and reset multiple times +// depending on the election. +func (am *AllocatorManager) SetUpGlobalAllocator(ctx context.Context, leadership *election.Leadership, startGlobalLeaderLoop bool) { + am.mu.Lock() + defer am.mu.Unlock() + + allocator := NewGlobalTSOAllocator(ctx, am, startGlobalLeaderLoop) + // Create a new allocatorGroup + ctx, cancel := context.WithCancel(ctx) + am.mu.allocatorGroups[GlobalDCLocation] = &allocatorGroup{ + dcLocation: GlobalDCLocation, + ctx: ctx, + cancel: cancel, + leadership: leadership, + allocator: allocator, + } +} + +// setUpLocalAllocator is used to set up an allocator, which will initialize the allocator and put it into allocator daemon. +// One TSO Allocator should only be set once, and may be initialized and reset multiple times depending on the election. +func (am *AllocatorManager) setUpLocalAllocator(parentCtx context.Context, dcLocation string, leadership *election.Leadership) { + am.mu.Lock() + defer am.mu.Unlock() + + if _, exist := am.mu.allocatorGroups[dcLocation]; exist { + return + } + allocator := NewLocalTSOAllocator(am, leadership, dcLocation) + // Create a new allocatorGroup + ctx, cancel := context.WithCancel(parentCtx) + am.mu.allocatorGroups[dcLocation] = &allocatorGroup{ + dcLocation: dcLocation, + ctx: ctx, + cancel: cancel, + leadership: leadership, + allocator: allocator, + } + // Start election of the Local TSO Allocator here + localTSOAllocator, _ := allocator.(*LocalTSOAllocator) + go am.allocatorLeaderLoop(parentCtx, localTSOAllocator) +} + +// getGroupID returns the keyspace group ID of the allocator manager. +func (am *AllocatorManager) getGroupID() uint32 { + if am == nil { + return 0 + } + return am.kgID +} + +// GetTimestampPath returns the timestamp path in etcd for the given DCLocation. +func (am *AllocatorManager) GetTimestampPath(dcLocation string) string { + if am == nil { + return "" + } + if len(dcLocation) == 0 { + dcLocation = GlobalDCLocation + } + + am.mu.RLock() + defer am.mu.RUnlock() + if allocatorGroup, exist := am.mu.allocatorGroups[dcLocation]; exist { + return path.Join(am.rootPath, allocatorGroup.allocator.GetTimestampPath()) + } + return "" +} + +// tsoAllocatorLoop is used to run the TSO Allocator updating daemon. +func (am *AllocatorManager) tsoAllocatorLoop() { + defer logutil.LogPanic() + defer am.svcLoopWG.Done() + + am.AllocatorDaemon(am.ctx) + log.Info("exit allocator loop", logutil.CondUint32("keyspace-group-id", am.kgID, am.kgID > 0)) +} + +// close is used to shutdown TSO Allocator updating daemon. +// tso service call this function to shutdown the loop here, but pd manages its own loop. +func (am *AllocatorManager) close() { + log.Info("closing the allocator manager", logutil.CondUint32("keyspace-group-id", am.kgID, am.kgID > 0)) + + if allocatorGroup, exist := am.getAllocatorGroup(GlobalDCLocation); exist { + allocatorGroup.allocator.(*GlobalTSOAllocator).close() + } + + am.cancel() + am.svcLoopWG.Wait() + + log.Info("closed the allocator manager", logutil.CondUint32("keyspace-group-id", am.kgID, am.kgID > 0)) +} + +// GetMember returns the ElectionMember of this AllocatorManager. +func (am *AllocatorManager) GetMember() ElectionMember { + return am.member } // SetLocalTSOConfig receives the zone label of this PD server and write it into etcd as dc-location // to make the whole cluster know the DC-level topology for later Local TSO Allocator campaign. func (am *AllocatorManager) SetLocalTSOConfig(dcLocation string) error { - serverName := am.member.Member().Name + serverName := am.member.Name() serverID := am.member.ID() if err := am.checkDCLocationUpperLimit(dcLocation); err != nil { log.Error("check dc-location upper limit failed", + logutil.CondUint32("keyspace-group-id", am.kgID, am.kgID != 0), zap.Int("upper-limit", int(math.Pow(2, MaxSuffixBits))-1), zap.String("dc-location", dcLocation), zap.String("server-name", serverName), @@ -230,12 +353,14 @@ func (am *AllocatorManager) SetLocalTSOConfig(dcLocation string) error { } if !resp.Succeeded { log.Warn("write dc-location configuration into etcd failed", + logutil.CondUint32("keyspace-group-id", am.kgID, am.kgID > 0), zap.String("dc-location", dcLocation), zap.String("server-name", serverName), zap.Uint64("server-id", serverID)) return errs.ErrEtcdTxnConflict.FastGenByArgs() } log.Info("write dc-location configuration into etcd", + logutil.CondUint32("keyspace-group-id", am.kgID, am.kgID > 0), zap.String("dc-location", dcLocation), zap.String("server-name", serverName), zap.Uint64("server-id", serverID)) @@ -278,6 +403,7 @@ func (am *AllocatorManager) GetClusterDCLocationsFromEtcd() (clusterDCLocations dcLocation := string(kv.Value) if err != nil { log.Warn("get server id and dcLocation from etcd failed, invalid server id", + logutil.CondUint32("keyspace-group-id", am.kgID, am.kgID > 0), zap.Any("splitted-serverPath", serverPath), zap.String("dc-location", dcLocation), errs.ZapError(err)) @@ -313,6 +439,7 @@ func (am *AllocatorManager) CleanUpDCLocation() error { return errs.ErrEtcdTxnConflict.FastGenByArgs() } log.Info("delete the dc-location key previously written in etcd", + logutil.CondUint32("keyspace-group-id", am.kgID, am.kgID > 0), zap.Uint64("server-id", serverID)) go am.ClusterDCLocationChecker() return nil @@ -361,40 +488,6 @@ func CalSuffixBits(maxSuffix int32) int { return int(math.Ceil(math.Log2(float64(maxSuffix + 1)))) } -// SetUpAllocator is used to set up an allocator, which will initialize the allocator and put it into allocator daemon. -// One TSO Allocator should only be set once, and may be initialized and reset multiple times depending on the election. -func (am *AllocatorManager) SetUpAllocator(parentCtx context.Context, dcLocation string, leadership *election.Leadership) { - am.mu.Lock() - defer am.mu.Unlock() - if _, exist := am.mu.allocatorGroups[dcLocation]; exist { - return - } - var allocator Allocator - if dcLocation == GlobalDCLocation { - allocator = NewGlobalTSOAllocator(am, leadership) - } else { - allocator = NewLocalTSOAllocator(am, leadership, dcLocation) - } - // Create a new allocatorGroup - ctx, cancel := context.WithCancel(parentCtx) - am.mu.allocatorGroups[dcLocation] = &allocatorGroup{ - dcLocation: dcLocation, - ctx: ctx, - cancel: cancel, - leadership: leadership, - allocator: allocator, - } - // Because the Global TSO Allocator only depends on PD leader's leadership, - // so we can directly return here. The election and initialization process - // will happen in server.campaignLeader(). - if dcLocation == GlobalDCLocation { - return - } - // Start election of the Local TSO Allocator here - localTSOAllocator, _ := allocator.(*LocalTSOAllocator) - go am.allocatorLeaderLoop(parentCtx, localTSOAllocator) -} - func (am *AllocatorManager) getAllocatorPath(dcLocation string) string { // For backward compatibility, the global timestamp's store path will still use the old one if dcLocation == GlobalDCLocation { @@ -411,9 +504,11 @@ func (am *AllocatorManager) getLocalTSOAllocatorPath() string { // similar logic with leaderLoop in server/server.go func (am *AllocatorManager) allocatorLeaderLoop(ctx context.Context, allocator *LocalTSOAllocator) { + defer logutil.LogPanic() defer log.Info("server is closed, return local tso allocator leader loop", + logutil.CondUint32("keyspace-group-id", am.kgID, am.kgID > 0), zap.String("dc-location", allocator.GetDCLocation()), - zap.String("local-tso-allocator-name", am.member.Member().Name)) + zap.String("local-tso-allocator-name", am.member.Name())) for { select { case <-ctx.Done(): @@ -428,11 +523,13 @@ func (am *AllocatorManager) allocatorLeaderLoop(ctx context.Context, allocator * } if allocatorLeader != nil { log.Info("start to watch allocator leader", + logutil.CondUint32("keyspace-group-id", am.kgID, am.kgID > 0), zap.Stringer(fmt.Sprintf("%s-allocator-leader", allocator.GetDCLocation()), allocatorLeader), - zap.String("local-tso-allocator-name", am.member.Member().Name)) + zap.String("local-tso-allocator-name", am.member.Name())) // WatchAllocatorLeader will keep looping and never return unless the Local TSO Allocator leader has changed. allocator.WatchAllocatorLeader(ctx, allocatorLeader, rev) log.Info("local tso allocator leader has changed, try to re-campaign a local tso allocator leader", + logutil.CondUint32("keyspace-group-id", am.kgID, am.kgID > 0), zap.String("dc-location", allocator.GetDCLocation())) } @@ -440,6 +537,7 @@ func (am *AllocatorManager) allocatorLeaderLoop(ctx context.Context, allocator * nextLeader, err := am.getNextLeaderID(allocator.GetDCLocation()) if err != nil { log.Error("get next leader from etcd failed", + logutil.CondUint32("keyspace-group-id", am.kgID, am.kgID > 0), zap.String("dc-location", allocator.GetDCLocation()), errs.ZapError(err)) time.Sleep(200 * time.Millisecond) @@ -449,7 +547,8 @@ func (am *AllocatorManager) allocatorLeaderLoop(ctx context.Context, allocator * if nextLeader != 0 { if nextLeader != am.member.ID() { log.Info("skip campaigning of the local tso allocator leader and check later", - zap.String("server-name", am.member.Member().Name), + logutil.CondUint32("keyspace-group-id", am.kgID, am.kgID > 0), + zap.String("server-name", am.member.Name()), zap.Uint64("server-id", am.member.ID()), zap.Uint64("next-leader-id", nextLeader)) time.Sleep(200 * time.Millisecond) @@ -463,6 +562,7 @@ func (am *AllocatorManager) allocatorLeaderLoop(ctx context.Context, allocator * ok, dcLocationInfo, err := am.getDCLocationInfoFromLeader(ctx, allocator.GetDCLocation()) if err != nil { log.Error("get dc-location info from pd leader failed", + logutil.CondUint32("keyspace-group-id", am.kgID, am.kgID > 0), zap.String("dc-location", allocator.GetDCLocation()), errs.ZapError(err)) // PD leader hasn't been elected out, wait for the campaign @@ -473,6 +573,7 @@ func (am *AllocatorManager) allocatorLeaderLoop(ctx context.Context, allocator * } if !ok || dcLocationInfo.Suffix <= 0 || dcLocationInfo.MaxTs == nil { log.Warn("pd leader is not aware of dc-location during allocatorLeaderLoop, wait next round", + logutil.CondUint32("keyspace-group-id", am.kgID, am.kgID > 0), zap.String("dc-location", allocator.GetDCLocation()), zap.Any("dc-location-info", dcLocationInfo), zap.String("wait-duration", checkStep.String())) @@ -509,9 +610,10 @@ func (am *AllocatorManager) campaignAllocatorLeader( isNextLeader bool, ) { log.Info("start to campaign local tso allocator leader", + logutil.CondUint32("keyspace-group-id", am.kgID, am.kgID > 0), zap.String("dc-location", allocator.GetDCLocation()), zap.Any("dc-location-info", dcLocationInfo), - zap.String("name", am.member.Member().Name)) + zap.String("name", am.member.Name())) cmps := make([]clientv3.Cmp, 0) nextLeaderKey := am.nextLeaderKey(allocator.GetDCLocation()) if !isNextLeader { @@ -529,17 +631,19 @@ func (am *AllocatorManager) campaignAllocatorLeader( } } }) - if err := allocator.CampaignAllocatorLeader(defaultAllocatorLeaderLease, cmps...); err != nil { + if err := allocator.CampaignAllocatorLeader(am.leaderLease, cmps...); err != nil { if err.Error() == errs.ErrEtcdTxnConflict.Error() { log.Info("failed to campaign local tso allocator leader due to txn conflict, another allocator may campaign successfully", + logutil.CondUint32("keyspace-group-id", am.kgID, am.kgID > 0), zap.String("dc-location", allocator.GetDCLocation()), zap.Any("dc-location-info", dcLocationInfo), - zap.String("name", am.member.Member().Name)) + zap.String("name", am.member.Name())) } else { log.Error("failed to campaign local tso allocator leader due to etcd error", + logutil.CondUint32("keyspace-group-id", am.kgID, am.kgID > 0), zap.String("dc-location", allocator.GetDCLocation()), zap.Any("dc-location-info", dcLocationInfo), - zap.String("name", am.member.Member().Name), + zap.String("name", am.member.Name()), errs.ZapError(err)) } return @@ -552,16 +656,19 @@ func (am *AllocatorManager) campaignAllocatorLeader( // Maintain the Local TSO Allocator leader go allocator.KeepAllocatorLeader(ctx) log.Info("campaign local tso allocator leader ok", + logutil.CondUint32("keyspace-group-id", am.kgID, am.kgID > 0), zap.String("dc-location", allocator.GetDCLocation()), zap.Any("dc-location-info", dcLocationInfo), - zap.String("name", am.member.Member().Name)) + zap.String("name", am.member.Name())) log.Info("initialize the local TSO allocator", + logutil.CondUint32("keyspace-group-id", am.kgID, am.kgID > 0), zap.String("dc-location", allocator.GetDCLocation()), zap.Any("dc-location-info", dcLocationInfo), - zap.String("name", am.member.Member().Name)) + zap.String("name", am.member.Name())) if err := allocator.Initialize(int(dcLocationInfo.Suffix)); err != nil { log.Error("failed to initialize the local TSO allocator", + logutil.CondUint32("keyspace-group-id", am.kgID, am.kgID > 0), zap.String("dc-location", allocator.GetDCLocation()), zap.Any("dc-location-info", dcLocationInfo), errs.ZapError(err)) @@ -570,6 +677,7 @@ func (am *AllocatorManager) campaignAllocatorLeader( if dcLocationInfo.GetMaxTs().GetPhysical() != 0 { if err := allocator.WriteTSO(dcLocationInfo.GetMaxTs()); err != nil { log.Error("failed to write the max local TSO after member changed", + logutil.CondUint32("keyspace-group-id", am.kgID, am.kgID > 0), zap.String("dc-location", allocator.GetDCLocation()), zap.Any("dc-location-info", dcLocationInfo), errs.ZapError(err)) @@ -581,11 +689,12 @@ func (am *AllocatorManager) campaignAllocatorLeader( // The next leader is me, delete it to finish campaigning am.deleteNextLeaderID(allocator.GetDCLocation()) log.Info("local tso allocator leader is ready to serve", + logutil.CondUint32("keyspace-group-id", am.kgID, am.kgID > 0), zap.String("dc-location", allocator.GetDCLocation()), zap.Any("dc-location-info", dcLocationInfo), - zap.String("name", am.member.Member().Name)) + zap.String("name", am.member.Name())) - leaderTicker := time.NewTicker(leaderTickInterval) + leaderTicker := time.NewTicker(mcsutils.LeaderTickInterval) defer leaderTicker.Stop() for { @@ -593,17 +702,19 @@ func (am *AllocatorManager) campaignAllocatorLeader( case <-leaderTicker.C: if !allocator.IsAllocatorLeader() { log.Info("no longer a local tso allocator leader because lease has expired, local tso allocator leader will step down", + logutil.CondUint32("keyspace-group-id", am.kgID, am.kgID > 0), zap.String("dc-location", allocator.GetDCLocation()), zap.Any("dc-location-info", dcLocationInfo), - zap.String("name", am.member.Member().Name)) + zap.String("name", am.member.Name())) return } case <-ctx.Done(): // Server is closed and it should return nil. log.Info("server is closed, reset the local tso allocator", + logutil.CondUint32("keyspace-group-id", am.kgID, am.kgID > 0), zap.String("dc-location", allocator.GetDCLocation()), zap.Any("dc-location-info", dcLocationInfo), - zap.String("name", am.member.Member().Name)) + zap.String("name", am.member.Name())) return } } @@ -611,7 +722,9 @@ func (am *AllocatorManager) campaignAllocatorLeader( // AllocatorDaemon is used to update every allocator's TSO and check whether we have // any new local allocator that needs to be set up. -func (am *AllocatorManager) AllocatorDaemon(serverCtx context.Context) { +func (am *AllocatorManager) AllocatorDaemon(ctx context.Context) { + log.Info("entering into allocator daemon", logutil.CondUint32("keyspace-group-id", am.kgID, am.kgID > 0)) + // allocatorPatroller should only work when enableLocalTSO is true to // set up the new Local TSO Allocator in time. var patrolTicker = &time.Ticker{} @@ -620,6 +733,10 @@ func (am *AllocatorManager) AllocatorDaemon(serverCtx context.Context) { defer patrolTicker.Stop() } tsTicker := time.NewTicker(am.updatePhysicalInterval) + failpoint.Inject("fastUpdatePhysicalInterval", func() { + tsTicker.Stop() + tsTicker = time.NewTicker(time.Millisecond) + }) defer tsTicker.Stop() checkerTicker := time.NewTicker(PriorityCheck) defer checkerTicker.Stop() @@ -628,7 +745,7 @@ func (am *AllocatorManager) AllocatorDaemon(serverCtx context.Context) { select { case <-patrolTicker.C: // Inspect the cluster dc-location info and set up the new Local TSO Allocator in time. - am.allocatorPatroller(serverCtx) + am.allocatorPatroller(ctx) case <-tsTicker.C: // Update the initialized TSO Allocator to advance TSO. am.allocatorUpdater() @@ -642,7 +759,8 @@ func (am *AllocatorManager) AllocatorDaemon(serverCtx context.Context) { } // PS: ClusterDCLocationChecker and PriorityChecker are time consuming and low frequent to run, // we should run them concurrently to speed up the progress. - case <-serverCtx.Done(): + case <-ctx.Done(): + log.Info("exit allocator daemon", logutil.CondUint32("keyspace-group-id", am.kgID, am.kgID > 0)) return } } @@ -662,21 +780,30 @@ func (am *AllocatorManager) allocatorUpdater() { // updateAllocator is used to update the allocator in the group. func (am *AllocatorManager) updateAllocator(ag *allocatorGroup) { + defer logutil.LogPanic() defer am.wg.Done() + select { case <-ag.ctx.Done(): // Resetting the allocator will clear TSO in memory ag.allocator.Reset() + log.Info("exit the allocator update loop", logutil.CondUint32("keyspace-group-id", am.kgID, am.kgID > 0)) return default: } if !ag.leadership.Check() { - log.Info("allocator doesn't campaign leadership yet", zap.String("dc-location", ag.dcLocation)) + log.Info("allocator doesn't campaign leadership yet", + logutil.CondUint32("keyspace-group-id", am.kgID, am.kgID > 0), + zap.String("dc-location", ag.dcLocation)) time.Sleep(200 * time.Millisecond) return } if err := ag.allocator.UpdateTSO(); err != nil { - log.Warn("failed to update allocator's timestamp", zap.String("dc-location", ag.dcLocation), errs.ZapError(err)) + log.Warn("failed to update allocator's timestamp", + logutil.CondUint32("keyspace-group-id", am.kgID, am.kgID > 0), + zap.String("dc-location", ag.dcLocation), + zap.String("name", am.member.Name()), + errs.ZapError(err)) am.ResetAllocatorGroup(ag.dcLocation) return } @@ -694,7 +821,7 @@ func (am *AllocatorManager) allocatorPatroller(serverCtx context.Context) { if slice.NoneOf(allocatorGroups, func(i int) bool { return allocatorGroups[i].dcLocation == dcLocation }) { - am.SetUpAllocator(serverCtx, dcLocation, election.NewLeadership( + am.setUpLocalAllocator(serverCtx, dcLocation, election.NewLeadership( am.member.Client(), am.getAllocatorPath(dcLocation), fmt.Sprintf("%s local allocator leader election", dcLocation), @@ -712,13 +839,16 @@ func (am *AllocatorManager) allocatorPatroller(serverCtx context.Context) { // ClusterDCLocationChecker collects all dc-locations of a cluster, computes some related info // and stores them into the DCLocationInfo, then finally writes them into am.mu.clusterDCLocations. func (am *AllocatorManager) ClusterDCLocationChecker() { - // Wait for the PD leader to be elected out. - if am.member.GetLeader() == nil { + defer logutil.LogPanic() + // Wait for the group leader to be elected out. + if !am.member.IsLeaderElected() { return } newClusterDCLocations, err := am.GetClusterDCLocationsFromEtcd() if err != nil { - log.Error("get cluster dc-locations from etcd failed", errs.ZapError(err)) + log.Error("get cluster dc-locations from etcd failed", + logutil.CondUint32("keyspace-group-id", am.kgID, am.kgID > 0), + errs.ZapError(err)) return } am.mu.Lock() @@ -741,14 +871,17 @@ func (am *AllocatorManager) ClusterDCLocationChecker() { } } // Only leader can write the TSO suffix to etcd in order to make it consistent in the cluster - if am.member.IsLeader() { + if am.IsLeader() { for dcLocation, info := range am.mu.clusterDCLocations { if info.Suffix > 0 { continue } suffix, err := am.getOrCreateLocalTSOSuffix(dcLocation) if err != nil { - log.Warn("get or create the local tso suffix failed", zap.String("dc-location", dcLocation), errs.ZapError(err)) + log.Warn("get or create the local tso suffix failed", + logutil.CondUint32("keyspace-group-id", am.kgID, am.kgID > 0), + zap.String("dc-location", dcLocation), + errs.ZapError(err)) continue } if suffix > am.mu.maxSuffix { @@ -760,7 +893,9 @@ func (am *AllocatorManager) ClusterDCLocationChecker() { // Follower should check and update the am.mu.maxSuffix maxSuffix, err := am.getMaxLocalTSOSuffix() if err != nil { - log.Error("get the max local tso suffix from etcd failed", errs.ZapError(err)) + log.Error("get the max local tso suffix from etcd failed", + logutil.CondUint32("keyspace-group-id", am.kgID, am.kgID > 0), + errs.ZapError(err)) // Rollback the new dc-locations we update before for _, dcLocation := range newDCLocations { delete(am.mu.clusterDCLocations, dcLocation) @@ -805,9 +940,10 @@ func (am *AllocatorManager) getOrCreateLocalTSOSuffix(dcLocation string) (int32, } if !txnResp.Succeeded { log.Warn("write local tso suffix into etcd failed", + logutil.CondUint32("keyspace-group-id", am.kgID, am.kgID > 0), zap.String("dc-location", dcLocation), zap.String("local-tso-suffix", localTSOSuffixValue), - zap.String("server-name", am.member.Member().Name), + zap.String("server-name", am.member.Name()), zap.Uint64("server-id", am.member.ID())) return -1, errs.ErrEtcdTxnConflict.FastGenByArgs() } @@ -868,6 +1004,8 @@ func (am *AllocatorManager) GetLocalTSOSuffixPath(dcLocation string) string { // 2. If all PD servers with dc-location="dc-1" are down, then the other PD servers // of DC could be elected. func (am *AllocatorManager) PriorityChecker() { + defer logutil.LogPanic() + serverID := am.member.ID() myServerDCLocation := am.getServerDCLocation(serverID) // Check all Local TSO Allocator followers to see if their priorities is higher than the leaders @@ -885,12 +1023,14 @@ func (am *AllocatorManager) PriorityChecker() { // find this allocator's dc-location isn't the same with server of dc-2 but is same with itself. if allocatorGroup.dcLocation != leaderServerDCLocation && allocatorGroup.dcLocation == myServerDCLocation { log.Info("try to move the local tso allocator", + logutil.CondUint32("keyspace-group-id", am.kgID, am.kgID > 0), zap.Uint64("old-leader-id", leaderServerID), zap.String("old-dc-location", leaderServerDCLocation), zap.Uint64("next-leader-id", serverID), zap.String("next-dc-location", myServerDCLocation)) if err := am.transferLocalAllocator(allocatorGroup.dcLocation, am.member.ID()); err != nil { log.Error("move the local tso allocator failed", + logutil.CondUint32("keyspace-group-id", am.kgID, am.kgID > 0), zap.Uint64("old-leader-id", leaderServerID), zap.String("old-dc-location", leaderServerDCLocation), zap.Uint64("next-leader-id", serverID), @@ -907,13 +1047,16 @@ func (am *AllocatorManager) PriorityChecker() { nextLeader, err := am.getNextLeaderID(allocatorGroup.dcLocation) if err != nil { log.Error("get next leader from etcd failed", + logutil.CondUint32("keyspace-group-id", am.kgID, am.kgID > 0), zap.String("dc-location", allocatorGroup.dcLocation), errs.ZapError(err)) continue } // nextLeader is not empty and isn't same with the server ID, resign the leader if nextLeader != 0 && nextLeader != serverID { - log.Info("next leader key found, resign current leader", zap.Uint64("nextLeaderID", nextLeader)) + log.Info("next leader key found, resign current leader", + logutil.CondUint32("keyspace-group-id", am.kgID, am.kgID > 0), + zap.Uint64("nextLeaderID", nextLeader)) am.ResetAllocatorGroup(allocatorGroup.dcLocation) } } @@ -991,9 +1134,9 @@ func (am *AllocatorManager) deleteAllocatorGroup(dcLocation string) { } } -// HandleTSORequest forwards TSO allocation requests to correct TSO Allocators. -func (am *AllocatorManager) HandleTSORequest(dcLocation string, count uint32) (pdpb.Timestamp, error) { - if dcLocation == "" { +// HandleRequest forwards TSO allocation requests to correct TSO Allocators. +func (am *AllocatorManager) HandleRequest(dcLocation string, count uint32) (pdpb.Timestamp, error) { + if len(dcLocation) == 0 { dcLocation = GlobalDCLocation } allocatorGroup, exist := am.getAllocatorGroup(dcLocation) @@ -1001,6 +1144,7 @@ func (am *AllocatorManager) HandleTSORequest(dcLocation string, count uint32) (p err := errs.ErrGetAllocator.FastGenByArgs(fmt.Sprintf("%s allocator not found, generate timestamp failed", dcLocation)) return pdpb.Timestamp{}, err } + return allocatorGroup.allocator.GenerateTSO(count) } @@ -1112,7 +1256,7 @@ func (am *AllocatorManager) getOrCreateGRPCConn(ctx context.Context, addr string } func (am *AllocatorManager) getDCLocationInfoFromLeader(ctx context.Context, dcLocation string) (bool, *pdpb.GetDCLocationInfoResponse, error) { - if am.member.IsLeader() { + if am.IsLeader() { info, ok := am.GetDCLocationInfo(dcLocation) if !ok { return false, &pdpb.GetDCLocationInfoResponse{}, nil @@ -1125,11 +1269,11 @@ func (am *AllocatorManager) getDCLocationInfoFromLeader(ctx context.Context, dcL return ok, dcLocationInfo, nil } - leaderAddrs := am.member.GetLeader().GetClientUrls() - if leaderAddrs == nil || len(leaderAddrs) < 1 { + leaderAddr := am.GetLeaderAddr() + if len(leaderAddr) < 1 { return false, &pdpb.GetDCLocationInfoResponse{}, fmt.Errorf("failed to get leader client url") } - conn, err := am.getOrCreateGRPCConn(ctx, leaderAddrs[0]) + conn, err := am.getOrCreateGRPCConn(ctx, leaderAddr) if err != nil { return false, &pdpb.GetDCLocationInfoResponse{}, err } @@ -1137,7 +1281,7 @@ func (am *AllocatorManager) getDCLocationInfoFromLeader(ctx context.Context, dcL defer cancel() resp, err := pdpb.NewPDClient(conn).GetDCLocationInfo(getCtx, &pdpb.GetDCLocationInfoRequest{ Header: &pdpb.RequestHeader{ - SenderId: am.member.Member().GetMemberId(), + SenderId: am.member.ID(), }, DcLocation: dcLocation, }) @@ -1175,15 +1319,6 @@ func (am *AllocatorManager) GetMaxLocalTSO(ctx context.Context) (*pdpb.Timestamp return maxTSO, nil } -// GetGlobalTSO returns global tso. -func (am *AllocatorManager) GetGlobalTSO() (*pdpb.Timestamp, error) { - globalAllocator, err := am.GetAllocator(GlobalDCLocation) - if err != nil { - return nil, err - } - return globalAllocator.(*GlobalTSOAllocator).getCurrentTSO() -} - func (am *AllocatorManager) getGRPCConn(addr string) (*grpc.ClientConn, bool) { am.localAllocatorConn.RLock() defer am.localAllocatorConn.RUnlock() @@ -1196,7 +1331,10 @@ func (am *AllocatorManager) setGRPCConn(newConn *grpc.ClientConn, addr string) { defer am.localAllocatorConn.Unlock() if _, ok := am.localAllocatorConn.clientConns[addr]; ok { newConn.Close() - log.Debug("use old connection", zap.String("target", newConn.Target()), zap.String("state", newConn.GetState().String())) + log.Debug("use old connection", + logutil.CondUint32("keyspace-group-id", am.kgID, am.kgID > 0), + zap.String("target", newConn.Target()), + zap.String("state", newConn.GetState().String())) return } am.localAllocatorConn.clientConns[addr] = newConn @@ -1212,7 +1350,9 @@ func (am *AllocatorManager) transferLocalAllocator(dcLocation string, serverID u if err != nil { err = errs.ErrEtcdGrantLease.Wrap(err).GenWithStackByCause() log.Error("failed to grant the lease of the next leader key", - zap.String("dc-location", dcLocation), zap.Uint64("serverID", serverID), + logutil.CondUint32("keyspace-group-id", am.kgID, am.kgID > 0), + zap.String("dc-location", dcLocation), + zap.Uint64("serverID", serverID), errs.ZapError(err)) return err } @@ -1223,12 +1363,15 @@ func (am *AllocatorManager) transferLocalAllocator(dcLocation string, serverID u if err != nil { err = errs.ErrEtcdTxnInternal.Wrap(err).GenWithStackByCause() log.Error("failed to write next leader key into etcd", + logutil.CondUint32("keyspace-group-id", am.kgID, am.kgID > 0), zap.String("dc-location", dcLocation), zap.Uint64("serverID", serverID), errs.ZapError(err)) return err } if !resp.Succeeded { - log.Warn("write next leader id into etcd unsuccessfully", zap.String("dc-location", dcLocation)) + log.Warn("write next leader id into etcd unsuccessfully", + logutil.CondUint32("keyspace-group-id", am.kgID, am.kgID > 0), + zap.String("dc-location", dcLocation)) return errs.ErrEtcdTxnConflict.GenWithStack("write next leader id into etcd unsuccessfully") } return nil @@ -1242,3 +1385,23 @@ func (am *AllocatorManager) nextLeaderKey(dcLocation string) string { func (am *AllocatorManager) EnableLocalTSO() bool { return am.enableLocalTSO } + +// IsLeader returns whether the current member is the leader in the election group. +func (am *AllocatorManager) IsLeader() bool { + if am == nil || am.member == nil || !am.member.IsLeader() { + return false + } + return true +} + +// GetLeaderAddr returns the address of leader in the election group. +func (am *AllocatorManager) GetLeaderAddr() string { + if am == nil || am.member == nil { + return "" + } + leaderAddrs := am.member.GetLeaderListenUrls() + if len(leaderAddrs) < 1 { + return "" + } + return leaderAddrs[0] +} diff --git a/pkg/tso/config.go b/pkg/tso/config.go new file mode 100644 index 00000000000..598f76004b1 --- /dev/null +++ b/pkg/tso/config.go @@ -0,0 +1,51 @@ +// Copyright 2023 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tso + +import ( + "time" + + "github.com/tikv/pd/pkg/utils/grpcutil" +) + +// ServiceConfig defines the configuration interface for the TSO service. +type ServiceConfig interface { + // GetName returns the Name + GetName() string + // GeBackendEndpoints returns the BackendEndpoints + GeBackendEndpoints() string + // GetListenAddr returns the ListenAddr + GetListenAddr() string + // GetAdvertiseListenAddr returns the AdvertiseListenAddr + GetAdvertiseListenAddr() string + // TSO-related configuration + Config +} + +// Config is used to provide TSO configuration. +type Config interface { + // GetLeaderLease returns the leader lease. + GetLeaderLease() int64 + // IsLocalTSOEnabled returns if the local TSO is enabled. + IsLocalTSOEnabled() bool + // GetTSOUpdatePhysicalInterval returns TSO update physical interval. + GetTSOUpdatePhysicalInterval() time.Duration + // GetTSOSaveInterval returns TSO save interval. + GetTSOSaveInterval() time.Duration + // GetMaxResetTSGap returns the MaxResetTSGap. + GetMaxResetTSGap() time.Duration + // GetTLSConfig returns the TLS config. + GetTLSConfig() *grpcutil.TLSConfig +} diff --git a/pkg/tso/global_allocator.go b/pkg/tso/global_allocator.go index b86ea01e394..e81c05f9707 100644 --- a/pkg/tso/global_allocator.go +++ b/pkg/tso/global_allocator.go @@ -16,18 +16,20 @@ package tso import ( "context" + "errors" "fmt" "sync" "sync/atomic" "time" - "github.com/pingcap/errors" "github.com/pingcap/failpoint" "github.com/pingcap/kvproto/pkg/pdpb" "github.com/pingcap/log" - "github.com/tikv/pd/pkg/election" "github.com/tikv/pd/pkg/errs" + mcsutils "github.com/tikv/pd/pkg/mcs/utils" "github.com/tikv/pd/pkg/slice" + "github.com/tikv/pd/pkg/storage/endpoint" + "github.com/tikv/pd/pkg/utils/logutil" "github.com/tikv/pd/pkg/utils/tsoutil" "github.com/tikv/pd/pkg/utils/typeutil" "go.uber.org/zap" @@ -44,6 +46,14 @@ type Allocator interface { IsInitialize() bool // UpdateTSO is used to update the TSO in memory and the time window in etcd. UpdateTSO() error + // GetTimestampPath returns the timestamp path in etcd, which is: + // 1. for the default keyspace group: + // a. timestamp in /pd/{cluster_id}/timestamp + // b. lta/{dc-location}/timestamp in /pd/{cluster_id}/lta/{dc-location}/timestamp + // 1. for the non-default keyspace groups: + // a. {group}/gts/timestamp in /ms/{cluster_id}/tso/{group}/gta/timestamp + // b. {group}/lts/{dc-location}/timestamp in /ms/{cluster_id}/tso/{group}/lta/{dc-location}/timestamp + GetTimestampPath() string // SetTSO sets the physical part with given TSO. It's mainly used for BR restore. // Cannot set the TSO smaller than now in any case. // if ignoreSmaller=true, if input ts is smaller than current, ignore silently, else return error @@ -58,11 +68,14 @@ type Allocator interface { // GlobalTSOAllocator is the global single point TSO allocator. type GlobalTSOAllocator struct { + ctx context.Context + cancel context.CancelFunc + wg sync.WaitGroup + // for global TSO synchronization - allocatorManager *AllocatorManager - // leadership is used to check the current PD server's leadership - // to determine whether a TSO request could be processed. - leadership *election.Leadership + am *AllocatorManager + // for election use + member ElectionMember timestampOracle *timestampOracle // syncRTT is the RTT duration a SyncMaxTS RPC call will cost, // which is used to estimate the MaxTS in a Global TSO generation @@ -72,16 +85,19 @@ type GlobalTSOAllocator struct { // NewGlobalTSOAllocator creates a new global TSO allocator. func NewGlobalTSOAllocator( + ctx context.Context, am *AllocatorManager, - leadership *election.Leadership, + startGlobalLeaderLoop bool, ) Allocator { + ctx, cancel := context.WithCancel(ctx) gta := &GlobalTSOAllocator{ - allocatorManager: am, - leadership: leadership, + ctx: ctx, + cancel: cancel, + am: am, + member: am.member, timestampOracle: ×tampOracle{ - client: leadership.GetClient(), - rootPath: am.rootPath, - ltsPath: "", + client: am.member.GetLeadership().GetClient(), + tsPath: endpoint.KeyspaceGroupTSPath(am.kgID), storage: am.storage, saveInterval: am.saveInterval, updatePhysicalInterval: am.updatePhysicalInterval, @@ -90,9 +106,30 @@ func NewGlobalTSOAllocator( tsoMux: &tsoObject{}, }, } + + if startGlobalLeaderLoop { + gta.wg.Add(1) + go gta.primaryElectionLoop() + } + return gta } +// close is used to shutdown the primary election loop. +// tso service call this function to shutdown the loop here, but pd manages its own loop. +func (gta *GlobalTSOAllocator) close() { + gta.cancel() + gta.wg.Wait() +} + +// getGroupID returns the keyspace group ID of the allocator. +func (gta *GlobalTSOAllocator) getGroupID() uint32 { + if gta.am == nil { + return 0 + } + return gta.am.getGroupID() +} + func (gta *GlobalTSOAllocator) setSyncRTT(rtt int64) { gta.syncRTT.Store(rtt) tsoGauge.WithLabelValues("global_tso_sync_rtt", gta.timestampOracle.dcLocation).Set(float64(rtt)) @@ -106,6 +143,14 @@ func (gta *GlobalTSOAllocator) getSyncRTT() int64 { return syncRTT.(int64) } +// GetTimestampPath returns the timestamp path in etcd. +func (gta *GlobalTSOAllocator) GetTimestampPath() string { + if gta == nil || gta.timestampOracle == nil { + return "" + } + return gta.timestampOracle.GetTimestampPath() +} + func (gta *GlobalTSOAllocator) estimateMaxTS(count uint32, suffixBits int) (*pdpb.Timestamp, bool, error) { physical, logical, lastUpdateTime := gta.timestampOracle.generateTSO(int64(count), 0) if physical == 0 { @@ -129,7 +174,7 @@ func (gta *GlobalTSOAllocator) Initialize(int) error { tsoAllocatorRole.WithLabelValues(gta.timestampOracle.dcLocation).Set(1) // The suffix of a Global TSO should always be 0. gta.timestampOracle.suffix = 0 - return gta.timestampOracle.SyncTimestamp(gta.leadership) + return gta.timestampOracle.SyncTimestamp(gta.member.GetLeadership()) } // IsInitialize is used to indicates whether this allocator is initialized. @@ -139,12 +184,12 @@ func (gta *GlobalTSOAllocator) IsInitialize() bool { // UpdateTSO is used to update the TSO in memory and the time window in etcd. func (gta *GlobalTSOAllocator) UpdateTSO() error { - return gta.timestampOracle.UpdateTimestamp(gta.leadership) + return gta.timestampOracle.UpdateTimestamp(gta.member.GetLeadership()) } // SetTSO sets the physical part with given TSO. func (gta *GlobalTSOAllocator) SetTSO(tso uint64, ignoreSmaller, skipUpperBoundCheck bool) error { - return gta.timestampOracle.resetUserTimestampInner(gta.leadership, tso, ignoreSmaller, skipUpperBoundCheck) + return gta.timestampOracle.resetUserTimestampInner(gta.member.GetLeadership(), tso, ignoreSmaller, skipUpperBoundCheck) } // GenerateTSO is used to generate the given number of TSOs. @@ -158,21 +203,21 @@ func (gta *GlobalTSOAllocator) SetTSO(tso uint64, ignoreSmaller, skipUpperBoundC // 2. Estimate a MaxTS and try to write it to all Local TSO Allocator leaders directly to reduce the RTT. // During the process, if the estimated MaxTS is not accurate, it will fallback to the collecting way. func (gta *GlobalTSOAllocator) GenerateTSO(count uint32) (pdpb.Timestamp, error) { - if !gta.leadership.Check() { + if !gta.member.GetLeadership().Check() { tsoCounter.WithLabelValues("not_leader", gta.timestampOracle.dcLocation).Inc() return pdpb.Timestamp{}, errs.ErrGenerateTimestamp.FastGenByArgs(fmt.Sprintf("requested pd %s of cluster", errs.NotLeaderErr)) } // To check if we have any dc-location configured in the cluster - dcLocationMap := gta.allocatorManager.GetClusterDCLocations() + dcLocationMap := gta.am.GetClusterDCLocations() // No dc-locations configured in the cluster, use the normal Global TSO generation way. // (without synchronization with other Local TSO Allocators) if len(dcLocationMap) == 0 { - return gta.timestampOracle.getTS(gta.leadership, count, 0) + return gta.timestampOracle.getTS(gta.member.GetLeadership(), count, 0) } // Have dc-locations configured in the cluster, use the Global TSO generation way. // (whit synchronization with other Local TSO Allocators) - ctx, cancel := context.WithCancel(context.Background()) + ctx, cancel := context.WithCancel(gta.ctx) defer cancel() for i := 0; i < maxRetryCount; i++ { var ( @@ -180,13 +225,15 @@ func (gta *GlobalTSOAllocator) GenerateTSO(count uint32) (pdpb.Timestamp, error) shouldRetry, skipCheck bool globalTSOResp pdpb.Timestamp estimatedMaxTSO *pdpb.Timestamp - suffixBits = gta.allocatorManager.GetSuffixBits() + suffixBits = gta.am.GetSuffixBits() ) // TODO: add a switch to control whether to enable the MaxTSO estimation. // 1. Estimate a MaxTS among all Local TSO Allocator leaders according to the RTT. estimatedMaxTSO, shouldRetry, err = gta.estimateMaxTS(count, suffixBits) if err != nil { - log.Error("global tso allocator estimates MaxTS failed", errs.ZapError(err)) + log.Error("global tso allocator estimates MaxTS failed", + logutil.CondUint32("keyspace-group-id", gta.getGroupID(), gta.getGroupID() > 0), + errs.ZapError(err)) continue } if shouldRetry { @@ -199,7 +246,9 @@ func (gta *GlobalTSOAllocator) GenerateTSO(count uint32) (pdpb.Timestamp, error) // we need to validate it first before we write it into every Local TSO Allocator's memory. globalTSOResp = *estimatedMaxTSO if err = gta.SyncMaxTS(ctx, dcLocationMap, &globalTSOResp, skipCheck); err != nil { - log.Error("global tso allocator synchronizes MaxTS failed", errs.ZapError(err)) + log.Error("global tso allocator synchronizes MaxTS failed", + logutil.CondUint32("keyspace-group-id", gta.getGroupID(), gta.getGroupID() > 0), + errs.ZapError(err)) continue } // 3. If skipCheck is false and the maxTSO is bigger than estimatedMaxTSO, @@ -216,32 +265,36 @@ func (gta *GlobalTSOAllocator) GenerateTSO(count uint32) (pdpb.Timestamp, error) skipCheck = true goto SETTING_PHASE } - // Is skipCheck is false and globalTSOResp remains the same, it means the estimatedTSO is valide. + // Is skipCheck is false and globalTSOResp remains the same, it means the estimatedTSO is valid. if !skipCheck && tsoutil.CompareTimestamp(&globalTSOResp, estimatedMaxTSO) == 0 { tsoCounter.WithLabelValues("global_tso_estimate", gta.timestampOracle.dcLocation).Inc() } // 4. Persist MaxTS into memory, and etcd if needed var currentGlobalTSO *pdpb.Timestamp if currentGlobalTSO, err = gta.getCurrentTSO(); err != nil { - log.Error("global tso allocator gets the current global tso in memory failed", errs.ZapError(err)) + log.Error("global tso allocator gets the current global tso in memory failed", + logutil.CondUint32("keyspace-group-id", gta.getGroupID(), gta.getGroupID() > 0), + errs.ZapError(err)) continue } if tsoutil.CompareTimestamp(currentGlobalTSO, &globalTSOResp) < 0 { tsoCounter.WithLabelValues("global_tso_persist", gta.timestampOracle.dcLocation).Inc() // Update the Global TSO in memory - if err = gta.timestampOracle.resetUserTimestamp(gta.leadership, tsoutil.GenerateTS(&globalTSOResp), true); err != nil { + if err = gta.timestampOracle.resetUserTimestamp(gta.member.GetLeadership(), tsoutil.GenerateTS(&globalTSOResp), true); err != nil { tsoCounter.WithLabelValues("global_tso_persist_err", gta.timestampOracle.dcLocation).Inc() - log.Error("global tso allocator update the global tso in memory failed", errs.ZapError(err)) + log.Error("global tso allocator update the global tso in memory failed", + logutil.CondUint32("keyspace-group-id", gta.getGroupID(), gta.getGroupID() > 0), + errs.ZapError(err)) continue } } // 5. Check leadership again before we returning the response. - if !gta.leadership.Check() { + if !gta.member.GetLeadership().Check() { tsoCounter.WithLabelValues("not_leader_anymore", gta.timestampOracle.dcLocation).Inc() - return pdpb.Timestamp{}, errs.ErrGenerateTimestamp.FastGenByArgs("not the pd leader anymore") + return pdpb.Timestamp{}, errs.ErrGenerateTimestamp.FastGenByArgs(fmt.Sprintf("requested %s anymore", errs.NotLeaderErr)) } - // 6. Differentiate the logical part to make the TSO unique globally by giving it a unique suffix in the whole cluster - globalTSOResp.Logical = gta.timestampOracle.differentiateLogical(globalTSOResp.GetLogical(), suffixBits) + // 6. Calibrate the logical part to make the TSO unique globally by giving it a unique suffix in the whole cluster + globalTSOResp.Logical = gta.timestampOracle.calibrateLogical(globalTSOResp.GetLogical(), suffixBits) globalTSOResp.SuffixBits = uint32(suffixBits) return globalTSOResp, nil } @@ -264,8 +317,9 @@ func (gta *GlobalTSOAllocator) precheckLogical(maxTSO *pdpb.Timestamp, suffixBit return false } // Check if the logical part will reach the overflow condition after being differentiated. - if differentiatedLogical := gta.timestampOracle.differentiateLogical(maxTSO.Logical, suffixBits); differentiatedLogical >= maxLogical { + if caliLogical := gta.timestampOracle.calibrateLogical(maxTSO.Logical, suffixBits); caliLogical >= maxLogical { log.Error("estimated logical part outside of max logical interval, please check ntp time", + logutil.CondUint32("keyspace-group-id", gta.getGroupID(), gta.getGroupID() > 0), zap.Reflect("max-tso", maxTSO), errs.ZapError(errs.ErrLogicOverflow)) tsoCounter.WithLabelValues("precheck_logical_overflow", gta.timestampOracle.dcLocation).Inc() return false @@ -288,7 +342,7 @@ type syncResp struct { // SyncMaxTS is used to sync MaxTS with all Local TSO Allocator leaders in dcLocationMap. // If maxTSO is the biggest TSO among all Local TSO Allocators, it will be written into -// each allocator and remines the same after the synchronization. +// each allocator and remains the same after the synchronization. // If not, it will be replaced with the new max Local TSO and return. func (gta *GlobalTSOAllocator) SyncMaxTS( ctx context.Context, @@ -301,7 +355,7 @@ func (gta *GlobalTSOAllocator) SyncMaxTS( // Collect all allocator leaders' client URLs allocatorLeaders := make(map[string]*pdpb.Member) for dcLocation := range dcLocationMap { - allocator, err := gta.allocatorManager.GetAllocator(dcLocation) + allocator, err := gta.am.GetAllocator(dcLocation) if err != nil { return err } @@ -327,19 +381,20 @@ func (gta *GlobalTSOAllocator) SyncMaxTS( wg := sync.WaitGroup{} request := &pdpb.SyncMaxTSRequest{ Header: &pdpb.RequestHeader{ - SenderId: gta.allocatorManager.member.ID(), + SenderId: gta.am.member.ID(), }, SkipCheck: skipCheck, MaxTs: maxTSO, } for _, leaderURL := range leaderURLs { - leaderConn, err := gta.allocatorManager.getOrCreateGRPCConn(ctx, leaderURL) + leaderConn, err := gta.am.getOrCreateGRPCConn(ctx, leaderURL) if err != nil { return err } // Send SyncMaxTSRequest to all allocator leaders concurrently. wg.Add(1) go func(ctx context.Context, conn *grpc.ClientConn, respCh chan<- *syncResp) { + defer logutil.LogPanic() defer wg.Done() syncMaxTSResp := &syncResp{} syncCtx, cancel := context.WithTimeout(ctx, rpcTimeout) @@ -350,12 +405,17 @@ func (gta *GlobalTSOAllocator) SyncMaxTS( cancel() respCh <- syncMaxTSResp if syncMaxTSResp.err != nil { - log.Error("sync max ts rpc failed, got an error", zap.String("local-allocator-leader-url", leaderConn.Target()), errs.ZapError(err)) + log.Error("sync max ts rpc failed, got an error", + logutil.CondUint32("keyspace-group-id", gta.getGroupID(), gta.getGroupID() > 0), + zap.String("local-allocator-leader-url", leaderConn.Target()), + errs.ZapError(err)) return } if syncMaxTSResp.rpcRes.GetHeader().GetError() != nil { - log.Error("sync max ts rpc failed, got an error", zap.String("local-allocator-leader-url", leaderConn.Target()), - errs.ZapError(errors.Errorf("%s", syncMaxTSResp.rpcRes.GetHeader().GetError().String()))) + log.Error("sync max ts rpc failed, got an error", + logutil.CondUint32("keyspace-group-id", gta.getGroupID(), gta.getGroupID() > 0), + zap.String("local-allocator-leader-url", leaderConn.Target()), + errs.ZapError(errors.New(syncMaxTSResp.rpcRes.GetHeader().GetError().String()))) return } }(ctx, leaderConn, respCh) @@ -403,15 +463,21 @@ func (gta *GlobalTSOAllocator) SyncMaxTS( } // Check whether all dc-locations have been considered during the synchronization and retry once if any dc-location missed. if ok, unsyncedDCs := gta.checkSyncedDCs(dcLocationMap, syncedDCs); !ok { - log.Info("unsynced dc-locations found, will retry", zap.Bool("skip-check", skipCheck), zap.Strings("synced-DCs", syncedDCs), zap.Strings("unsynced-DCs", unsyncedDCs)) + log.Info("unsynced dc-locations found, will retry", + logutil.CondUint32("keyspace-group-id", gta.getGroupID(), gta.getGroupID() > 0), + zap.Bool("skip-check", skipCheck), + zap.Strings("synced-DCs", syncedDCs), + zap.Strings("unsynced-DCs", unsyncedDCs)) if i < syncMaxRetryCount-1 { // maxTSO should remain the same. *maxTSO = originalMaxTSO // To make sure we have the latest dc-location info - gta.allocatorManager.ClusterDCLocationChecker() + gta.am.ClusterDCLocationChecker() continue } - return errs.ErrSyncMaxTS.FastGenByArgs(fmt.Sprintf("unsynced dc-locations found, skip-check: %t, synced dc-locations: %+v, unsynced dc-locations: %+v", skipCheck, syncedDCs, unsyncedDCs)) + return errs.ErrSyncMaxTS.FastGenByArgs( + fmt.Sprintf("unsynced dc-locations found, skip-check: %t, synced dc-locations: %+v, unsynced dc-locations: %+v", + skipCheck, syncedDCs, unsyncedDCs)) } // Update the sync RTT to help estimate MaxTS later. if maxTSORtt != 0 { @@ -428,7 +494,10 @@ func (gta *GlobalTSOAllocator) checkSyncedDCs(dcLocationMap map[string]DCLocatio unsyncedDCs = append(unsyncedDCs, dcLocation) } } - log.Debug("check unsynced dc-locations", zap.Strings("unsynced-DCs", unsyncedDCs), zap.Strings("synced-DCs", syncedDCs)) + log.Debug("check unsynced dc-locations", + logutil.CondUint32("keyspace-group-id", gta.getGroupID(), gta.getGroupID() > 0), + zap.Strings("unsynced-DCs", unsyncedDCs), + zap.Strings("synced-DCs", syncedDCs)) return len(unsyncedDCs) == 0, unsyncedDCs } @@ -445,3 +514,123 @@ func (gta *GlobalTSOAllocator) Reset() { tsoAllocatorRole.WithLabelValues(gta.timestampOracle.dcLocation).Set(0) gta.timestampOracle.ResetTimestamp() } + +func (gta *GlobalTSOAllocator) primaryElectionLoop() { + defer logutil.LogPanic() + defer gta.wg.Done() + + for { + select { + case <-gta.ctx.Done(): + log.Info("exit the global tso primary election loop", + logutil.CondUint32("keyspace-group-id", gta.getGroupID(), gta.getGroupID() > 0)) + return + default: + } + + primary, checkAgain := gta.member.CheckLeader() + if checkAgain { + continue + } + if primary != nil { + log.Info("start to watch the primary", + logutil.CondUint32("keyspace-group-id", gta.getGroupID(), gta.getGroupID() > 0), + zap.String("campaign-tso-primary-name", gta.member.Name()), + zap.Stringer("tso-primary", primary)) + // Watch will keep looping and never return unless the primary has changed. + primary.Watch(gta.ctx) + log.Info("the tso primary has changed, try to re-campaign a primary", + logutil.CondUint32("keyspace-group-id", gta.getGroupID(), gta.getGroupID() > 0)) + } + + gta.campaignLeader() + } +} + +func (gta *GlobalTSOAllocator) campaignLeader() { + log.Info("start to campaign the primary", + logutil.CondUint32("keyspace-group-id", gta.getGroupID(), gta.getGroupID() > 0), + zap.String("campaign-tso-primary-name", gta.member.Name())) + if err := gta.am.member.CampaignLeader(gta.am.leaderLease); err != nil { + if errors.Is(err, errs.ErrEtcdTxnConflict) { + log.Info("campaign tso primary meets error due to txn conflict, another tso server may campaign successfully", + logutil.CondUint32("keyspace-group-id", gta.getGroupID(), gta.getGroupID() > 0), + zap.String("campaign-tso-primary-name", gta.member.Name())) + } else if errors.Is(err, errs.ErrCheckCampaign) { + log.Info("campaign tso primary meets error due to pre-check campaign failed, the tso keyspace group may be in split", + logutil.CondUint32("keyspace-group-id", gta.getGroupID(), gta.getGroupID() > 0), + zap.String("campaign-tso-primary-name", gta.member.Name())) + } else { + log.Error("campaign tso primary meets error due to etcd error", + logutil.CondUint32("keyspace-group-id", gta.getGroupID(), gta.getGroupID() > 0), + zap.String("campaign-tso-primary-name", gta.member.Name()), errs.ZapError(err)) + } + return + } + + // Start keepalive the leadership and enable TSO service. + // TSO service is strictly enabled/disabled by the leader lease for 2 reasons: + // 1. lease based approach is not affected by thread pause, slow runtime schedule, etc. + // 2. load region could be slow. Based on lease we can recover TSO service faster. + ctx, cancel := context.WithCancel(gta.ctx) + var resetLeaderOnce sync.Once + defer resetLeaderOnce.Do(func() { + cancel() + gta.member.ResetLeader() + }) + + // maintain the the leadership, after this, TSO can be service. + gta.member.KeepLeader(ctx) + log.Info("campaign tso primary ok", + logutil.CondUint32("keyspace-group-id", gta.getGroupID(), gta.getGroupID() > 0), + zap.String("campaign-tso-primary-name", gta.member.Name())) + + allocator, err := gta.am.GetAllocator(GlobalDCLocation) + if err != nil { + log.Error("failed to get the global tso allocator", + logutil.CondUint32("keyspace-group-id", gta.getGroupID(), gta.getGroupID() > 0), + errs.ZapError(err)) + return + } + log.Info("initializing the global tso allocator") + if err := allocator.Initialize(0); err != nil { + log.Error("failed to initialize the global tso allocator", + logutil.CondUint32("keyspace-group-id", gta.getGroupID(), gta.getGroupID() > 0), + errs.ZapError(err)) + return + } + defer func() { + gta.am.ResetAllocatorGroup(GlobalDCLocation) + }() + + gta.member.EnableLeader() + defer resetLeaderOnce.Do(func() { + cancel() + gta.member.ResetLeader() + }) + + // TODO: if enable-local-tso is true, check the cluster dc-location after the primary is elected + // go gta.tsoAllocatorManager.ClusterDCLocationChecker() + log.Info("tso primary is ready to serve", + logutil.CondUint32("keyspace-group-id", gta.getGroupID(), gta.getGroupID() > 0), + zap.String("tso-primary-name", gta.member.Name())) + + leaderTicker := time.NewTicker(mcsutils.LeaderTickInterval) + defer leaderTicker.Stop() + + for { + select { + case <-leaderTicker.C: + if !gta.member.IsLeader() { + log.Info("no longer a primary because lease has expired, the tso primary will step down", + logutil.CondUint32("keyspace-group-id", gta.getGroupID(), gta.getGroupID() > 0)) + return + } + case <-ctx.Done(): + // Server is closed and it should return nil. + log.Info("exit leader campaign", + logutil.CondUint32("keyspace-group-id", gta.getGroupID(), gta.getGroupID() > 0)) + return + } + } +} diff --git a/pkg/tso/keyspace_group_manager.go b/pkg/tso/keyspace_group_manager.go new file mode 100644 index 00000000000..9dd06242a78 --- /dev/null +++ b/pkg/tso/keyspace_group_manager.go @@ -0,0 +1,1433 @@ +// Copyright 2023 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tso + +import ( + "context" + "encoding/json" + "fmt" + "math" + "net/http" + "regexp" + "sort" + "strings" + "sync" + "time" + + perrors "github.com/pingcap/errors" + "github.com/pingcap/failpoint" + "github.com/pingcap/kvproto/pkg/pdpb" + "github.com/pingcap/log" + "github.com/tikv/pd/pkg/election" + "github.com/tikv/pd/pkg/errs" + "github.com/tikv/pd/pkg/mcs/discovery" + mcsutils "github.com/tikv/pd/pkg/mcs/utils" + "github.com/tikv/pd/pkg/member" + "github.com/tikv/pd/pkg/slice" + "github.com/tikv/pd/pkg/storage/endpoint" + "github.com/tikv/pd/pkg/storage/kv" + "github.com/tikv/pd/pkg/utils/apiutil" + "github.com/tikv/pd/pkg/utils/etcdutil" + "github.com/tikv/pd/pkg/utils/logutil" + "github.com/tikv/pd/pkg/utils/memberutil" + "github.com/tikv/pd/pkg/utils/tsoutil" + "github.com/tikv/pd/pkg/utils/typeutil" + "go.etcd.io/etcd/clientv3" + "go.etcd.io/etcd/mvcc/mvccpb" + "go.uber.org/zap" +) + +const ( + // mergingCheckInterval is the interval for merging check to see if the keyspace groups + // merging process could be moved forward. + mergingCheckInterval = 5 * time.Second + // defaultPrimaryPriorityCheckInterval is the default interval for checking if the priorities + // of the primaries on this TSO server/pod have changed. A goroutine will periodically check + // do this check and re-distribute the primaries if necessary. + defaultPrimaryPriorityCheckInterval = 10 * time.Second + groupPatrolInterval = time.Minute +) + +type state struct { + sync.RWMutex + // ams stores the allocator managers of the keyspace groups. Each keyspace group is + // assigned with an allocator manager managing its global/local tso allocators. + // Use a fixed size array to maximize the efficiency of concurrent access to + // different keyspace groups for tso service. + ams [mcsutils.MaxKeyspaceGroupCountInUse]*AllocatorManager + // kgs stores the keyspace groups' membership/distribution meta. + kgs [mcsutils.MaxKeyspaceGroupCountInUse]*endpoint.KeyspaceGroup + // keyspaceLookupTable is a map from keyspace to the keyspace group to which it belongs. + keyspaceLookupTable map[uint32]uint32 + // splittingGroups is the cache of splitting keyspace group related information. + splittingGroups map[uint32]struct{} + // deletedGroups is the cache of deleted keyspace group related information. + deletedGroups map[uint32]struct{} +} + +func (s *state) initialize() { + s.keyspaceLookupTable = make(map[uint32]uint32) + s.splittingGroups = make(map[uint32]struct{}) + s.deletedGroups = make(map[uint32]struct{}) +} + +func (s *state) deInitialize() { + log.Info("closing all keyspace groups") + + s.Lock() + defer s.Unlock() + + wg := sync.WaitGroup{} + for _, am := range s.ams { + if am != nil { + wg.Add(1) + go func(am *AllocatorManager) { + defer logutil.LogPanic() + defer wg.Done() + am.close() + log.Info("keyspace group closed", zap.Uint32("keyspace-group-id", am.kgID)) + }(am) + } + } + wg.Wait() + + log.Info("all keyspace groups closed") +} + +// getKeyspaceGroupMeta returns the meta of the given keyspace group +func (s *state) getKeyspaceGroupMeta( + groupID uint32, +) (*AllocatorManager, *endpoint.KeyspaceGroup) { + s.RLock() + defer s.RUnlock() + return s.ams[groupID], s.kgs[groupID] +} + +// getSplittingGroups returns the IDs of the splitting keyspace groups. +func (s *state) getSplittingGroups() []uint32 { + s.RLock() + defer s.RUnlock() + groups := make([]uint32, 0, len(s.splittingGroups)) + for groupID := range s.splittingGroups { + groups = append(groups, groupID) + } + return groups +} + +// getDeletedGroups returns the IDs of the deleted keyspace groups. +func (s *state) getDeletedGroups() []uint32 { + s.RLock() + defer s.RUnlock() + groups := make([]uint32, 0, len(s.deletedGroups)) + for groupID := range s.deletedGroups { + groups = append(groups, groupID) + } + return groups +} + +func (s *state) checkTSOSplit( + targetGroupID uint32, +) (splitTargetAM, splitSourceAM *AllocatorManager, err error) { + s.RLock() + defer s.RUnlock() + splitTargetAM, splitTargetGroup := s.ams[targetGroupID], s.kgs[targetGroupID] + // Only the split target keyspace group needs to check the TSO split. + if !splitTargetGroup.IsSplitTarget() { + return nil, nil, nil // it isn't in the split state + } + sourceGroupID := splitTargetGroup.SplitSource() + splitSourceAM, splitSourceGroup := s.ams[sourceGroupID], s.kgs[sourceGroupID] + if splitSourceAM == nil || splitSourceGroup == nil { + log.Error("the split source keyspace group is not initialized", + zap.Uint32("source", sourceGroupID)) + return nil, nil, errs.ErrKeyspaceGroupNotInitialized.FastGenByArgs(sourceGroupID) + } + return splitTargetAM, splitSourceAM, nil +} + +// Reject any request if the keyspace group is in merging state, +// we need to wait for the merging checker to finish the TSO merging. +func (s *state) checkTSOMerge( + groupID uint32, +) error { + s.RLock() + defer s.RUnlock() + if s.kgs[groupID] == nil || !s.kgs[groupID].IsMerging() { + return nil + } + return errs.ErrKeyspaceGroupIsMerging.FastGenByArgs(groupID) +} + +// getKeyspaceGroupMetaWithCheck returns the keyspace group meta of the given keyspace. +// It also checks if the keyspace is served by the given keyspace group. If not, it returns the meta +// of the keyspace group to which the keyspace currently belongs and returns NotServed (by the given +// keyspace group) error. If the keyspace doesn't belong to any keyspace group, it returns the +// NotAssigned error, which could happen because loading keyspace group meta isn't atomic when there is +// keyspace movement between keyspace groups. +func (s *state) getKeyspaceGroupMetaWithCheck( + keyspaceID, keyspaceGroupID uint32, +) (*AllocatorManager, *endpoint.KeyspaceGroup, uint32, error) { + s.RLock() + defer s.RUnlock() + + if am := s.ams[keyspaceGroupID]; am != nil { + kg := s.kgs[keyspaceGroupID] + if kg != nil { + if _, ok := kg.KeyspaceLookupTable[keyspaceID]; ok { + return am, kg, keyspaceGroupID, nil + } + } + } + + // The keyspace doesn't belong to this keyspace group, we should check if it belongs to any other + // keyspace groups, and return the correct keyspace group meta to the client. + if kgid, ok := s.keyspaceLookupTable[keyspaceID]; ok { + if s.ams[kgid] != nil { + return s.ams[kgid], s.kgs[kgid], kgid, nil + } + return nil, s.kgs[kgid], kgid, genNotServedErr(errs.ErrGetAllocatorManager, keyspaceGroupID) + } + + // The keyspace doesn't belong to any keyspace group but the keyspace has been assigned to a + // keyspace group before, which means the keyspace group hasn't initialized yet. + if keyspaceGroupID != mcsutils.DefaultKeyspaceGroupID { + return nil, nil, keyspaceGroupID, errs.ErrKeyspaceNotAssigned.FastGenByArgs(keyspaceID) + } + + // For migrating the existing keyspaces which have no keyspace group assigned as configured + // in the keyspace meta. All these keyspaces will be served by the default keyspace group. + if s.ams[mcsutils.DefaultKeyspaceGroupID] == nil { + return nil, nil, mcsutils.DefaultKeyspaceGroupID, + errs.ErrKeyspaceNotAssigned.FastGenByArgs(keyspaceID) + } + return s.ams[mcsutils.DefaultKeyspaceGroupID], + s.kgs[mcsutils.DefaultKeyspaceGroupID], + mcsutils.DefaultKeyspaceGroupID, nil +} + +func (s *state) getNextPrimaryToReset( + groupID int, localAddress string, +) (member ElectionMember, kg *endpoint.KeyspaceGroup, localPriority, nextGroupID int) { + s.RLock() + defer s.RUnlock() + + // Both s.ams and s.kgs are arrays with the fixed size defined by the const value MaxKeyspaceGroupCountInUse. + groupSize := int(mcsutils.MaxKeyspaceGroupCountInUse) + groupID %= groupSize + for j := 0; j < groupSize; groupID, j = (groupID+1)%groupSize, j+1 { + am := s.ams[groupID] + kg := s.kgs[groupID] + if am != nil && kg != nil && am.GetMember().IsLeader() { + maxPriority := math.MinInt32 + localPriority := math.MaxInt32 + for _, member := range kg.Members { + if member.Priority > maxPriority { + maxPriority = member.Priority + } + if member.Address == localAddress { + localPriority = member.Priority + } + } + + if localPriority < maxPriority { + // return here and reset the primary outside of the critical section + // as resetting the primary may take some time. + return am.GetMember(), kg, localPriority, (groupID + 1) % groupSize + } + } + } + + return nil, nil, 0, groupID +} + +// KeyspaceGroupManager manages the members of the keyspace groups assigned to this host. +// The replicas campaign for the leaders which provide the tso service for the corresponding +// keyspace groups. +type KeyspaceGroupManager struct { + // state is the in-memory state of the keyspace groups + state + + ctx context.Context + cancel context.CancelFunc + wg sync.WaitGroup + + // tsoServiceID is the service ID of the TSO service, registered in the service discovery + tsoServiceID *discovery.ServiceRegistryEntry + etcdClient *clientv3.Client + httpClient *http.Client + // electionNamePrefix is the name prefix to generate the unique name of a participant, + // which participate in the election of its keyspace group's primary, in the format of + // "electionNamePrefix:keyspace-group-id" + electionNamePrefix string + // tsoServiceKey is the path for storing the registered tso servers. + // Key: /ms/{cluster_id}/tso/registry/{tsoServerAddress} + // Value: discover.ServiceRegistryEntry + tsoServiceKey string + // legacySvcRootPath defines the legacy root path for all etcd paths which derives from + // the PD/API service. It's in the format of "/pd/{cluster_id}". + // The main paths for different usages include: + // 1. The path, used by the default keyspace group, for LoadTimestamp/SaveTimestamp in the + // storage endpoint. + // Key: /pd/{cluster_id}/timestamp + // Value: ts(time.Time) + // Key: /pd/{cluster_id}/lta/{dc-location}/timestamp + // Value: ts(time.Time) + // 2. The path for storing keyspace group membership/distribution metadata. + // Key: /pd/{cluster_id}/tso/keyspace_groups/membership/{group} + // Value: endpoint.KeyspaceGroup + // Note: The {group} is 5 digits integer with leading zeros. + legacySvcRootPath string + // tsoSvcRootPath defines the root path for all etcd paths used in the tso microservices. + // It is in the format of "/ms//tso". + // The main paths for different usages include: + // 1. The path for keyspace group primary election. + // default keyspace group: "/ms/{cluster_id}/tso/00000/primary". + // non-default keyspace group: "/ms/{cluster_id}/tso/keyspace_groups/election/{group}/primary". + // 2. The path for LoadTimestamp/SaveTimestamp in the storage endpoint for all the non-default + // keyspace groups. + // Key: /ms/{cluster_id}/tso/{group}/gta/timestamp + // Value: ts(time.Time) + // Key: /ms/{cluster_id}/tso/{group}/lta/{dc-location}/timestamp + // Value: ts(time.Time) + // Note: The {group} is 5 digits integer with leading zeros. + tsoSvcRootPath string + // legacySvcStorage is storage with legacySvcRootPath. + legacySvcStorage *endpoint.StorageEndpoint + // tsoSvcStorage is storage with tsoSvcRootPath. + tsoSvcStorage *endpoint.StorageEndpoint + // cfg is the TSO config + cfg ServiceConfig + + // loadKeyspaceGroupsTimeout is the timeout for loading the initial keyspace group assignment. + loadKeyspaceGroupsTimeout time.Duration + loadKeyspaceGroupsBatchSize int64 + loadFromEtcdMaxRetryTimes int + + // compiledKGMembershipIDRegexp is the compiled regular expression for matching keyspace group id + // in the keyspace group membership path. + compiledKGMembershipIDRegexp *regexp.Regexp + // groupUpdateRetryList is the list of keyspace groups which failed to update and need to retry. + groupUpdateRetryList map[uint32]*endpoint.KeyspaceGroup + groupWatcher *etcdutil.LoopWatcher + + // mergeCheckerCancelMap is the cancel function map for the merge checker of each keyspace group. + mergeCheckerCancelMap sync.Map // GroupID -> context.CancelFunc + + primaryPriorityCheckInterval time.Duration + + // tsoNodes is the registered tso servers. + tsoNodes sync.Map // store as map[string]struct{} + // serviceRegistryMap stores the mapping from the service registry key to the service address. + // Note: it is only used in tsoNodesWatcher. + serviceRegistryMap map[string]string + // tsoNodesWatcher is the watcher for the registered tso servers. + tsoNodesWatcher *etcdutil.LoopWatcher +} + +// NewKeyspaceGroupManager creates a new Keyspace Group Manager. +func NewKeyspaceGroupManager( + ctx context.Context, + tsoServiceID *discovery.ServiceRegistryEntry, + etcdClient *clientv3.Client, + httpClient *http.Client, + electionNamePrefix string, + tsoServiceKey string, + legacySvcRootPath string, + tsoSvcRootPath string, + cfg ServiceConfig, +) *KeyspaceGroupManager { + if mcsutils.MaxKeyspaceGroupCountInUse > mcsutils.MaxKeyspaceGroupCount { + log.Fatal("MaxKeyspaceGroupCountInUse is larger than MaxKeyspaceGroupCount", + zap.Uint32("max-keyspace-group-count-in-use", mcsutils.MaxKeyspaceGroupCountInUse), + zap.Uint32("max-keyspace-group-count", mcsutils.MaxKeyspaceGroupCount)) + } + + ctx, cancel := context.WithCancel(ctx) + kgm := &KeyspaceGroupManager{ + ctx: ctx, + cancel: cancel, + tsoServiceID: tsoServiceID, + etcdClient: etcdClient, + httpClient: httpClient, + electionNamePrefix: electionNamePrefix, + tsoServiceKey: tsoServiceKey, + legacySvcRootPath: legacySvcRootPath, + tsoSvcRootPath: tsoSvcRootPath, + primaryPriorityCheckInterval: defaultPrimaryPriorityCheckInterval, + cfg: cfg, + groupUpdateRetryList: make(map[uint32]*endpoint.KeyspaceGroup), + serviceRegistryMap: make(map[string]string), + } + kgm.legacySvcStorage = endpoint.NewStorageEndpoint( + kv.NewEtcdKVBase(kgm.etcdClient, kgm.legacySvcRootPath), nil) + kgm.tsoSvcStorage = endpoint.NewStorageEndpoint( + kv.NewEtcdKVBase(kgm.etcdClient, kgm.tsoSvcRootPath), nil) + kgm.compiledKGMembershipIDRegexp = endpoint.GetCompiledKeyspaceGroupIDRegexp() + kgm.state.initialize() + return kgm +} + +// Initialize this KeyspaceGroupManager +func (kgm *KeyspaceGroupManager) Initialize() error { + if err := kgm.InitializeTSOServerWatchLoop(); err != nil { + log.Error("failed to initialize tso server watch loop", zap.Error(err)) + kgm.Close() // Close the manager to clean up the allocated resources. + return errs.ErrLoadKeyspaceGroupsTerminated.Wrap(err) + } + if err := kgm.InitializeGroupWatchLoop(); err != nil { + log.Error("failed to initialize group watch loop", zap.Error(err)) + kgm.Close() // Close the manager to clean up the loaded keyspace groups. + return errs.ErrLoadKeyspaceGroupsTerminated.Wrap(err) + } + + kgm.wg.Add(3) + go kgm.primaryPriorityCheckLoop() + go kgm.groupSplitPatroller() + go kgm.deletedGroupCleaner() + + return nil +} + +// Close this KeyspaceGroupManager +func (kgm *KeyspaceGroupManager) Close() { + log.Info("closing keyspace group manager") + + // Note: don't change the order. We need to cancel all service loops in the keyspace group manager + // before closing all keyspace groups. It's to prevent concurrent addition/removal of keyspace groups + // during critical periods such as service shutdown and online keyspace group, while the former requires + // snapshot isolation to ensure all keyspace groups are properly closed and no new keyspace group is + // added/initialized after that. + kgm.cancel() + kgm.wg.Wait() + kgm.state.deInitialize() + + log.Info("keyspace group manager closed") +} + +// GetServiceConfig returns the service config. +func (kgm *KeyspaceGroupManager) GetServiceConfig() ServiceConfig { + return kgm.cfg +} + +// InitializeTSOServerWatchLoop initializes the watch loop monitoring the path for storing the +// registered tso servers. +// Key: /ms/{cluster_id}/tso/registry/{tsoServerAddress} +// Value: discover.ServiceRegistryEntry +func (kgm *KeyspaceGroupManager) InitializeTSOServerWatchLoop() error { + tsoServiceEndKey := clientv3.GetPrefixRangeEnd(kgm.tsoServiceKey) + "/" + + putFn := func(kv *mvccpb.KeyValue) error { + s := &discovery.ServiceRegistryEntry{} + if err := json.Unmarshal(kv.Value, s); err != nil { + log.Warn("failed to unmarshal service registry entry", + zap.String("event-kv-key", string(kv.Key)), zap.Error(err)) + return err + } + kgm.tsoNodes.Store(s.ServiceAddr, struct{}{}) + kgm.serviceRegistryMap[string(kv.Key)] = s.ServiceAddr + return nil + } + deleteFn := func(kv *mvccpb.KeyValue) error { + key := string(kv.Key) + if serviceAddr, ok := kgm.serviceRegistryMap[key]; ok { + delete(kgm.serviceRegistryMap, key) + kgm.tsoNodes.Delete(serviceAddr) + return nil + } + return perrors.Errorf("failed to find the service address for key %s", key) + } + + kgm.tsoNodesWatcher = etcdutil.NewLoopWatcher( + kgm.ctx, + &kgm.wg, + kgm.etcdClient, + "tso-nodes-watcher", + kgm.tsoServiceKey, + putFn, + deleteFn, + func() error { return nil }, + clientv3.WithRange(tsoServiceEndKey), + ) + kgm.tsoNodesWatcher.StartWatchLoop() + if err := kgm.tsoNodesWatcher.WaitLoad(); err != nil { + log.Error("failed to load the registered tso servers", errs.ZapError(err)) + return err + } + + return nil +} + +// InitializeGroupWatchLoop initializes the watch loop monitoring the path for storing keyspace group +// membership/distribution metadata. +// Key: /pd/{cluster_id}/tso/keyspace_groups/membership/{group} +// Value: endpoint.KeyspaceGroup +func (kgm *KeyspaceGroupManager) InitializeGroupWatchLoop() error { + rootPath := kgm.legacySvcRootPath + startKey := strings.Join([]string{rootPath, endpoint.KeyspaceGroupIDPath(mcsutils.DefaultKeyspaceGroupID)}, "/") + endKey := strings.Join( + []string{rootPath, clientv3.GetPrefixRangeEnd(endpoint.KeyspaceGroupIDPrefix())}, "/") + + defaultKGConfigured := false + putFn := func(kv *mvccpb.KeyValue) error { + group := &endpoint.KeyspaceGroup{} + if err := json.Unmarshal(kv.Value, group); err != nil { + return errs.ErrJSONUnmarshal.Wrap(err).FastGenWithCause() + } + kgm.updateKeyspaceGroup(group) + if group.ID == mcsutils.DefaultKeyspaceGroupID { + defaultKGConfigured = true + } + return nil + } + deleteFn := func(kv *mvccpb.KeyValue) error { + groupID, err := ExtractKeyspaceGroupIDFromPath(kgm.compiledKGMembershipIDRegexp, string(kv.Key)) + if err != nil { + return err + } + kgm.deleteKeyspaceGroup(groupID) + return nil + } + postEventFn := func() error { + // Retry the groups that are not initialized successfully before. + for id, group := range kgm.groupUpdateRetryList { + delete(kgm.groupUpdateRetryList, id) + kgm.updateKeyspaceGroup(group) + } + return nil + } + kgm.groupWatcher = etcdutil.NewLoopWatcher( + kgm.ctx, + &kgm.wg, + kgm.etcdClient, + "keyspace-watcher", + startKey, + putFn, + deleteFn, + postEventFn, + clientv3.WithRange(endKey), + ) + if kgm.loadKeyspaceGroupsTimeout > 0 { + kgm.groupWatcher.SetLoadTimeout(kgm.loadKeyspaceGroupsTimeout) + } + if kgm.loadFromEtcdMaxRetryTimes > 0 { + kgm.groupWatcher.SetLoadRetryTimes(kgm.loadFromEtcdMaxRetryTimes) + } + if kgm.loadKeyspaceGroupsBatchSize > 0 { + kgm.groupWatcher.SetLoadBatchSize(kgm.loadKeyspaceGroupsBatchSize) + } + kgm.groupWatcher.StartWatchLoop() + if err := kgm.groupWatcher.WaitLoad(); err != nil { + log.Error("failed to initialize keyspace group manager", errs.ZapError(err)) + // We might have partially loaded/initialized the keyspace groups. Close the manager to clean up. + kgm.Close() + return errs.ErrLoadKeyspaceGroupsTerminated.Wrap(err) + } + + if !defaultKGConfigured { + log.Info("initializing default keyspace group") + group := &endpoint.KeyspaceGroup{ + ID: mcsutils.DefaultKeyspaceGroupID, + Members: []endpoint.KeyspaceGroupMember{{ + Address: kgm.tsoServiceID.ServiceAddr, + Priority: mcsutils.DefaultKeyspaceGroupReplicaPriority, + }}, + Keyspaces: []uint32{mcsutils.DefaultKeyspaceID}, + } + kgm.updateKeyspaceGroup(group) + } + return nil +} + +func (kgm *KeyspaceGroupManager) primaryPriorityCheckLoop() { + defer logutil.LogPanic() + defer kgm.wg.Done() + + failpoint.Inject("fastPrimaryPriorityCheck", func() { + kgm.primaryPriorityCheckInterval = 200 * time.Millisecond + }) + + ticker := time.NewTicker(kgm.primaryPriorityCheckInterval) + defer ticker.Stop() + ctx, cancel := context.WithCancel(kgm.ctx) + defer cancel() + groupID := 0 + for { + select { + case <-ctx.Done(): + log.Info("exit primary priority check loop") + return + case <-ticker.C: + // Every primaryPriorityCheckInterval, we only reset the primary of one keyspace group + member, kg, localPriority, nextGroupID := kgm.getNextPrimaryToReset(groupID, kgm.tsoServiceID.ServiceAddr) + if member != nil { + aliveTSONodes := make(map[string]struct{}) + kgm.tsoNodes.Range(func(key, _ interface{}) bool { + aliveTSONodes[key.(string)] = struct{}{} + return true + }) + if len(aliveTSONodes) == 0 { + log.Warn("no alive tso node", zap.String("local-address", kgm.tsoServiceID.ServiceAddr)) + continue + } + // If there is a alive member with higher priority, reset the leader. + resetLeader := false + for _, member := range kg.Members { + if member.Priority <= localPriority { + continue + } + if _, ok := aliveTSONodes[member.Address]; ok { + resetLeader = true + break + } + } + if resetLeader { + select { + case <-ctx.Done(): + default: + member.ResetLeader() + log.Info("reset primary", + zap.String("local-address", kgm.tsoServiceID.ServiceAddr), + zap.Uint32("keyspace-group-id", kg.ID), + zap.Int("local-priority", localPriority)) + } + } else { + log.Warn("no need to reset primary as the replicas with higher priority are offline", + zap.String("local-address", kgm.tsoServiceID.ServiceAddr), + zap.Uint32("keyspace-group-id", kg.ID), + zap.Int("local-priority", localPriority)) + } + } + groupID = nextGroupID + } + } +} + +func (kgm *KeyspaceGroupManager) isAssignedToMe(group *endpoint.KeyspaceGroup) bool { + return slice.AnyOf(group.Members, func(i int) bool { + return group.Members[i].Address == kgm.tsoServiceID.ServiceAddr + }) +} + +// updateKeyspaceGroup applies the given keyspace group. If the keyspace group is just assigned to +// this host/pod, it will join the primary election. +func (kgm *KeyspaceGroupManager) updateKeyspaceGroup(group *endpoint.KeyspaceGroup) { + if err := kgm.checkKeySpaceGroupID(group.ID); err != nil { + log.Warn("keyspace group ID is invalid, ignore it", zap.Error(err)) + return + } + + // If the default keyspace group isn't assigned to any tso node/pod, assign it to everyone. + if group.ID == mcsutils.DefaultKeyspaceGroupID && len(group.Members) == 0 { + // TODO: fill members with all tso nodes/pods. + group.Members = []endpoint.KeyspaceGroupMember{{ + Address: kgm.tsoServiceID.ServiceAddr, + Priority: mcsutils.DefaultKeyspaceGroupReplicaPriority, + }} + } + + if !kgm.isAssignedToMe(group) { + // Not assigned to me. If this host/pod owns a replica of this keyspace group, + // it should resign the election membership now. + kgm.exitElectionMembership(group) + return + } + + oldAM, oldGroup := kgm.getKeyspaceGroupMeta(group.ID) + // If this host owns a replica of the keyspace group which is the merge target, + // it should run the merging checker when the merge state first time changes. + if !oldGroup.IsMergeTarget() && group.IsMergeTarget() { + ctx, cancel := context.WithCancel(kgm.ctx) + kgm.mergeCheckerCancelMap.Store(group.ID, cancel) + kgm.wg.Add(1) + go kgm.mergingChecker(ctx, group.ID, group.MergeState.MergeList) + } + // If the merge state has been finished, cancel its merging checker. + if oldGroup.IsMergeTarget() && !group.IsMergeTarget() { + if cancel, loaded := kgm.mergeCheckerCancelMap.LoadAndDelete(group.ID); loaded && cancel != nil { + cancel.(context.CancelFunc)() + } + } + + // If this host is already assigned a replica of this keyspace group, i.e., the election member + // is already initialized, just update the meta. + if oldAM != nil { + kgm.updateKeyspaceGroupMembership(oldGroup, group, true) + return + } + + // If the keyspace group is not initialized, initialize it. + // The format of leader name is address-groupID. + uniqueName := fmt.Sprintf("%s-%05d", kgm.electionNamePrefix, group.ID) + uniqueID := memberutil.GenerateUniqueID(uniqueName) + log.Info("joining primary election", + zap.Uint32("keyspace-group-id", group.ID), + zap.String("participant-name", uniqueName), + zap.Uint64("participant-id", uniqueID)) + // Initialize the participant info to join the primary election. + participant := member.NewParticipant(kgm.etcdClient) + participant.InitInfo( + uniqueName, uniqueID, endpoint.KeyspaceGroupsElectionPath(kgm.tsoSvcRootPath, group.ID), + mcsutils.PrimaryKey, "keyspace group primary election", kgm.cfg.GetAdvertiseListenAddr()) + // If the keyspace group is in split, we should ensure that the primary elected by the new keyspace group + // is always on the same TSO Server node as the primary of the old keyspace group, and this constraint cannot + // be broken until the entire split process is completed. + if group.IsSplitTarget() { + splitSource := group.SplitSource() + log.Info("keyspace group is in split", + zap.Uint32("target", group.ID), + zap.Uint32("source", splitSource)) + splitSourceAM, splitSourceGroup := kgm.getKeyspaceGroupMeta(splitSource) + if !validateSplit(splitSourceAM, group, splitSourceGroup) { + // Put the group into the retry list to retry later. + kgm.groupUpdateRetryList[group.ID] = group + return + } + participant.SetCampaignChecker(func(leadership *election.Leadership) bool { + return splitSourceAM.GetMember().IsLeader() + }) + } + // Only the default keyspace group uses the legacy service root path for LoadTimestamp/SyncTimestamp. + var ( + tsRootPath string + storage *endpoint.StorageEndpoint + ) + if group.ID == mcsutils.DefaultKeyspaceGroupID { + tsRootPath = kgm.legacySvcRootPath + storage = kgm.legacySvcStorage + } else { + tsRootPath = kgm.tsoSvcRootPath + storage = kgm.tsoSvcStorage + } + // Initialize all kinds of maps. + am := NewAllocatorManager(kgm.ctx, group.ID, participant, tsRootPath, storage, kgm.cfg, true) + log.Info("created allocator manager", + zap.Uint32("keyspace-group-id", group.ID), + zap.String("timestamp-path", am.GetTimestampPath(""))) + kgm.Lock() + group.KeyspaceLookupTable = make(map[uint32]struct{}) + for _, kid := range group.Keyspaces { + group.KeyspaceLookupTable[kid] = struct{}{} + kgm.keyspaceLookupTable[kid] = group.ID + } + kgm.kgs[group.ID] = group + kgm.ams[group.ID] = am + // If the group is the split target, add it to the splitting group map. + if group.IsSplitTarget() { + kgm.splittingGroups[group.ID] = struct{}{} + } + kgm.Unlock() +} + +// validateSplit checks whether the meta info of split keyspace group +// to ensure that the split process could be continued. +func validateSplit( + sourceAM *AllocatorManager, + targetGroup, sourceGroup *endpoint.KeyspaceGroup, +) bool { + splitSourceID := targetGroup.SplitSource() + // Make sure that the split source keyspace group has been initialized. + if sourceAM == nil || sourceGroup == nil { + log.Error("the split source keyspace group is not initialized", + zap.Uint32("target", targetGroup.ID), + zap.Uint32("source", splitSourceID)) + return false + } + // Since the target group is derived from the source group and both of them + // could not be modified during the split process, so we can only check the + // member count of the source group here. + memberCount := len(sourceGroup.Members) + if memberCount < mcsutils.DefaultKeyspaceGroupReplicaCount { + log.Error("the split source keyspace group does not have enough members", + zap.Uint32("target", targetGroup.ID), + zap.Uint32("source", splitSourceID), + zap.Int("member-count", memberCount), + zap.Int("replica-count", mcsutils.DefaultKeyspaceGroupReplicaCount)) + return false + } + return true +} + +// updateKeyspaceGroupMembership updates the keyspace lookup table for the given keyspace group. +func (kgm *KeyspaceGroupManager) updateKeyspaceGroupMembership( + oldGroup, newGroup *endpoint.KeyspaceGroup, updateWithLock bool, +) { + var ( + oldKeyspaces []uint32 + oldKeyspaceLookupTable map[uint32]struct{} + ) + + if oldGroup != nil { + oldKeyspaces = oldGroup.Keyspaces + oldKeyspaceLookupTable = oldGroup.KeyspaceLookupTable + } + + groupID := newGroup.ID + newKeyspaces := newGroup.Keyspaces + oldLen, newLen := len(oldKeyspaces), len(newKeyspaces) + + // Sort the keyspaces in ascending order + sort.Slice(newKeyspaces, func(i, j int) bool { + return newKeyspaces[i] < newKeyspaces[j] + }) + + // Mostly, the membership has no change, so optimize for this case. + sameMembership := true + if oldLen != newLen { + sameMembership = false + } else { + for i := 0; i < oldLen; i++ { + if oldKeyspaces[i] != newKeyspaces[i] { + sameMembership = false + break + } + } + } + + if updateWithLock { + kgm.Lock() + defer kgm.Unlock() + } + + if sameMembership { + // The keyspace group membership is not changed. Reuse the old one. + newGroup.KeyspaceLookupTable = oldKeyspaceLookupTable + } else { + // The keyspace list might be too long, so we only log the length, though there is a rare case that + // the old length and the new length are the same but the keyspace list is changed. + log.Info("the keyspace group's keyspace list is changed", + zap.Uint32("keyspace-group-id", groupID), + zap.Int("old-keyspaces-count", oldLen), + zap.Int("new-keyspaces-count", newLen)) + // The keyspace group membership is changed. Update the keyspace lookup table. + newGroup.KeyspaceLookupTable = make(map[uint32]struct{}) + for i, j := 0, 0; i < oldLen || j < newLen; { + if i < oldLen && j < newLen && oldKeyspaces[i] == newKeyspaces[j] { + newGroup.KeyspaceLookupTable[newKeyspaces[j]] = struct{}{} + i++ + j++ + } else if i < oldLen && j < newLen && oldKeyspaces[i] < newKeyspaces[j] || j == newLen { + // kgm.keyspaceLookupTable is a global lookup table for all keyspace groups, storing the + // keyspace group ID for each keyspace. If the keyspace group of this keyspace in this + // lookup table isn't the current keyspace group, it means the keyspace has been moved + // to another keyspace group which has already declared the ownership of the keyspace, + // and we shouldn't delete and overwrite the ownership. + if curGroupID, ok := kgm.keyspaceLookupTable[oldKeyspaces[i]]; ok && curGroupID == groupID { + delete(kgm.keyspaceLookupTable, oldKeyspaces[i]) + } + i++ + } else { + newGroup.KeyspaceLookupTable[newKeyspaces[j]] = struct{}{} + kgm.keyspaceLookupTable[newKeyspaces[j]] = groupID + j++ + } + } + if groupID == mcsutils.DefaultKeyspaceGroupID { + if _, ok := newGroup.KeyspaceLookupTable[mcsutils.DefaultKeyspaceID]; !ok { + log.Warn("default keyspace is not in default keyspace group. add it back") + kgm.keyspaceLookupTable[mcsutils.DefaultKeyspaceID] = groupID + newGroup.KeyspaceLookupTable[mcsutils.DefaultKeyspaceID] = struct{}{} + newGroup.Keyspaces = make([]uint32, 1+len(newKeyspaces)) + newGroup.Keyspaces[0] = mcsutils.DefaultKeyspaceID + copy(newGroup.Keyspaces[1:], newKeyspaces) + } + } else { + if _, ok := newGroup.KeyspaceLookupTable[mcsutils.DefaultKeyspaceID]; ok { + log.Warn("default keyspace is in non-default keyspace group. remove it") + kgm.keyspaceLookupTable[mcsutils.DefaultKeyspaceID] = mcsutils.DefaultKeyspaceGroupID + delete(newGroup.KeyspaceLookupTable, mcsutils.DefaultKeyspaceID) + newGroup.Keyspaces = newKeyspaces[1:] + } + } + } + // Check if the split is completed. + if oldGroup != nil && oldGroup.IsSplitTarget() && !newGroup.IsSplitting() { + kgm.ams[groupID].GetMember().(*member.Participant).SetCampaignChecker(nil) + delete(kgm.splittingGroups, groupID) + } + kgm.kgs[groupID] = newGroup +} + +// deleteKeyspaceGroup deletes the given keyspace group. +func (kgm *KeyspaceGroupManager) deleteKeyspaceGroup(groupID uint32) { + log.Info("delete keyspace group", zap.Uint32("keyspace-group-id", groupID)) + + if groupID == mcsutils.DefaultKeyspaceGroupID { + log.Info("removed default keyspace group meta config from the storage. " + + "now every tso node/pod will initialize it") + group := &endpoint.KeyspaceGroup{ + ID: mcsutils.DefaultKeyspaceGroupID, + Members: []endpoint.KeyspaceGroupMember{{ + Address: kgm.tsoServiceID.ServiceAddr, + Priority: mcsutils.DefaultKeyspaceGroupReplicaPriority, + }}, + Keyspaces: []uint32{mcsutils.DefaultKeyspaceID}, + } + kgm.updateKeyspaceGroup(group) + return + } + + kgm.Lock() + defer kgm.Unlock() + + kg := kgm.kgs[groupID] + if kg != nil { + for _, kid := range kg.Keyspaces { + // if kid == kg.ID, it means the keyspace still belongs to this keyspace group, + // so we decouple the relationship in the global keyspace lookup table. + // if kid != kg.ID, it means the keyspace has been moved to another keyspace group + // which has already declared the ownership of the keyspace, so we don't need + // delete it from the global keyspace lookup table and overwrite the ownership. + if kid == kg.ID { + delete(kgm.keyspaceLookupTable, kid) + } + } + kgm.kgs[groupID] = nil + } + + am := kgm.ams[groupID] + if am != nil { + am.close() + kgm.ams[groupID] = nil + } + + kgm.deletedGroups[groupID] = struct{}{} +} + +// exitElectionMembership exits the election membership of the given keyspace group by +// de-initializing the allocator manager, but still keeps the keyspace group info. +func (kgm *KeyspaceGroupManager) exitElectionMembership(group *endpoint.KeyspaceGroup) { + log.Info("resign election membership", zap.Uint32("keyspace-group-id", group.ID)) + + kgm.Lock() + defer kgm.Unlock() + + am := kgm.ams[group.ID] + if am != nil { + am.close() + kgm.ams[group.ID] = nil + } + + oldGroup := kgm.kgs[group.ID] + kgm.updateKeyspaceGroupMembership(oldGroup, group, false) +} + +// GetAllocatorManager returns the AllocatorManager of the given keyspace group +func (kgm *KeyspaceGroupManager) GetAllocatorManager(keyspaceGroupID uint32) (*AllocatorManager, error) { + if err := kgm.checkKeySpaceGroupID(keyspaceGroupID); err != nil { + return nil, err + } + if am, _ := kgm.getKeyspaceGroupMeta(keyspaceGroupID); am != nil { + return am, nil + } + return nil, genNotServedErr(errs.ErrGetAllocatorManager, keyspaceGroupID) +} + +// FindGroupByKeyspaceID returns the keyspace group that contains the keyspace with the given ID. +func (kgm *KeyspaceGroupManager) FindGroupByKeyspaceID( + keyspaceID uint32, +) (*AllocatorManager, *endpoint.KeyspaceGroup, uint32, error) { + curAM, curKeyspaceGroup, curKeyspaceGroupID, err := + kgm.getKeyspaceGroupMetaWithCheck(keyspaceID, mcsutils.DefaultKeyspaceGroupID) + if err != nil { + return nil, nil, curKeyspaceGroupID, err + } + return curAM, curKeyspaceGroup, curKeyspaceGroupID, nil +} + +// GetElectionMember returns the election member of the keyspace group serving the given keyspace. +func (kgm *KeyspaceGroupManager) GetElectionMember( + keyspaceID, keyspaceGroupID uint32, +) (ElectionMember, error) { + if err := kgm.checkKeySpaceGroupID(keyspaceGroupID); err != nil { + return nil, err + } + am, _, _, err := kgm.getKeyspaceGroupMetaWithCheck(keyspaceID, keyspaceGroupID) + if err != nil { + return nil, err + } + return am.GetMember(), nil +} + +// GetKeyspaceGroups returns all keyspace groups managed by the current keyspace group manager. +func (kgm *KeyspaceGroupManager) GetKeyspaceGroups() map[uint32]*endpoint.KeyspaceGroup { + kgm.RLock() + defer kgm.RUnlock() + keyspaceGroups := make(map[uint32]*endpoint.KeyspaceGroup) + for _, keyspaceGroupID := range kgm.keyspaceLookupTable { + if _, ok := keyspaceGroups[keyspaceGroupID]; ok { + continue + } + keyspaceGroups[keyspaceGroupID] = kgm.kgs[keyspaceGroupID] + } + return keyspaceGroups +} + +// HandleTSORequest forwards TSO allocation requests to correct TSO Allocators of the given keyspace group. +func (kgm *KeyspaceGroupManager) HandleTSORequest( + keyspaceID, keyspaceGroupID uint32, + dcLocation string, count uint32, +) (ts pdpb.Timestamp, curKeyspaceGroupID uint32, err error) { + if err := kgm.checkKeySpaceGroupID(keyspaceGroupID); err != nil { + return pdpb.Timestamp{}, keyspaceGroupID, err + } + am, _, curKeyspaceGroupID, err := kgm.getKeyspaceGroupMetaWithCheck(keyspaceID, keyspaceGroupID) + if err != nil { + return pdpb.Timestamp{}, curKeyspaceGroupID, err + } + err = kgm.checkTSOSplit(curKeyspaceGroupID, dcLocation) + if err != nil { + return pdpb.Timestamp{}, curKeyspaceGroupID, err + } + err = kgm.state.checkTSOMerge(curKeyspaceGroupID) + if err != nil { + return pdpb.Timestamp{}, curKeyspaceGroupID, err + } + ts, err = am.HandleRequest(dcLocation, count) + return ts, curKeyspaceGroupID, err +} + +func (kgm *KeyspaceGroupManager) checkKeySpaceGroupID(id uint32) error { + if id < mcsutils.MaxKeyspaceGroupCountInUse { + return nil + } + return errs.ErrKeyspaceGroupIDInvalid.FastGenByArgs( + fmt.Sprintf("%d shouldn't >= %d", id, mcsutils.MaxKeyspaceGroupCountInUse)) +} + +// GetMinTS returns the minimum timestamp across all keyspace groups served by this TSO server/pod. +func (kgm *KeyspaceGroupManager) GetMinTS( + dcLocation string, +) (_ pdpb.Timestamp, kgAskedCount, kgTotalCount uint32, err error) { + kgm.RLock() + defer kgm.RUnlock() + + var minTS *pdpb.Timestamp + for i, am := range kgm.ams { + if kgm.kgs[i] != nil { + kgTotalCount++ + } + // If any keyspace group hasn't elected primary, we can't know its current timestamp of + // the group, so as to the min ts across all keyspace groups. Return error in this case. + if am != nil && !am.member.IsLeaderElected() { + return pdpb.Timestamp{}, kgAskedCount, kgTotalCount, errs.ErrGetMinTS.FastGenByArgs("leader is not elected") + } + // Skip the keyspace groups that are not served by this TSO Server/Pod. + if am == nil || !am.IsLeader() { + continue + } + kgAskedCount++ + // Skip the keyspace groups that are split targets, because they always have newer + // time lines than the existing split sources thus won't contribute to the min ts. + if kgm.kgs[i] != nil && kgm.kgs[i].IsSplitTarget() { + continue + } + ts, err := am.HandleRequest(dcLocation, 1) + if err != nil { + return pdpb.Timestamp{}, kgAskedCount, kgTotalCount, err + } + if minTS == nil || tsoutil.CompareTimestamp(&ts, minTS) < 0 { + minTS = &ts + } + } + + if minTS == nil { + // This TSO server/pod is not serving any keyspace group, return an empty timestamp, + // and the client needs to skip the empty timestamps when collecting the min timestamp + // from all TSO servers/pods. + return pdpb.Timestamp{}, kgAskedCount, kgTotalCount, nil + } + + return *minTS, kgAskedCount, kgTotalCount, nil +} + +func genNotServedErr(perr *perrors.Error, keyspaceGroupID uint32) error { + return perr.FastGenByArgs( + fmt.Sprintf( + "requested keyspace group with id %d %s by this host/pod", + keyspaceGroupID, errs.NotServedErr)) +} + +// checkTSOSplit checks if the given keyspace group is in split state, and if so, it will make sure the +// newly split TSO keep consistent with the original one. +func (kgm *KeyspaceGroupManager) checkTSOSplit( + keyspaceGroupID uint32, + dcLocation string, +) error { + splitTargetAM, splitSourceAM, err := kgm.state.checkTSOSplit(keyspaceGroupID) + if err != nil || splitTargetAM == nil { + return err + } + splitTargetAllocator, err := splitTargetAM.GetAllocator(dcLocation) + if err != nil { + return err + } + splitSourceAllocator, err := splitSourceAM.GetAllocator(dcLocation) + if err != nil { + return err + } + splitTargetTSO, err := splitTargetAllocator.GenerateTSO(1) + if err != nil { + return err + } + splitSourceTSO, err := splitSourceAllocator.GenerateTSO(1) + if err != nil { + return err + } + // If the split source TSO is not greater than the newly split TSO, we don't need to do anything. + if tsoutil.CompareTimestamp(&splitSourceTSO, &splitTargetTSO) <= 0 { + log.Info("the split source tso is less than the newly split tso", + zap.Int64("split-source-tso-physical", splitSourceTSO.Physical), + zap.Int64("split-source-tso-logical", splitSourceTSO.Logical), + zap.Int64("split-tso-physical", splitTargetTSO.Physical), + zap.Int64("split-tso-logical", splitTargetTSO.Logical)) + // Finish the split state directly. + return kgm.finishSplitKeyspaceGroup(keyspaceGroupID) + } + // If the split source TSO is greater than the newly split TSO, we need to update the split + // TSO to make sure the following TSO will be greater than the split keyspaces ever had + // in the past. + err = splitTargetAllocator.SetTSO(tsoutil.GenerateTS(&pdpb.Timestamp{ + Physical: splitSourceTSO.Physical + 1, + Logical: splitSourceTSO.Logical, + }), true, true) + if err != nil { + return err + } + log.Info("the split source tso is greater than the newly split tso", + zap.Int64("split-source-tso-physical", splitSourceTSO.Physical), + zap.Int64("split-source-tso-logical", splitSourceTSO.Logical), + zap.Int64("split-tso-physical", splitTargetTSO.Physical), + zap.Int64("split-tso-logical", splitTargetTSO.Logical)) + // Finish the split state. + return kgm.finishSplitKeyspaceGroup(keyspaceGroupID) +} + +const keyspaceGroupsAPIPrefix = "/pd/api/v2/tso/keyspace-groups" + +// Put the code below into the critical section to prevent from sending too many HTTP requests. +func (kgm *KeyspaceGroupManager) finishSplitKeyspaceGroup(id uint32) error { + kgm.Lock() + defer kgm.Unlock() + // Check if the keyspace group is in split state. + splitGroup := kgm.kgs[id] + if !splitGroup.IsSplitTarget() { + return nil + } + // Check if the HTTP client is initialized. + if kgm.httpClient == nil { + return nil + } + statusCode, err := apiutil.DoDelete( + kgm.httpClient, + kgm.cfg.GeBackendEndpoints()+keyspaceGroupsAPIPrefix+fmt.Sprintf("/%d/split", id)) + if err != nil { + return err + } + if statusCode != http.StatusOK { + log.Warn("failed to finish split keyspace group", + zap.Uint32("keyspace-group-id", id), + zap.Int("status-code", statusCode)) + return errs.ErrSendRequest.FastGenByArgs() + } + // Pre-update the split keyspace group's split state in memory. + // Note: to avoid data race with state read APIs, we always replace the group in memory as a whole. + // For now, we only have scenarios to update split state/merge state, and the other fields are always + // loaded from etcd without any modification, so we can simply copy the group and replace the state. + newSplitGroup := *splitGroup + newSplitGroup.SplitState = nil + kgm.kgs[id] = &newSplitGroup + return nil +} + +func (kgm *KeyspaceGroupManager) finishMergeKeyspaceGroup(id uint32) error { + kgm.Lock() + defer kgm.Unlock() + // Check if the keyspace group is in the merging state. + mergeTarget := kgm.kgs[id] + if !mergeTarget.IsMergeTarget() { + return nil + } + // Check if the HTTP client is initialized. + if kgm.httpClient == nil { + return nil + } + statusCode, err := apiutil.DoDelete( + kgm.httpClient, + kgm.cfg.GeBackendEndpoints()+keyspaceGroupsAPIPrefix+fmt.Sprintf("/%d/merge", id)) + if err != nil { + return err + } + if statusCode != http.StatusOK { + log.Warn("failed to finish merging keyspace group", + zap.Uint32("keyspace-group-id", id), + zap.Int("status-code", statusCode)) + return errs.ErrSendRequest.FastGenByArgs() + } + + // Pre-update the merge target keyspace group's merge state in memory. + // Note: to avoid data race with state read APIs, we always replace the group in memory as a whole. + // For now, we only have scenarios to update split state/merge state, and the other fields are always + // loaded from etcd without any modification, so we can simply copy the group and replace the state. + newTargetGroup := *mergeTarget + newTargetGroup.MergeState = nil + kgm.kgs[id] = &newTargetGroup + return nil +} + +// mergingChecker is used to check if the keyspace group is in merge state, and if so, it will +// make sure the newly merged TSO keep consistent with the original ones. +func (kgm *KeyspaceGroupManager) mergingChecker(ctx context.Context, mergeTargetID uint32, mergeList []uint32) { + log.Info("start to merge the keyspace group", + zap.String("member", kgm.tsoServiceID.ServiceAddr), + zap.Uint32("merge-target-id", mergeTargetID), + zap.Any("merge-list", mergeList)) + defer logutil.LogPanic() + defer kgm.wg.Done() + + checkTicker := time.NewTicker(mergingCheckInterval) + defer checkTicker.Stop() + // Prepare the merge map. + mergeMap := make(map[uint32]struct{}, len(mergeList)) + for _, id := range mergeList { + mergeMap[id] = struct{}{} + } + + for { + select { + case <-ctx.Done(): + log.Info("merging checker is closed", + zap.String("member", kgm.tsoServiceID.ServiceAddr), + zap.Uint32("merge-target-id", mergeTargetID), + zap.Any("merge-list", mergeList)) + return + case <-checkTicker.C: + } + // Check if current TSO node is the merge target TSO primary node. + am, err := kgm.GetAllocatorManager(mergeTargetID) + if err != nil { + log.Warn("unable to get the merge target allocator manager", + zap.String("member", kgm.tsoServiceID.ServiceAddr), + zap.Uint32("keyspace-group-id", mergeTargetID), + zap.Any("merge-list", mergeList), + zap.Error(err)) + continue + } + // If the current TSO node is not the merge target TSO primary node, + // we still need to keep this loop running to avoid unexpected primary changes. + if !am.IsLeader() { + log.Debug("current tso node is not the merge target primary", + zap.String("member", kgm.tsoServiceID.ServiceAddr), + zap.Uint32("merge-target-id", mergeTargetID), + zap.Any("merge-list", mergeList)) + continue + } + // Check if the keyspace group primaries in the merge map are all gone. + if len(mergeMap) != 0 { + for id := range mergeMap { + leaderPath := endpoint.KeyspaceGroupPrimaryPath(kgm.tsoSvcRootPath, id) + val, err := kgm.tsoSvcStorage.Load(leaderPath) + if err != nil { + log.Error("failed to check if the keyspace group primary in the merge list has gone", + zap.String("member", kgm.tsoServiceID.ServiceAddr), + zap.Uint32("merge-target-id", mergeTargetID), + zap.Any("merge-list", mergeList), + zap.Uint32("merge-id", id), + zap.Any("remaining", mergeMap), + zap.Error(err)) + continue + } + if len(val) == 0 { + delete(mergeMap, id) + } + } + } + if len(mergeMap) > 0 { + continue + } + log.Info("all the keyspace group primaries in the merge list are gone, "+ + "start to calculate the newly merged TSO", + zap.String("member", kgm.tsoServiceID.ServiceAddr), + zap.Uint32("merge-target-id", mergeTargetID), + zap.Any("merge-list", mergeList)) + // All the keyspace group primaries in the merge list are gone, + // calculate the newly merged TSO to make sure it is greater than the original ones. + var mergedTS time.Time + for _, id := range mergeList { + ts, err := kgm.tsoSvcStorage.LoadTimestamp(endpoint.KeyspaceGroupTSPath(id)) + if err != nil { + log.Error("failed to load the keyspace group TSO", + zap.String("member", kgm.tsoServiceID.ServiceAddr), + zap.Uint32("merge-target-id", mergeTargetID), + zap.Any("merge-list", mergeList), + zap.Uint32("merge-id", id), + zap.Time("ts", ts), + zap.Error(err)) + break + } + if ts.After(mergedTS) { + mergedTS = ts + } + } + if err != nil { + continue + } + // Update the newly merged TSO if the merged TSO is not zero. + if mergedTS != typeutil.ZeroTime { + log.Info("start to set the newly merged TSO", + zap.String("member", kgm.tsoServiceID.ServiceAddr), + zap.Uint32("merge-target-id", mergeTargetID), + zap.Any("merge-list", mergeList), + zap.Time("merged-ts", mergedTS)) + // TODO: support the Local TSO Allocator. + allocator, err := am.GetAllocator(GlobalDCLocation) + if err != nil { + log.Error("failed to get the allocator", + zap.String("member", kgm.tsoServiceID.ServiceAddr), + zap.Uint32("merge-target-id", mergeTargetID), + zap.Any("merge-list", mergeList), + zap.Error(err)) + continue + } + err = allocator.SetTSO( + tsoutil.GenerateTS(tsoutil.GenerateTimestamp(mergedTS, 1)), + true, true) + if err != nil { + log.Error("failed to update the newly merged TSO", + zap.String("member", kgm.tsoServiceID.ServiceAddr), + zap.Uint32("merge-target-id", mergeTargetID), + zap.Any("merge-list", mergeList), + zap.Time("merged-ts", mergedTS), + zap.Error(err)) + continue + } + } + // Finish the merge. + err = kgm.finishMergeKeyspaceGroup(mergeTargetID) + if err != nil { + log.Error("failed to finish the merge", + zap.String("member", kgm.tsoServiceID.ServiceAddr), + zap.Uint32("merge-target-id", mergeTargetID), + zap.Any("merge-list", mergeList), + zap.Error(err)) + continue + } + log.Info("finished merging keyspace group", + zap.String("member", kgm.tsoServiceID.ServiceAddr), + zap.Uint32("merge-target-id", mergeTargetID), + zap.Any("merge-list", mergeList), + zap.Time("merged-ts", mergedTS)) + return + } +} + +// groupSplitPatroller is used to patrol the groups that are in the on-going +// split state and to check if we could speed up the split process. +func (kgm *KeyspaceGroupManager) groupSplitPatroller() { + defer logutil.LogPanic() + defer kgm.wg.Done() + patrolInterval := groupPatrolInterval + failpoint.Inject("fastGroupSplitPatroller", func() { + patrolInterval = 200 * time.Millisecond + }) + ticker := time.NewTicker(patrolInterval) + defer ticker.Stop() + log.Info("group split patroller is started", + zap.Duration("patrol-interval", patrolInterval)) + for { + select { + case <-kgm.ctx.Done(): + log.Info("group split patroller exited") + return + case <-ticker.C: + } + for _, groupID := range kgm.getSplittingGroups() { + am, group := kgm.getKeyspaceGroupMeta(groupID) + if !am.IsLeader() { + continue + } + if len(group.Keyspaces) == 0 { + log.Warn("abnormal keyspace group with empty keyspace list", + zap.Uint32("keyspace-group-id", groupID)) + continue + } + log.Info("request tso for the splitting keyspace group", + zap.Uint32("keyspace-group-id", groupID), + zap.Uint32("keyspace-id", group.Keyspaces[0])) + // Request the TSO manually to speed up the split process. + _, _, err := kgm.HandleTSORequest(group.Keyspaces[0], groupID, GlobalDCLocation, 1) + if err != nil { + log.Warn("failed to request tso for the splitting keyspace group", + zap.Uint32("keyspace-group-id", groupID), + zap.Uint32("keyspace-id", group.Keyspaces[0]), + zap.Error(err)) + continue + } + } + } +} + +// deletedGroupCleaner is used to clean the deleted keyspace groups related data. +// For example, the TSO keys of the merged keyspace groups remain in the storage. +func (kgm *KeyspaceGroupManager) deletedGroupCleaner() { + defer logutil.LogPanic() + defer kgm.wg.Done() + patrolInterval := groupPatrolInterval + failpoint.Inject("fastDeletedGroupCleaner", func() { + patrolInterval = 200 * time.Millisecond + }) + ticker := time.NewTicker(patrolInterval) + defer ticker.Stop() + log.Info("deleted group cleaner is started", + zap.Duration("patrol-interval", patrolInterval)) + for { + select { + case <-kgm.ctx.Done(): + log.Info("deleted group cleaner exited") + return + case <-ticker.C: + } + for _, groupID := range kgm.getDeletedGroups() { + // Do not clean the default keyspace group data. + if groupID == mcsutils.DefaultKeyspaceGroupID { + continue + } + // Make sure the allocator and group meta are not in use anymore. + am, _ := kgm.getKeyspaceGroupMeta(groupID) + if am != nil { + log.Info("the keyspace group tso allocator has not been closed yet", + zap.Uint32("keyspace-group-id", groupID)) + continue + } + log.Info("delete the keyspace group tso key", + zap.Uint32("keyspace-group-id", groupID)) + // Clean up the remaining TSO keys. + // TODO: support the Local TSO Allocator clean up. + err := kgm.tsoSvcStorage.DeleteTimestamp( + endpoint.TimestampPath( + endpoint.KeyspaceGroupTSPath(groupID), + ), + ) + if err != nil { + log.Warn("failed to delete the keyspace group tso key", + zap.Uint32("keyspace-group-id", groupID), + zap.Error(err)) + continue + } + kgm.Lock() + delete(kgm.deletedGroups, groupID) + kgm.Unlock() + } + } +} diff --git a/pkg/tso/keyspace_group_manager_test.go b/pkg/tso/keyspace_group_manager_test.go new file mode 100644 index 00000000000..2ffa802b4f3 --- /dev/null +++ b/pkg/tso/keyspace_group_manager_test.go @@ -0,0 +1,1236 @@ +// Copyright 2023 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tso + +import ( + "context" + "encoding/json" + "fmt" + "math/rand" + "path" + "reflect" + "sort" + "strings" + "sync" + "testing" + "time" + + "github.com/google/uuid" + "github.com/pingcap/failpoint" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + "github.com/tikv/pd/pkg/errs" + "github.com/tikv/pd/pkg/mcs/discovery" + mcsutils "github.com/tikv/pd/pkg/mcs/utils" + "github.com/tikv/pd/pkg/storage/endpoint" + "github.com/tikv/pd/pkg/utils/tempurl" + "github.com/tikv/pd/pkg/utils/testutil" + "github.com/tikv/pd/pkg/utils/tsoutil" + "github.com/tikv/pd/pkg/utils/typeutil" + "go.etcd.io/etcd/clientv3" + "go.etcd.io/etcd/mvcc/mvccpb" + "go.uber.org/goleak" +) + +func TestMain(m *testing.M) { + goleak.VerifyTestMain(m, testutil.LeakOptions...) +} + +type keyspaceGroupManagerTestSuite struct { + suite.Suite + ctx context.Context + cancel context.CancelFunc + ClusterID uint64 + backendEndpoints string + etcdClient *clientv3.Client + clean func() + cfg *TestServiceConfig +} + +func TestKeyspaceGroupManagerTestSuite(t *testing.T) { + suite.Run(t, new(keyspaceGroupManagerTestSuite)) +} + +func (suite *keyspaceGroupManagerTestSuite) SetupSuite() { + t := suite.T() + suite.ctx, suite.cancel = context.WithCancel(context.Background()) + suite.ClusterID = rand.Uint64() + suite.backendEndpoints, suite.etcdClient, suite.clean = startEmbeddedEtcd(t) + suite.cfg = suite.createConfig() +} + +func (suite *keyspaceGroupManagerTestSuite) TearDownSuite() { + suite.clean() + suite.cancel() +} + +func (suite *keyspaceGroupManagerTestSuite) createConfig() *TestServiceConfig { + addr := tempurl.Alloc() + return &TestServiceConfig{ + Name: "tso-test-name-default", + BackendEndpoints: suite.backendEndpoints, + ListenAddr: addr, + AdvertiseListenAddr: addr, + LeaderLease: mcsutils.DefaultLeaderLease, + LocalTSOEnabled: false, + TSOUpdatePhysicalInterval: 50 * time.Millisecond, + TSOSaveInterval: time.Duration(mcsutils.DefaultLeaderLease) * time.Second, + MaxResetTSGap: time.Hour * 24, + TLSConfig: nil, + } +} + +func (suite *keyspaceGroupManagerTestSuite) TestDeletedGroupCleanup() { + re := suite.Require() + re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/tso/fastDeletedGroupCleaner", "return(true)")) + + // Start with the empty keyspace group assignment. + mgr := suite.newUniqueKeyspaceGroupManager(0) + re.NotNil(mgr) + defer mgr.Close() + err := mgr.Initialize() + re.NoError(err) + + rootPath := mgr.legacySvcRootPath + svcAddr := mgr.tsoServiceID.ServiceAddr + + // Add keyspace group 1. + suite.applyEtcdEvents(re, rootPath, []*etcdEvent{generateKeyspaceGroupPutEvent(1, []uint32{1}, []string{svcAddr})}) + // Check if the TSO key is created. + testutil.Eventually(re, func() bool { + ts, err := mgr.tsoSvcStorage.LoadTimestamp(endpoint.KeyspaceGroupTSPath(1)) + re.NoError(err) + return ts != typeutil.ZeroTime + }) + // Delete keyspace group 1. + suite.applyEtcdEvents(re, rootPath, []*etcdEvent{generateKeyspaceGroupDeleteEvent(1)}) + // Check if the TSO key is deleted. + testutil.Eventually(re, func() bool { + ts, err := mgr.tsoSvcStorage.LoadTimestamp(endpoint.KeyspaceGroupTSPath(1)) + re.NoError(err) + return ts == typeutil.ZeroTime + }) + // Check if the keyspace group is deleted completely. + mgr.RLock() + re.Nil(mgr.ams[1]) + re.Nil(mgr.kgs[1]) + re.NotContains(mgr.deletedGroups, 1) + mgr.RUnlock() + // Try to delete the default keyspace group. + suite.applyEtcdEvents(re, rootPath, []*etcdEvent{generateKeyspaceGroupDeleteEvent(mcsutils.DefaultKeyspaceGroupID)}) + // Default keyspace group should NOT be deleted. + mgr.RLock() + re.NotNil(mgr.ams[mcsutils.DefaultKeyspaceGroupID]) + re.NotNil(mgr.kgs[mcsutils.DefaultKeyspaceGroupID]) + re.NotContains(mgr.deletedGroups, mcsutils.DefaultKeyspaceGroupID) + mgr.RUnlock() + // Default keyspace group TSO key should NOT be deleted. + ts, err := mgr.legacySvcStorage.LoadTimestamp(endpoint.KeyspaceGroupTSPath(mcsutils.DefaultKeyspaceGroupID)) + re.NoError(err) + re.NotEmpty(ts) + + re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/tso/fastDeletedGroupCleaner")) +} + +// TestNewKeyspaceGroupManager tests the initialization of KeyspaceGroupManager. +// It should initialize the allocator manager with the desired configurations and parameters. +func (suite *keyspaceGroupManagerTestSuite) TestNewKeyspaceGroupManager() { + re := suite.Require() + + tsoServiceID := &discovery.ServiceRegistryEntry{ServiceAddr: suite.cfg.AdvertiseListenAddr} + guid := uuid.New().String() + tsoServiceKey := discovery.ServicePath(guid, "tso") + legacySvcRootPath := path.Join("/pd", guid) + tsoSvcRootPath := path.Join(mcsutils.MicroserviceRootPath, guid, "tso") + electionNamePrefix := "tso-server-" + guid + + kgm := NewKeyspaceGroupManager( + suite.ctx, tsoServiceID, suite.etcdClient, nil, electionNamePrefix, + tsoServiceKey, legacySvcRootPath, tsoSvcRootPath, suite.cfg) + defer kgm.Close() + err := kgm.Initialize() + re.NoError(err) + + re.Equal(tsoServiceID, kgm.tsoServiceID) + re.Equal(suite.etcdClient, kgm.etcdClient) + re.Equal(electionNamePrefix, kgm.electionNamePrefix) + re.Equal(legacySvcRootPath, kgm.legacySvcRootPath) + re.Equal(tsoSvcRootPath, kgm.tsoSvcRootPath) + re.Equal(suite.cfg, kgm.cfg) + + am, err := kgm.GetAllocatorManager(mcsutils.DefaultKeyspaceGroupID) + re.NoError(err) + re.False(am.enableLocalTSO) + re.Equal(mcsutils.DefaultKeyspaceGroupID, am.kgID) + re.Equal(mcsutils.DefaultLeaderLease, am.leaderLease) + re.Equal(time.Hour*24, am.maxResetTSGap()) + re.Equal(legacySvcRootPath, am.rootPath) + re.Equal(time.Duration(mcsutils.DefaultLeaderLease)*time.Second, am.saveInterval) + re.Equal(time.Duration(50)*time.Millisecond, am.updatePhysicalInterval) +} + +// TestLoadKeyspaceGroupsAssignment tests the loading of the keyspace group assignment. +func (suite *keyspaceGroupManagerTestSuite) TestLoadKeyspaceGroupsAssignment() { + re := suite.Require() + maxCountInUse := 512 + // Test loading of empty keyspace group assignment. + suite.runTestLoadKeyspaceGroupsAssignment(re, 0, 0, 100) + // Test loading of single keyspace group assignment. + suite.runTestLoadKeyspaceGroupsAssignment(re, 1, 0, 100) + // Test loading of multiple keyspace group assignment. + suite.runTestLoadKeyspaceGroupsAssignment(re, 3, 0, 100) + suite.runTestLoadKeyspaceGroupsAssignment(re, maxCountInUse-1, 0, 10) + suite.runTestLoadKeyspaceGroupsAssignment(re, maxCountInUse, 0, 10) + // Test loading of the keyspace group assignment which exceeds the maximum keyspace group count. + // In this case, the manager should only load/serve the first MaxKeyspaceGroupCountInUse keyspace + // groups and ignore the rest. + suite.runTestLoadKeyspaceGroupsAssignment(re, maxCountInUse+1, 0, 10) +} + +// TestLoadWithDifferentBatchSize tests the loading of the keyspace group assignment with the different batch size. +func (suite *keyspaceGroupManagerTestSuite) TestLoadWithDifferentBatchSize() { + re := suite.Require() + + batchSize := int64(17) + maxCount := uint32(1024) + params := []struct { + batchSize int64 + count int + probabilityAssignToMe int // percentage of assigning keyspace groups to this host/pod + }{ + {batchSize: 1, count: 1, probabilityAssignToMe: 100}, + {batchSize: 2, count: int(maxCount / 10), probabilityAssignToMe: 100}, + {batchSize: 7, count: int(maxCount / 10), probabilityAssignToMe: 100}, + {batchSize: batchSize, count: int(batchSize), probabilityAssignToMe: 50}, + {batchSize: int64(maxCount / 13), count: int(maxCount / 13), probabilityAssignToMe: 50}, + {batchSize: int64(maxCount), count: int(maxCount / 13), probabilityAssignToMe: 10}, + } + + for _, param := range params { + suite.runTestLoadKeyspaceGroupsAssignment(re, param.count-1, param.batchSize, param.probabilityAssignToMe) + suite.runTestLoadKeyspaceGroupsAssignment(re, param.count, param.batchSize, param.probabilityAssignToMe) + suite.runTestLoadKeyspaceGroupsAssignment(re, param.count+1, param.batchSize, param.probabilityAssignToMe) + } +} + +// TestLoadKeyspaceGroupsTimeout tests there is timeout when loading the initial keyspace group assignment +// from etcd. The initialization of the keyspace group manager should fail. +func (suite *keyspaceGroupManagerTestSuite) TestLoadKeyspaceGroupsTimeout() { + re := suite.Require() + + mgr := suite.newUniqueKeyspaceGroupManager(1) + re.NotNil(mgr) + defer mgr.Close() + + addKeyspaceGroupAssignment( + suite.ctx, suite.etcdClient, uint32(0), mgr.legacySvcRootPath, + []string{mgr.tsoServiceID.ServiceAddr}, []int{0}, []uint32{0}) + + // Set the timeout to 1 second and inject the delayLoad to return 3 seconds to let + // the loading sleep 3 seconds. + mgr.loadKeyspaceGroupsTimeout = time.Second + re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/utils/etcdutil/delayLoad", "return(3)")) + err := mgr.Initialize() + // If loading keyspace groups timeout, the initialization should fail with ErrLoadKeyspaceGroupsTerminated. + re.Contains(err.Error(), errs.ErrLoadKeyspaceGroupsTerminated.Error()) + re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/utils/etcdutil/delayLoad")) +} + +// TestLoadKeyspaceGroupsSucceedWithTempFailures tests the initialization should succeed when there are temporary +// failures during loading the initial keyspace group assignment from etcd. +func (suite *keyspaceGroupManagerTestSuite) TestLoadKeyspaceGroupsSucceedWithTempFailures() { + re := suite.Require() + + mgr := suite.newUniqueKeyspaceGroupManager(1) + re.NotNil(mgr) + defer mgr.Close() + + addKeyspaceGroupAssignment( + suite.ctx, suite.etcdClient, uint32(0), mgr.legacySvcRootPath, + []string{mgr.tsoServiceID.ServiceAddr}, []int{0}, []uint32{0}) + + // Set the max retry times to 3 and inject the loadTemporaryFail to return 2 to let + // loading from etcd fail 2 times but the whole initialization still succeeds. + mgr.loadFromEtcdMaxRetryTimes = 3 + re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/utils/etcdutil/loadTemporaryFail", "return(2)")) + err := mgr.Initialize() + re.NoError(err) + re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/utils/etcdutil/loadTemporaryFail")) +} + +// TestLoadKeyspaceGroupsFailed tests the initialization should fail when there are too many failures +// during loading the initial keyspace group assignment from etcd. +func (suite *keyspaceGroupManagerTestSuite) TestLoadKeyspaceGroupsFailed() { + re := suite.Require() + + mgr := suite.newUniqueKeyspaceGroupManager(1) + re.NotNil(mgr) + defer mgr.Close() + + addKeyspaceGroupAssignment( + suite.ctx, suite.etcdClient, uint32(0), mgr.legacySvcRootPath, + []string{mgr.tsoServiceID.ServiceAddr}, []int{0}, []uint32{0}) + + // Set the max retry times to 3 and inject the loadTemporaryFail to return 3 to let + // loading from etcd fail 3 times which should cause the whole initialization to fail. + mgr.loadFromEtcdMaxRetryTimes = 3 + re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/utils/etcdutil/loadTemporaryFail", "return(3)")) + err := mgr.Initialize() + re.Error(err) + re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/utils/etcdutil/loadTemporaryFail")) +} + +// TestWatchAndDynamicallyApplyChanges tests the keyspace group manager watch and dynamically apply +// keyspace groups' membership/distribution meta changes. +func (suite *keyspaceGroupManagerTestSuite) TestWatchAndDynamicallyApplyChanges() { + re := suite.Require() + + // Start with the empty keyspace group assignment. + mgr := suite.newUniqueKeyspaceGroupManager(0) + re.NotNil(mgr) + defer mgr.Close() + err := mgr.Initialize() + re.NoError(err) + + rootPath := mgr.legacySvcRootPath + svcAddr := mgr.tsoServiceID.ServiceAddr + + // Initialize PUT/DELETE events + events := []*etcdEvent{} + // Assign keyspace group 0 to this host/pod/keyspace-group-manager. + // final result: assigned [0], loaded [0] + events = append(events, generateKeyspaceGroupPutEvent(0, []uint32{0}, []string{svcAddr})) + // Assign keyspace group 1 to this host/pod/keyspace-group-manager. + // final result: assigned [0,1], loaded [0,1] + events = append(events, generateKeyspaceGroupPutEvent(1, []uint32{1}, []string{"unknown", svcAddr})) + // Assign keyspace group 2 to other host/pod/keyspace-group-manager. + // final result: assigned [0,1], loaded [0,1,2] + events = append(events, generateKeyspaceGroupPutEvent(2, []uint32{2}, []string{"unknown"})) + // Assign keyspace group 3 to this host/pod/keyspace-group-manager. + // final result: assigned [0,1,3], loaded [0,1,2,3] + events = append(events, generateKeyspaceGroupPutEvent(3, []uint32{3}, []string{svcAddr})) + // Delete keyspace group 0. Every tso node/pod now should initialize keyspace group 0. + // final result: assigned [0,1,3], loaded [0,1,2,3] + events = append(events, generateKeyspaceGroupDeleteEvent(0)) + // Put keyspace group 4 which doesn't belong to anyone. + // final result: assigned [0,1,3], loaded [0,1,2,3,4] + events = append(events, generateKeyspaceGroupPutEvent(4, []uint32{4}, []string{})) + // Put keyspace group 5 which doesn't belong to anyone. + // final result: assigned [0,1,3], loaded [0,1,2,3,4,5] + events = append(events, generateKeyspaceGroupPutEvent(5, []uint32{5}, []string{})) + // Assign keyspace group 2 to this host/pod/keyspace-group-manager. + // final result: assigned [0,1,2,3], loaded [0,1,2,3,4,5] + events = append(events, generateKeyspaceGroupPutEvent(2, []uint32{2}, []string{svcAddr})) + // Reassign keyspace group 3 to no one. + // final result: assigned [0,1,2], loaded [0,1,2,3,4,5] + events = append(events, generateKeyspaceGroupPutEvent(3, []uint32{3}, []string{})) + // Reassign keyspace group 4 to this host/pod/keyspace-group-manager. + // final result: assigned [0,1,2,4], loaded [0,1,2,3,4,5] + events = append(events, generateKeyspaceGroupPutEvent(4, []uint32{4}, []string{svcAddr})) + // Delete keyspace group 2. + // final result: assigned [0,1,4], loaded [0,1,3,4,5] + events = append(events, generateKeyspaceGroupDeleteEvent(2)) + + // Apply the keyspace group assignment change events to etcd. + suite.applyEtcdEvents(re, rootPath, events) + + // Verify the keyspace groups assigned. + // Eventually, this keyspace groups manager is expected to serve the following keyspace groups. + expectedAssignedGroups := []uint32{0, 1, 4} + testutil.Eventually(re, func() bool { + assignedGroups := collectAssignedKeyspaceGroupIDs(re, mgr) + return reflect.DeepEqual(expectedAssignedGroups, assignedGroups) + }) + + // Verify the keyspace groups loaded. + // Eventually, this keyspace groups manager is expected to load the following keyspace groups + // in which keyspace group 3, 5 aren't served by this tso node/pod. + expectedLoadedGroups := []uint32{0, 1, 3, 4, 5} + testutil.Eventually(re, func() bool { + loadedGroups := collectAllLoadedKeyspaceGroupIDs(mgr) + return reflect.DeepEqual(expectedLoadedGroups, loadedGroups) + }) +} + +// TestDefaultKeyspaceGroup tests the initialization logic of the default keyspace group. +// If the default keyspace group isn't configured in the etcd, every tso node/pod should initialize +// it and join the election for the primary of this group. +// If the default keyspace group is configured in the etcd, the tso nodes/pods which are assigned with +// this group will initialize it and join the election for the primary of this group. +func (suite *keyspaceGroupManagerTestSuite) TestInitDefaultKeyspaceGroup() { + re := suite.Require() + + var ( + expectedGroupIDs []uint32 + event *etcdEvent + ) + + // Start with the empty keyspace group assignment. + mgr := suite.newUniqueKeyspaceGroupManager(0) + defer mgr.Close() + err := mgr.Initialize() + re.NoError(err) + + rootPath := mgr.legacySvcRootPath + svcAddr := mgr.tsoServiceID.ServiceAddr + + expectedGroupIDs = []uint32{0} + assignedGroupIDs := collectAssignedKeyspaceGroupIDs(re, mgr) + re.Equal(expectedGroupIDs, assignedGroupIDs) + + // Config keyspace group 0 in the storage but assigned to no one. + // final result: [] + expectedGroupIDs = []uint32{} + event = generateKeyspaceGroupPutEvent(0, []uint32{0}, []string{"unknown"}) + err = putKeyspaceGroupToEtcd(suite.ctx, suite.etcdClient, rootPath, event.ksg) + re.NoError(err) + testutil.Eventually(re, func() bool { + assignedGroupIDs := collectAssignedKeyspaceGroupIDs(re, mgr) + return reflect.DeepEqual(expectedGroupIDs, assignedGroupIDs) + }) + // Config keyspace group 0 in the storage and assigned to this host/pod/keyspace-group-manager. + // final result: [0] + expectedGroupIDs = []uint32{0} + event = generateKeyspaceGroupPutEvent(0, []uint32{0}, []string{svcAddr}) + err = putKeyspaceGroupToEtcd(suite.ctx, suite.etcdClient, rootPath, event.ksg) + re.NoError(err) + testutil.Eventually(re, func() bool { + assignedGroupIDs := collectAssignedKeyspaceGroupIDs(re, mgr) + return reflect.DeepEqual(expectedGroupIDs, assignedGroupIDs) + }) + // Delete keyspace group 0. Every tso node/pod now should initialize keyspace group 0. + // final result: [0] + expectedGroupIDs = []uint32{0} + event = generateKeyspaceGroupDeleteEvent(0) + err = deleteKeyspaceGroupInEtcd(suite.ctx, suite.etcdClient, rootPath, event.ksg.ID) + re.NoError(err) + testutil.Eventually(re, func() bool { + assignedGroupIDs := collectAssignedKeyspaceGroupIDs(re, mgr) + return reflect.DeepEqual(expectedGroupIDs, assignedGroupIDs) + }) + // Config keyspace group 0 in the storage and assigned to this host/pod/keyspace-group-manager. + // final result: [0] + expectedGroupIDs = []uint32{0} + event = generateKeyspaceGroupPutEvent(0, []uint32{0}, []string{svcAddr}) + err = putKeyspaceGroupToEtcd(suite.ctx, suite.etcdClient, rootPath, event.ksg) + re.NoError(err) + testutil.Eventually(re, func() bool { + assignedGroupIDs := collectAssignedKeyspaceGroupIDs(re, mgr) + return reflect.DeepEqual(expectedGroupIDs, assignedGroupIDs) + }) +} + +// TestGetKeyspaceGroupMetaWithCheck tests GetKeyspaceGroupMetaWithCheck. +func (suite *keyspaceGroupManagerTestSuite) TestGetKeyspaceGroupMetaWithCheck() { + re := suite.Require() + + mgr := suite.newUniqueKeyspaceGroupManager(1) + re.NotNil(mgr) + defer mgr.Close() + + var ( + am *AllocatorManager + kg *endpoint.KeyspaceGroup + kgid uint32 + err error + ) + + // Create keyspace group 0 which contains keyspace 0, 1, 2. + addKeyspaceGroupAssignment( + suite.ctx, suite.etcdClient, uint32(0), mgr.legacySvcRootPath, + []string{mgr.tsoServiceID.ServiceAddr}, []int{0}, []uint32{0, 1, 2}) + + err = mgr.Initialize() + re.NoError(err) + + // Should be able to get AM for the default/null keyspace and keyspace 1, 2 in keyspace group 0. + am, kg, kgid, err = mgr.getKeyspaceGroupMetaWithCheck(mcsutils.DefaultKeyspaceID, 0) + re.NoError(err) + re.Equal(mcsutils.DefaultKeyspaceGroupID, kgid) + re.NotNil(am) + re.NotNil(kg) + am, kg, kgid, err = mgr.getKeyspaceGroupMetaWithCheck(mcsutils.NullKeyspaceID, 0) + re.NoError(err) + re.Equal(mcsutils.DefaultKeyspaceGroupID, kgid) + re.NotNil(am) + re.NotNil(kg) + am, kg, kgid, err = mgr.getKeyspaceGroupMetaWithCheck(1, 0) + re.NoError(err) + re.Equal(mcsutils.DefaultKeyspaceGroupID, kgid) + re.NotNil(am) + re.NotNil(kg) + am, kg, kgid, err = mgr.getKeyspaceGroupMetaWithCheck(2, 0) + re.NoError(err) + re.Equal(mcsutils.DefaultKeyspaceGroupID, kgid) + re.NotNil(am) + re.NotNil(kg) + // Should still succeed even keyspace 3 isn't explicitly assigned to any + // keyspace group. It will be assigned to the default keyspace group. + am, kg, kgid, err = mgr.getKeyspaceGroupMetaWithCheck(3, 0) + re.NoError(err) + re.Equal(mcsutils.DefaultKeyspaceGroupID, kgid) + re.NotNil(am) + re.NotNil(kg) + // Should succeed and get the meta of keyspace group 0, because keyspace 0 + // belongs to group 0, though the specified group 1 doesn't exist. + am, kg, kgid, err = mgr.getKeyspaceGroupMetaWithCheck(mcsutils.DefaultKeyspaceID, 1) + re.NoError(err) + re.Equal(mcsutils.DefaultKeyspaceGroupID, kgid) + re.NotNil(am) + re.NotNil(kg) + // Should fail because keyspace 3 isn't explicitly assigned to any keyspace + // group, and the specified group isn't the default keyspace group. + am, kg, kgid, err = mgr.getKeyspaceGroupMetaWithCheck(3, 100) + re.Error(err) + re.Equal(uint32(100), kgid) + re.Nil(am) + re.Nil(kg) +} + +// TestDefaultMembershipRestriction tests the restriction of default keyspace always +// belongs to default keyspace group. +func (suite *keyspaceGroupManagerTestSuite) TestDefaultMembershipRestriction() { + re := suite.Require() + + mgr := suite.newUniqueKeyspaceGroupManager(1) + re.NotNil(mgr) + defer mgr.Close() + + rootPath := mgr.legacySvcRootPath + svcAddr := mgr.tsoServiceID.ServiceAddr + + var ( + am *AllocatorManager + kg *endpoint.KeyspaceGroup + kgid uint32 + err error + event *etcdEvent + ) + + // Create keyspace group 0 which contains keyspace 0, 1, 2. + addKeyspaceGroupAssignment( + suite.ctx, suite.etcdClient, mcsutils.DefaultKeyspaceGroupID, rootPath, + []string{svcAddr}, []int{0}, []uint32{mcsutils.DefaultKeyspaceID, 1, 2}) + // Create keyspace group 3 which contains keyspace 3, 4. + addKeyspaceGroupAssignment( + suite.ctx, suite.etcdClient, uint32(3), mgr.legacySvcRootPath, + []string{mgr.tsoServiceID.ServiceAddr}, []int{0}, []uint32{3, 4}) + + err = mgr.Initialize() + re.NoError(err) + + // Should be able to get AM for keyspace 0 in keyspace group 0. + am, kg, kgid, err = mgr.getKeyspaceGroupMetaWithCheck( + mcsutils.DefaultKeyspaceID, mcsutils.DefaultKeyspaceGroupID) + re.NoError(err) + re.Equal(mcsutils.DefaultKeyspaceGroupID, kgid) + re.NotNil(am) + re.NotNil(kg) + + event = generateKeyspaceGroupPutEvent( + mcsutils.DefaultKeyspaceGroupID, []uint32{1, 2}, []string{svcAddr}) + err = putKeyspaceGroupToEtcd(suite.ctx, suite.etcdClient, rootPath, event.ksg) + re.NoError(err) + event = generateKeyspaceGroupPutEvent( + 3, []uint32{mcsutils.DefaultKeyspaceID, 3, 4}, []string{svcAddr}) + err = putKeyspaceGroupToEtcd(suite.ctx, suite.etcdClient, rootPath, event.ksg) + re.NoError(err) + + // Sleep for a while to wait for the events to propagate. If the logic doesn't work + // as expected, it will cause random failure. + time.Sleep(1 * time.Second) + // Should still be able to get AM for keyspace 0 in keyspace group 0. + am, kg, kgid, err = mgr.getKeyspaceGroupMetaWithCheck( + mcsutils.DefaultKeyspaceID, mcsutils.DefaultKeyspaceGroupID) + re.NoError(err) + re.Equal(mcsutils.DefaultKeyspaceGroupID, kgid) + re.NotNil(am) + re.NotNil(kg) + // Should succeed and return the keyspace group meta from the default keyspace group + am, kg, kgid, err = mgr.getKeyspaceGroupMetaWithCheck(mcsutils.DefaultKeyspaceID, 3) + re.NoError(err) + re.Equal(mcsutils.DefaultKeyspaceGroupID, kgid) + re.NotNil(am) + re.NotNil(kg) +} + +// TestKeyspaceMovementConsistency tests the consistency of keyspace movement. +// When a keyspace is moved from one keyspace group to another, the allocator manager +// update source group and target group state in etcd atomically. The TSO keyspace group +// manager watches the state change in persistent store but hard to apply the movement state +// change across two groups atomically. This test case is to test the movement state is +// eventually consistent, for example, if a keyspace "move to group B" event is applied +// before "move away from group A" event, the second event shouldn't overwrite the global +// state, such as the global keyspace group lookup table. +func (suite *keyspaceGroupManagerTestSuite) TestKeyspaceMovementConsistency() { + re := suite.Require() + + mgr := suite.newUniqueKeyspaceGroupManager(1) + re.NotNil(mgr) + defer mgr.Close() + + rootPath := mgr.legacySvcRootPath + svcAddr := mgr.tsoServiceID.ServiceAddr + + var ( + am *AllocatorManager + kg *endpoint.KeyspaceGroup + kgid uint32 + err error + event *etcdEvent + ) + + // Create keyspace group 0 which contains keyspace 0, 1, 2. + addKeyspaceGroupAssignment( + suite.ctx, suite.etcdClient, mcsutils.DefaultKeyspaceGroupID, + rootPath, []string{svcAddr}, []int{0}, []uint32{mcsutils.DefaultKeyspaceID, 10, 20}) + // Create keyspace group 1 which contains keyspace 3, 4. + addKeyspaceGroupAssignment( + suite.ctx, suite.etcdClient, uint32(1), rootPath, + []string{svcAddr}, []int{0}, []uint32{11, 21}) + + err = mgr.Initialize() + re.NoError(err) + + // Should be able to get AM for keyspace 10 in keyspace group 0. + am, kg, kgid, err = mgr.getKeyspaceGroupMetaWithCheck(10, mcsutils.DefaultKeyspaceGroupID) + re.NoError(err) + re.Equal(mcsutils.DefaultKeyspaceGroupID, kgid) + re.NotNil(am) + re.NotNil(kg) + + // Move keyspace 10 from keyspace group 0 to keyspace group 1 and apply this state change + // to TSO first. + event = generateKeyspaceGroupPutEvent(1, []uint32{10, 11, 21}, []string{svcAddr}) + err = putKeyspaceGroupToEtcd(suite.ctx, suite.etcdClient, rootPath, event.ksg) + re.NoError(err) + // Wait until the keyspace 10 is served by keyspace group 1. + testutil.Eventually(re, func() bool { + _, _, kgid, err = mgr.getKeyspaceGroupMetaWithCheck(10, 1) + return err == nil && kgid == 1 + }, testutil.WithWaitFor(3*time.Second), testutil.WithTickInterval(50*time.Millisecond)) + + event = generateKeyspaceGroupPutEvent( + mcsutils.DefaultKeyspaceGroupID, []uint32{mcsutils.DefaultKeyspaceID, 20}, []string{svcAddr}) + err = putKeyspaceGroupToEtcd(suite.ctx, suite.etcdClient, rootPath, event.ksg) + re.NoError(err) + + // Sleep for a while to wait for the events to propagate. If the restriction is not working, + // it will cause random failure. + time.Sleep(1 * time.Second) + // Should still be able to get AM for keyspace 10 in keyspace group 1. + _, _, kgid, err = mgr.getKeyspaceGroupMetaWithCheck(10, 1) + re.NoError(err) + re.Equal(uint32(1), kgid) +} + +// TestHandleTSORequestWithWrongMembership tests the case that HandleTSORequest receives +// a tso request with mismatched keyspace and keyspace group. +func (suite *keyspaceGroupManagerTestSuite) TestHandleTSORequestWithWrongMembership() { + re := suite.Require() + + mgr := suite.newUniqueKeyspaceGroupManager(1) + re.NotNil(mgr) + defer mgr.Close() + + // Create keyspace group 0 which contains keyspace 0, 1, 2. + addKeyspaceGroupAssignment( + suite.ctx, suite.etcdClient, uint32(0), mgr.legacySvcRootPath, + []string{mgr.tsoServiceID.ServiceAddr}, []int{0}, []uint32{0, 1, 2}) + + err := mgr.Initialize() + re.NoError(err) + + // Wait until the keyspace group 0 is ready for serving tso requests. + testutil.Eventually(re, func() bool { + member, err := mgr.GetElectionMember(0, 0) + if err != nil { + return false + } + return member.IsLeader() + }, testutil.WithWaitFor(5*time.Second), testutil.WithTickInterval(50*time.Millisecond)) + + // Should succeed because keyspace 0 is actually in keyspace group 0, which is served + // by the current keyspace group manager, instead of keyspace group 1 in ask, and + // keyspace group 0 is returned in the response. + _, keyspaceGroupBelongTo, err := mgr.HandleTSORequest(0, 1, GlobalDCLocation, 1) + re.NoError(err) + re.Equal(uint32(0), keyspaceGroupBelongTo) + + // Should succeed because keyspace 100 doesn't belong to any keyspace group, so it will + // be served by the default keyspace group 0, and keyspace group 0 is returned in the response. + _, keyspaceGroupBelongTo, err = mgr.HandleTSORequest(100, 0, GlobalDCLocation, 1) + re.NoError(err) + re.Equal(uint32(0), keyspaceGroupBelongTo) + + // Should fail because keyspace 100 doesn't belong to any keyspace group, and the keyspace group + // 1 in ask doesn't exist. + _, keyspaceGroupBelongTo, err = mgr.HandleTSORequest(100, 1, GlobalDCLocation, 1) + re.Error(err) + re.Equal(uint32(1), keyspaceGroupBelongTo) +} + +type etcdEvent struct { + eventType mvccpb.Event_EventType + ksg *endpoint.KeyspaceGroup +} + +func generateKeyspaceGroupPutEvent( + groupID uint32, keyspaces []uint32, addrs []string, splitState ...*endpoint.SplitState, +) *etcdEvent { + members := []endpoint.KeyspaceGroupMember{} + for _, addr := range addrs { + members = append(members, endpoint.KeyspaceGroupMember{Address: addr}) + } + var ss *endpoint.SplitState + if len(splitState) > 0 { + ss = splitState[0] + } + + return &etcdEvent{ + eventType: mvccpb.PUT, + ksg: &endpoint.KeyspaceGroup{ + ID: groupID, + Members: members, + Keyspaces: keyspaces, + SplitState: ss, + }, + } +} + +func generateKeyspaceGroupDeleteEvent(groupID uint32) *etcdEvent { + return &etcdEvent{ + eventType: mvccpb.DELETE, + ksg: &endpoint.KeyspaceGroup{ + ID: groupID, + }, + } +} + +func (suite *keyspaceGroupManagerTestSuite) applyEtcdEvents( + re *require.Assertions, + rootPath string, + events []*etcdEvent, +) { + var err error + for _, event := range events { + switch event.eventType { + case mvccpb.PUT: + err = putKeyspaceGroupToEtcd(suite.ctx, suite.etcdClient, rootPath, event.ksg) + case mvccpb.DELETE: + err = deleteKeyspaceGroupInEtcd(suite.ctx, suite.etcdClient, rootPath, event.ksg.ID) + } + re.NoError(err) + } +} + +// runTestLoadMultipleKeyspaceGroupsAssignment tests the loading of multiple keyspace group assignment. +func (suite *keyspaceGroupManagerTestSuite) runTestLoadKeyspaceGroupsAssignment( + re *require.Assertions, + numberOfKeyspaceGroupsToAdd int, + loadKeyspaceGroupsBatchSize int64, // set to 0 to use the default value + probabilityAssignToMe int, // percentage of assigning keyspace groups to this host/pod +) { + expectedGroupIDs := []uint32{} + mgr := suite.newUniqueKeyspaceGroupManager(loadKeyspaceGroupsBatchSize) + re.NotNil(mgr) + defer mgr.Close() + + step := 30 + mux := sync.Mutex{} + wg := sync.WaitGroup{} + for i := 0; i < numberOfKeyspaceGroupsToAdd; i += step { + wg.Add(1) + go func(startID int) { + defer wg.Done() + + endID := startID + step + if endID > numberOfKeyspaceGroupsToAdd { + endID = numberOfKeyspaceGroupsToAdd + } + + randomGen := rand.New(rand.NewSource(time.Now().UnixNano())) + for j := startID; j < endID; j++ { + assignToMe := false + // Assign the keyspace group to this host/pod with the given probability, + // and the keyspace group manager only loads the keyspace groups with id + // less than len(mgr.ams). + if j < len(mgr.ams) && randomGen.Intn(100) < probabilityAssignToMe { + assignToMe = true + mux.Lock() + expectedGroupIDs = append(expectedGroupIDs, uint32(j)) + mux.Unlock() + } + + svcAddrs := make([]string, 0) + if assignToMe { + svcAddrs = append(svcAddrs, mgr.tsoServiceID.ServiceAddr) + } else { + svcAddrs = append(svcAddrs, uuid.NewString()) + } + addKeyspaceGroupAssignment( + suite.ctx, suite.etcdClient, uint32(j), mgr.legacySvcRootPath, + svcAddrs, []int{0}, []uint32{uint32(j)}) + } + }(i) + } + wg.Wait() + + err := mgr.Initialize() + re.NoError(err) + + // If no keyspace group is assigned to this host/pod, the default keyspace group should be initialized. + if numberOfKeyspaceGroupsToAdd <= 0 { + expectedGroupIDs = append(expectedGroupIDs, mcsutils.DefaultKeyspaceGroupID) + } + + // Verify the keyspace group assignment. + // Sort the keyspaces in ascending order + sort.Slice(expectedGroupIDs, func(i, j int) bool { + return expectedGroupIDs[i] < expectedGroupIDs[j] + }) + assignedGroupIDs := collectAssignedKeyspaceGroupIDs(re, mgr) + re.Equal(expectedGroupIDs, assignedGroupIDs) +} + +func (suite *keyspaceGroupManagerTestSuite) newUniqueKeyspaceGroupManager( + loadKeyspaceGroupsBatchSize int64, // set to 0 to use the default value +) *KeyspaceGroupManager { + return suite.newKeyspaceGroupManager(loadKeyspaceGroupsBatchSize, uuid.New().String(), suite.cfg) +} + +func (suite *keyspaceGroupManagerTestSuite) newKeyspaceGroupManager( + loadKeyspaceGroupsBatchSize int64, // set to 0 to use the default value + uniqueStr string, + cfg *TestServiceConfig, +) *KeyspaceGroupManager { + tsoServiceID := &discovery.ServiceRegistryEntry{ServiceAddr: cfg.GetAdvertiseListenAddr()} + tsoServiceKey := discovery.ServicePath(uniqueStr, "tso") + legacySvcRootPath := path.Join("/pd", uniqueStr) + tsoSvcRootPath := path.Join(mcsutils.MicroserviceRootPath, uniqueStr, "tso") + electionNamePrefix := "kgm-test-" + cfg.GetAdvertiseListenAddr() + + kgm := NewKeyspaceGroupManager( + suite.ctx, tsoServiceID, suite.etcdClient, nil, electionNamePrefix, + tsoServiceKey, legacySvcRootPath, tsoSvcRootPath, cfg) + if loadKeyspaceGroupsBatchSize != 0 { + kgm.loadKeyspaceGroupsBatchSize = loadKeyspaceGroupsBatchSize + } + return kgm +} + +// putKeyspaceGroupToEtcd puts a keyspace group to etcd. +func putKeyspaceGroupToEtcd( + ctx context.Context, etcdClient *clientv3.Client, + rootPath string, group *endpoint.KeyspaceGroup, +) error { + key := strings.Join([]string{rootPath, endpoint.KeyspaceGroupIDPath(group.ID)}, "/") + value, err := json.Marshal(group) + if err != nil { + return err + } + + if _, err := etcdClient.Put(ctx, key, string(value)); err != nil { + return err + } + + return nil +} + +// deleteKeyspaceGroupInEtcd deletes a keyspace group in etcd. +func deleteKeyspaceGroupInEtcd( + ctx context.Context, etcdClient *clientv3.Client, + rootPath string, id uint32, +) error { + key := strings.Join([]string{rootPath, endpoint.KeyspaceGroupIDPath(id)}, "/") + + if _, err := etcdClient.Delete(ctx, key); err != nil { + return err + } + + return nil +} + +// addKeyspaceGroupAssignment adds a keyspace group assignment to etcd. +func addKeyspaceGroupAssignment( + ctx context.Context, + etcdClient *clientv3.Client, + groupID uint32, + rootPath string, + svcAddrs []string, + priorites []int, + keyspaces []uint32, +) error { + members := make([]endpoint.KeyspaceGroupMember, len(svcAddrs)) + for i, svcAddr := range svcAddrs { + members[i] = endpoint.KeyspaceGroupMember{Address: svcAddr, Priority: priorites[i]} + } + group := &endpoint.KeyspaceGroup{ + ID: groupID, + Members: members, + Keyspaces: keyspaces, + } + + key := strings.Join([]string{rootPath, endpoint.KeyspaceGroupIDPath(groupID)}, "/") + value, err := json.Marshal(group) + if err != nil { + return err + } + + if _, err := etcdClient.Put(ctx, key, string(value)); err != nil { + return err + } + + return nil +} + +func collectAssignedKeyspaceGroupIDs(re *require.Assertions, kgm *KeyspaceGroupManager) []uint32 { + kgm.RLock() + defer kgm.RUnlock() + + ids := []uint32{} + for i := 0; i < len(kgm.kgs); i++ { + kg := kgm.kgs[i] + if kg == nil { + re.Nil(kgm.ams[i], fmt.Sprintf("ksg is nil but am is not nil for id %d", i)) + } else { + am := kgm.ams[i] + if am != nil { + re.Equal(i, int(am.kgID)) + re.Equal(i, int(kg.ID)) + for _, m := range kg.Members { + if m.Address == kgm.tsoServiceID.ServiceAddr { + ids = append(ids, uint32(i)) + break + } + } + } + } + } + + return ids +} + +func collectAllLoadedKeyspaceGroupIDs(kgm *KeyspaceGroupManager) []uint32 { + kgm.RLock() + defer kgm.RUnlock() + + ids := []uint32{} + for i := 0; i < len(kgm.kgs); i++ { + kg := kgm.kgs[i] + if kg != nil { + ids = append(ids, uint32(i)) + } + } + + return ids +} + +func (suite *keyspaceGroupManagerTestSuite) TestUpdateKeyspaceGroupMembership() { + re := suite.Require() + + // Start from an empty keyspace group. + // Use non-default keyspace group ID. + // The default keyspace group always contains the default keyspace. + // We have dedicated tests for the default keyspace group. + groupID := uint32(1) + oldGroup := &endpoint.KeyspaceGroup{ID: groupID, Keyspaces: []uint32{}} + newGroup := &endpoint.KeyspaceGroup{ID: groupID, Keyspaces: []uint32{}} + kgm := &KeyspaceGroupManager{ + state: state{ + keyspaceLookupTable: make(map[uint32]uint32), + }} + + kgm.updateKeyspaceGroupMembership(oldGroup, newGroup, true) + verifyLocalKeyspaceLookupTable(re, newGroup.KeyspaceLookupTable, newGroup.Keyspaces) + verifyGlobalKeyspaceLookupTable(re, kgm.keyspaceLookupTable, newGroup.KeyspaceLookupTable) + + targetKeyspacesList := [][]uint32{ + {1}, // Add keyspace 1 to the keyspace group. + {1, 2}, // Add keyspace 2 to the keyspace group. + {1, 2}, // No change. + {1, 2, 3, 4}, // Add keyspace 3 and 4 to the keyspace group. + {5, 6, 7}, // Remove keyspace 1, 2, 3, 4 from the keyspace group and add 5, 6, 7 + {7, 8, 9}, // Partially update the keyspace group. + {1, 2, 3, 4, 5, 6, 7, 8, 9}, // Add more keyspace to the keyspace group. + {9, 8, 4, 5, 6}, // Out of order. + {9, 8, 4, 5, 6}, // No change. Out of order. + {8, 9}, // Remove + {10}, // Remove + {}, // End with the empty keyspace group. + } + + for _, keyspaces := range targetKeyspacesList { + oldGroup = newGroup + keyspacesCopy := make([]uint32, len(keyspaces)) + copy(keyspacesCopy, keyspaces) + newGroup = &endpoint.KeyspaceGroup{ID: groupID, Keyspaces: keyspacesCopy} + kgm.updateKeyspaceGroupMembership(oldGroup, newGroup, true) + verifyLocalKeyspaceLookupTable(re, newGroup.KeyspaceLookupTable, newGroup.Keyspaces) + verifyGlobalKeyspaceLookupTable(re, kgm.keyspaceLookupTable, newGroup.KeyspaceLookupTable) + + // Verify the keyspaces loaded is sorted. + re.Equal(len(keyspaces), len(newGroup.Keyspaces)) + for i := 0; i < len(newGroup.Keyspaces); i++ { + if i > 0 { + re.True(newGroup.Keyspaces[i-1] < newGroup.Keyspaces[i]) + } + } + } +} + +func verifyLocalKeyspaceLookupTable( + re *require.Assertions, keyspaceLookupTable map[uint32]struct{}, newKeyspaces []uint32, +) { + re.Equal(len(newKeyspaces), len(keyspaceLookupTable), + fmt.Sprintf("%v %v", newKeyspaces, keyspaceLookupTable)) + for _, keyspace := range newKeyspaces { + _, ok := keyspaceLookupTable[keyspace] + re.True(ok) + } +} + +func verifyGlobalKeyspaceLookupTable( + re *require.Assertions, + gKeyspaceLookupTable map[uint32]uint32, + lKeyspaceLookupTable map[uint32]struct{}, +) { + for keyspace := range gKeyspaceLookupTable { + _, ok := lKeyspaceLookupTable[keyspace] + re.True(ok) + } + for keyspace := range lKeyspaceLookupTable { + _, ok := gKeyspaceLookupTable[keyspace] + re.True(ok) + } +} + +func (suite *keyspaceGroupManagerTestSuite) TestGroupSplitUpdateRetry() { + re := suite.Require() + + // Start with the empty keyspace group assignment. + mgr := suite.newUniqueKeyspaceGroupManager(0) + re.NotNil(mgr) + defer mgr.Close() + err := mgr.Initialize() + re.NoError(err) + + rootPath := mgr.legacySvcRootPath + svcAddr := mgr.tsoServiceID.ServiceAddr + + events := []*etcdEvent{} + // Split target keyspace group event arrives first. + events = append(events, generateKeyspaceGroupPutEvent(2, []uint32{2} /* Mock 2 replicas */, []string{svcAddr, svcAddr}, &endpoint.SplitState{ + SplitSource: 1, + })) + // Split source keyspace group event arrives later. + events = append(events, generateKeyspaceGroupPutEvent(1, []uint32{1}, []string{svcAddr, svcAddr}, &endpoint.SplitState{ + SplitSource: 1, + })) + + // Eventually, this keyspace groups manager is expected to serve the following keyspace groups. + expectedGroupIDs := []uint32{0, 1, 2} + + // Apply the keyspace group assignment change events to etcd. + suite.applyEtcdEvents(re, rootPath, events) + + // Verify the keyspace group assignment. + testutil.Eventually(re, func() bool { + assignedGroupIDs := collectAssignedKeyspaceGroupIDs(re, mgr) + return reflect.DeepEqual(expectedGroupIDs, assignedGroupIDs) + }) +} + +// TestPrimaryPriorityChange tests the case that the primary priority of a keyspace group changes +// and the locations of the primaries should be updated accordingly. +func (suite *keyspaceGroupManagerTestSuite) TestPrimaryPriorityChange() { + re := suite.Require() + re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/tso/fastPrimaryPriorityCheck", `return(true)`)) + defer func() { + re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/tso/fastPrimaryPriorityCheck")) + }() + + var err error + defaultPriority := mcsutils.DefaultKeyspaceGroupReplicaPriority + uniqueStr := uuid.New().String() + rootPath := path.Join("/pd", uniqueStr) + cfg1 := suite.createConfig() + cfg2 := suite.createConfig() + svcAddr1 := cfg1.GetAdvertiseListenAddr() + svcAddr2 := cfg2.GetAdvertiseListenAddr() + + // Register TSO server 1 + err = suite.registerTSOServer(re, uniqueStr, svcAddr1, cfg1) + re.NoError(err) + defer func() { + re.NoError(suite.deregisterTSOServer(uniqueStr, svcAddr1)) + }() + + // Create three keyspace groups on two TSO servers with default replica priority. + ids := []uint32{0, mcsutils.MaxKeyspaceGroupCountInUse / 2, mcsutils.MaxKeyspaceGroupCountInUse - 1} + for _, id := range ids { + addKeyspaceGroupAssignment( + suite.ctx, suite.etcdClient, id, rootPath, + []string{svcAddr1, svcAddr2}, []int{defaultPriority, defaultPriority}, []uint32{id}) + } + + // Create the first TSO server which loads all three keyspace groups created above. + // All primaries should be on the first TSO server. + mgr1 := suite.newKeyspaceGroupManager(1, uniqueStr, cfg1) + re.NotNil(mgr1) + defer mgr1.Close() + err = mgr1.Initialize() + re.NoError(err) + // Wait until all keyspace groups are ready for serving tso requests. + waitForPrimariesServing(re, []*KeyspaceGroupManager{mgr1, mgr1, mgr1}, ids) + + // We increase the priority of the TSO server 2 which hasn't started yet. The primaries + // on the TSO server 1 shouldn't move. + for _, id := range ids { + addKeyspaceGroupAssignment( + suite.ctx, suite.etcdClient, id, rootPath, + []string{svcAddr1, svcAddr2}, []int{defaultPriority, defaultPriority + 1}, []uint32{id}) + } + + // And the primaries on TSO Server 1 should continue to serve TSO requests without any failures. + for i := 0; i < 100; i++ { + for _, id := range ids { + _, keyspaceGroupBelongTo, err := mgr1.HandleTSORequest(id, id, GlobalDCLocation, 1) + re.NoError(err) + re.Equal(id, keyspaceGroupBelongTo) + } + } + + // Continually sending TSO requests to the TSO server 1 to make sure the primaries will move back + // to it at the end of test + ctx, cancel := context.WithCancel(context.Background()) + var wg sync.WaitGroup + checkTSO(ctx, re, &wg, mgr1, ids) + + // Create the Second TSO server. + err = suite.registerTSOServer(re, uniqueStr, svcAddr2, cfg2) + re.NoError(err) + mgr2 := suite.newKeyspaceGroupManager(1, uniqueStr, cfg2) + re.NotNil(mgr2) + err = mgr2.Initialize() + re.NoError(err) + // All primaries should eventually move to the second TSO server because of the higher priority. + waitForPrimariesServing(re, []*KeyspaceGroupManager{mgr2, mgr2, mgr2}, ids) + + // Shutdown the second TSO server. + mgr2.Close() + re.NoError(suite.deregisterTSOServer(uniqueStr, svcAddr2)) + // The primaries should move back to the first TSO server. + waitForPrimariesServing(re, []*KeyspaceGroupManager{mgr1, mgr1, mgr1}, ids) + + // Restart the Second TSO server. + err = suite.registerTSOServer(re, uniqueStr, svcAddr2, cfg2) + re.NoError(err) + defer func() { + re.NoError(suite.deregisterTSOServer(uniqueStr, svcAddr2)) + }() + mgr2 = suite.newKeyspaceGroupManager(1, uniqueStr, cfg2) + re.NotNil(mgr2) + defer mgr2.Close() + err = mgr2.Initialize() + re.NoError(err) + // All primaries should eventually move to the second TSO server because of the higher priority. + waitForPrimariesServing(re, []*KeyspaceGroupManager{mgr2, mgr2, mgr2}, ids) + + mgrs := []*KeyspaceGroupManager{mgr2, mgr2, mgr2} + for i, id := range ids { + // Set the keyspace group replica on the first TSO server to have higher priority. + addKeyspaceGroupAssignment( + suite.ctx, suite.etcdClient, id, rootPath, + []string{svcAddr1, svcAddr2}, []int{defaultPriority - 1, defaultPriority - 2}, []uint32{id}) + // The primary of this keyspace group should move back to the first TSO server. + mgrs[i] = mgr1 + waitForPrimariesServing(re, mgrs, ids) + } + + cancel() + wg.Wait() +} + +// Register TSO server. +func (suite *keyspaceGroupManagerTestSuite) registerTSOServer( + re *require.Assertions, clusterID, svcAddr string, cfg *TestServiceConfig, +) error { + // Register TSO server 1 + serviceID := &discovery.ServiceRegistryEntry{ServiceAddr: cfg.GetAdvertiseListenAddr()} + serializedEntry, err := serviceID.Serialize() + re.NoError(err) + serviceKey := discovery.RegistryPath(clusterID, mcsutils.TSOServiceName, svcAddr) + _, err = suite.etcdClient.Put(suite.ctx, serviceKey, serializedEntry) + return err +} + +// Deregister TSO server. +func (suite *keyspaceGroupManagerTestSuite) deregisterTSOServer(clusterID, svcAddr string) error { + serviceKey := discovery.RegistryPath(clusterID, mcsutils.TSOServiceName, svcAddr) + if _, err := suite.etcdClient.Delete(suite.ctx, serviceKey); err != nil { + return err + } + return nil +} + +func checkTSO( + ctx context.Context, re *require.Assertions, wg *sync.WaitGroup, + mgr *KeyspaceGroupManager, ids []uint32, +) { + wg.Add(len(ids)) + for _, id := range ids { + go func(id uint32) { + defer wg.Done() + var ts, lastTS uint64 + for { + select { + case <-ctx.Done(): + // Make sure the lastTS is not empty + re.NotEmpty(lastTS) + return + default: + } + respTS, respGroupID, err := mgr.HandleTSORequest(id, id, GlobalDCLocation, 1) + // omit the error check since there are many kinds of errors during primaries movement + if err != nil { + continue + } + re.Equal(id, respGroupID) + ts = tsoutil.ComposeTS(respTS.Physical, respTS.Logical) + re.Less(lastTS, ts) + lastTS = ts + } + }(id) + } +} + +func waitForPrimariesServing( + re *require.Assertions, mgrs []*KeyspaceGroupManager, ids []uint32, +) { + testutil.Eventually(re, func() bool { + for i := 0; i < 100; i++ { + for j, id := range ids { + if member, err := mgrs[j].GetElectionMember(id, id); err != nil || !member.IsLeader() { + return false + } + if _, _, err := mgrs[j].HandleTSORequest(id, id, GlobalDCLocation, 1); err != nil { + return false + } + } + } + return true + }, testutil.WithWaitFor(5*time.Second), testutil.WithTickInterval(50*time.Millisecond)) +} diff --git a/pkg/tso/local_allocator.go b/pkg/tso/local_allocator.go index 1a54e966961..9995d5cec3f 100644 --- a/pkg/tso/local_allocator.go +++ b/pkg/tso/local_allocator.go @@ -25,6 +25,8 @@ import ( "github.com/pingcap/log" "github.com/tikv/pd/pkg/election" "github.com/tikv/pd/pkg/errs" + "github.com/tikv/pd/pkg/mcs/utils" + "github.com/tikv/pd/pkg/utils/logutil" "github.com/tikv/pd/pkg/utils/tsoutil" "github.com/tikv/pd/pkg/utils/typeutil" "go.etcd.io/etcd/clientv3" @@ -42,7 +44,7 @@ type LocalTSOAllocator struct { // for election use, notice that the leadership that member holds is // the leadership for PD leader. Local TSO Allocator's leadership is for the // election of Local TSO Allocator leader among several PD servers and - // Local TSO Allocator only use member's some etcd and pbpd.Member info. + // Local TSO Allocator only use member's some etcd and pdpb.Member info. // So it's not conflicted. rootPath string allocatorLeader atomic.Value // stored as *pdpb.Member @@ -54,13 +56,23 @@ func NewLocalTSOAllocator( leadership *election.Leadership, dcLocation string, ) Allocator { + // Construct the timestampOracle path prefix, which is: + // 1. for the default keyspace group: + // lta/{dc-location} in /pd/{cluster_id}/lta/{dc-location}/timestamp + // 2. for the non-default keyspace groups: + // {group}/lta/{dc-location} in /ms/{cluster_id}/tso/{group}/lta/{dc-location}/timestamp + var tsPath string + if am.kgID == utils.DefaultKeyspaceGroupID { + tsPath = path.Join(localTSOAllocatorEtcdPrefix, dcLocation) + } else { + tsPath = path.Join(fmt.Sprintf("%05d", am.kgID), localTSOAllocatorEtcdPrefix, dcLocation) + } return &LocalTSOAllocator{ allocatorManager: am, leadership: leadership, timestampOracle: ×tampOracle{ client: leadership.GetClient(), - rootPath: am.rootPath, - ltsPath: path.Join(localTSOAllocatorEtcdPrefix, dcLocation), + tsPath: tsPath, storage: am.storage, saveInterval: am.saveInterval, updatePhysicalInterval: am.updatePhysicalInterval, @@ -72,6 +84,14 @@ func NewLocalTSOAllocator( } } +// GetTimestampPath returns the timestamp path in etcd. +func (lta *LocalTSOAllocator) GetTimestampPath() string { + if lta == nil || lta.timestampOracle == nil { + return "" + } + return lta.timestampOracle.GetTimestampPath() +} + // GetDCLocation returns the local allocator's dc-location. func (lta *LocalTSOAllocator) GetDCLocation() string { return lta.timestampOracle.dcLocation @@ -105,7 +125,8 @@ func (lta *LocalTSOAllocator) SetTSO(tso uint64, ignoreSmaller, skipUpperBoundCh func (lta *LocalTSOAllocator) GenerateTSO(count uint32) (pdpb.Timestamp, error) { if !lta.leadership.Check() { tsoCounter.WithLabelValues("not_leader", lta.timestampOracle.dcLocation).Inc() - return pdpb.Timestamp{}, errs.ErrGenerateTimestamp.FastGenByArgs(fmt.Sprintf("requested pd %s of %s allocator", errs.NotLeaderErr, lta.timestampOracle.dcLocation)) + return pdpb.Timestamp{}, errs.ErrGenerateTimestamp.FastGenByArgs( + fmt.Sprintf("requested pd %s of %s allocator", errs.NotLeaderErr, lta.timestampOracle.dcLocation)) } return lta.timestampOracle.getTS(lta.leadership, count, lta.allocatorManager.GetSuffixBits()) } @@ -117,7 +138,7 @@ func (lta *LocalTSOAllocator) Reset() { } // setAllocatorLeader sets the current Local TSO Allocator leader. -func (lta *LocalTSOAllocator) setAllocatorLeader(member *pdpb.Member) { +func (lta *LocalTSOAllocator) setAllocatorLeader(member interface{}) { lta.allocatorLeader.Store(member) } @@ -136,8 +157,8 @@ func (lta *LocalTSOAllocator) GetAllocatorLeader() *pdpb.Member { } // GetMember returns the Local TSO Allocator's member value. -func (lta *LocalTSOAllocator) GetMember() *pdpb.Member { - return lta.allocatorManager.member.Member() +func (lta *LocalTSOAllocator) GetMember() ElectionMember { + return lta.allocatorManager.member } // GetCurrentTSO returns current TSO in memory. @@ -164,7 +185,7 @@ func (lta *LocalTSOAllocator) WriteTSO(maxTS *pdpb.Timestamp) error { // EnableAllocatorLeader sets the Local TSO Allocator itself to a leader. func (lta *LocalTSOAllocator) EnableAllocatorLeader() { - lta.setAllocatorLeader(lta.allocatorManager.member.Member()) + lta.setAllocatorLeader(lta.allocatorManager.member.GetMember()) } // CampaignAllocatorLeader is used to campaign a Local TSO Allocator's leadership. @@ -174,23 +195,24 @@ func (lta *LocalTSOAllocator) CampaignAllocatorLeader(leaseTimeout int64, cmps . // KeepAllocatorLeader is used to keep the PD leader's leadership. func (lta *LocalTSOAllocator) KeepAllocatorLeader(ctx context.Context) { + defer logutil.LogPanic() lta.leadership.Keep(ctx) } // IsAllocatorLeader returns whether the allocator is still a // Local TSO Allocator leader by checking its leadership's lease and leader info. func (lta *LocalTSOAllocator) IsAllocatorLeader() bool { - return lta.leadership.Check() && lta.GetAllocatorLeader().GetMemberId() == lta.GetMember().GetMemberId() + return lta.leadership.Check() && lta.GetAllocatorLeader().GetMemberId() == lta.GetMember().ID() } // isSameAllocatorLeader checks whether a server is the leader itself. func (lta *LocalTSOAllocator) isSameAllocatorLeader(leader *pdpb.Member) bool { - return leader.GetMemberId() == lta.allocatorManager.member.Member().MemberId + return leader.GetMemberId() == lta.allocatorManager.member.ID() } // CheckAllocatorLeader checks who is the current Local TSO Allocator leader, and returns true if it is needed to check later. func (lta *LocalTSOAllocator) CheckAllocatorLeader() (*pdpb.Member, int64, bool) { - if err := lta.allocatorManager.member.PrecheckLeader(); err != nil { + if err := lta.allocatorManager.member.PreCheckLeader(); err != nil { log.Error("no etcd leader, check local tso allocator leader later", zap.String("dc-location", lta.timestampOracle.dcLocation), errs.ZapError(err)) time.Sleep(200 * time.Millisecond) diff --git a/pkg/tso/testutil.go b/pkg/tso/testutil.go new file mode 100644 index 00000000000..9225b21dfac --- /dev/null +++ b/pkg/tso/testutil.go @@ -0,0 +1,112 @@ +// Copyright 2023 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tso + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" + "github.com/tikv/pd/pkg/utils/etcdutil" + "github.com/tikv/pd/pkg/utils/grpcutil" + "go.etcd.io/etcd/clientv3" + "go.etcd.io/etcd/embed" +) + +var _ ServiceConfig = (*TestServiceConfig)(nil) + +// TestServiceConfig implements the ServiceConfig interface. +type TestServiceConfig struct { + Name string // Name of the service. + BackendEndpoints string // Backend endpoints of the service. + ListenAddr string // Address the service listens on. + AdvertiseListenAddr string // Address the service advertises to the clients. + LeaderLease int64 // Leader lease. + LocalTSOEnabled bool // Whether local TSO is enabled. + TSOUpdatePhysicalInterval time.Duration // Interval to update TSO in physical storage. + TSOSaveInterval time.Duration // Interval to save TSO to physical storage. + MaxResetTSGap time.Duration // Maximum gap to reset TSO. + TLSConfig *grpcutil.TLSConfig // TLS configuration. +} + +// GetName returns the Name field of TestServiceConfig. +func (c *TestServiceConfig) GetName() string { + return c.Name +} + +// GeBackendEndpoints returns the BackendEndpoints field of TestServiceConfig. +func (c *TestServiceConfig) GeBackendEndpoints() string { + return c.BackendEndpoints +} + +// GetListenAddr returns the ListenAddr field of TestServiceConfig. +func (c *TestServiceConfig) GetListenAddr() string { + return c.ListenAddr +} + +// GetAdvertiseListenAddr returns the AdvertiseListenAddr field of TestServiceConfig. +func (c *TestServiceConfig) GetAdvertiseListenAddr() string { + return c.AdvertiseListenAddr +} + +// GetLeaderLease returns the LeaderLease field of TestServiceConfig. +func (c *TestServiceConfig) GetLeaderLease() int64 { + return c.LeaderLease +} + +// IsLocalTSOEnabled returns the LocalTSOEnabled field of TestServiceConfig. +func (c *TestServiceConfig) IsLocalTSOEnabled() bool { + return c.LocalTSOEnabled +} + +// GetTSOUpdatePhysicalInterval returns the TSOUpdatePhysicalInterval field of TestServiceConfig. +func (c *TestServiceConfig) GetTSOUpdatePhysicalInterval() time.Duration { + return c.TSOUpdatePhysicalInterval +} + +// GetTSOSaveInterval returns the TSOSaveInterval field of TestServiceConfig. +func (c *TestServiceConfig) GetTSOSaveInterval() time.Duration { + return c.TSOSaveInterval +} + +// GetMaxResetTSGap returns the MaxResetTSGap field of TestServiceConfig. +func (c *TestServiceConfig) GetMaxResetTSGap() time.Duration { + return c.MaxResetTSGap +} + +// GetTLSConfig returns the TLSConfig field of TestServiceConfig. +func (c *TestServiceConfig) GetTLSConfig() *grpcutil.TLSConfig { + return c.TLSConfig +} + +func startEmbeddedEtcd(t *testing.T) (backendEndpoint string, etcdClient *clientv3.Client, clean func()) { + re := require.New(t) + cfg := etcdutil.NewTestSingleConfig(t) + etcd, err := embed.StartEtcd(cfg) + re.NoError(err) + clean = func() { + etcd.Close() + } + + backendEndpoint = cfg.LCUrls[0].String() + re.NoError(err) + + etcdClient, err = clientv3.NewFromURL(backendEndpoint) + re.NoError(err) + + <-etcd.Server.ReadyNotify() + + return +} diff --git a/pkg/tso/tso.go b/pkg/tso/tso.go index 0b4ad7da745..33da6e8d11f 100644 --- a/pkg/tso/tso.go +++ b/pkg/tso/tso.go @@ -16,7 +16,6 @@ package tso import ( "fmt" - "path" "sync/atomic" "time" @@ -34,7 +33,6 @@ import ( ) const ( - timestampKey = "timestamp" // UpdateTimestampGuard is the min timestamp interval. UpdateTimestampGuard = time.Millisecond // maxLogical is the max upper limit for logical time. @@ -60,10 +58,9 @@ type tsoObject struct { // timestampOracle is used to maintain the logic of TSO. type timestampOracle struct { - client *clientv3.Client - rootPath string - // When ltsPath is empty, it means that it is a global timestampOracle. - ltsPath string + client *clientv3.Client + // When tsPath is empty, it means that it is a global timestampOracle. + tsPath string storage endpoint.TSOStorage // TODO: remove saveInterval saveInterval time.Duration @@ -116,7 +113,7 @@ func (t *timestampOracle) generateTSO(count int64, suffixBits int) (physical int t.tsoMux.logical += count logical = t.tsoMux.logical if suffixBits > 0 && t.suffix >= 0 { - logical = t.differentiateLogical(logical, suffixBits) + logical = t.calibrateLogical(logical, suffixBits) } // Return the last update time lastUpdateTime = t.tsoMux.updateTime @@ -137,12 +134,13 @@ func (t *timestampOracle) generateTSO(count int64, suffixBits int) (physical int // dc-1: xxxxxxxxxx00000001 // dc-2: xxxxxxxxxx00000010 // dc-3: xxxxxxxxxx00000011 -func (t *timestampOracle) differentiateLogical(rawLogical int64, suffixBits int) int64 { +func (t *timestampOracle) calibrateLogical(rawLogical int64, suffixBits int) int64 { return rawLogical<= current + maxResetTSGap func (t *timestampOracle) resetUserTimestamp(leadership *election.Leadership, tso uint64, ignoreSmaller bool) error { return t.resetUserTimestampInner(leadership, tso, ignoreSmaller, false) @@ -240,7 +241,7 @@ func (t *timestampOracle) resetUserTimestampInner(leadership *election.Leadershi // save into etcd only if nextPhysical is close to lastSavedTime if typeutil.SubRealTimeByWallClock(t.lastSavedTime.Load().(time.Time), nextPhysical) <= UpdateTimestampGuard { save := nextPhysical.Add(t.saveInterval) - if err := t.storage.SaveTimestamp(t.getTimestampPath(), save); err != nil { + if err := t.storage.SaveTimestamp(t.GetTimestampPath(), save); err != nil { tsoCounter.WithLabelValues("err_save_reset_ts", t.dcLocation).Inc() return err } @@ -285,7 +286,11 @@ func (t *timestampOracle) UpdateTimestamp(leadership *election.Leadership) error jetLag := typeutil.SubRealTimeByWallClock(now, prevPhysical) if jetLag > 3*t.updatePhysicalInterval && jetLag > jetLagWarningThreshold { - log.Warn("clock offset", zap.Duration("jet-lag", jetLag), zap.Time("prev-physical", prevPhysical), zap.Time("now", now), zap.Duration("update-physical-interval", t.updatePhysicalInterval)) + log.Warn("clock offset", + zap.Duration("jet-lag", jetLag), + zap.Time("prev-physical", prevPhysical), + zap.Time("now", now), + zap.Duration("update-physical-interval", t.updatePhysicalInterval)) tsoCounter.WithLabelValues("slow_save", t.dcLocation).Inc() } @@ -312,7 +317,11 @@ func (t *timestampOracle) UpdateTimestamp(leadership *election.Leadership) error // The time window needs to be updated and saved to etcd. if typeutil.SubRealTimeByWallClock(t.lastSavedTime.Load().(time.Time), next) <= UpdateTimestampGuard { save := next.Add(t.saveInterval) - if err := t.storage.SaveTimestamp(t.getTimestampPath(), save); err != nil { + if err := t.storage.SaveTimestamp(t.GetTimestampPath(), save); err != nil { + log.Warn("save timestamp failed", + zap.String("dc-location", t.dcLocation), + zap.String("timestamp-path", t.GetTimestampPath()), + zap.Error(err)) tsoCounter.WithLabelValues("err_save_update_ts", t.dcLocation).Inc() return err } @@ -358,7 +367,7 @@ func (t *timestampOracle) getTS(leadership *election.Leadership, count uint32, s } // In case lease expired after the first check. if !leadership.Check() { - return pdpb.Timestamp{}, errs.ErrGenerateTimestamp.FastGenByArgs("not the pd or local tso allocator leader anymore") + return pdpb.Timestamp{}, errs.ErrGenerateTimestamp.FastGenByArgs(fmt.Sprintf("requested %s anymore", errs.NotLeaderErr)) } resp.SuffixBits = uint32(suffixBits) return resp, nil diff --git a/pkg/tso/util.go b/pkg/tso/util.go new file mode 100644 index 00000000000..d7672373a32 --- /dev/null +++ b/pkg/tso/util.go @@ -0,0 +1,35 @@ +// Copyright 2023 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tso + +import ( + "fmt" + "regexp" + "strconv" +) + +// ExtractKeyspaceGroupIDFromPath extracts keyspace group id from the given path, which contains +// the pattern of `tso/keyspace_groups/membership/(\d{5})$`. +func ExtractKeyspaceGroupIDFromPath(compiledRegexp *regexp.Regexp, path string) (uint32, error) { + match := compiledRegexp.FindStringSubmatch(path) + if match == nil { + return 0, fmt.Errorf("invalid keyspace group id path: %s", path) + } + id, err := strconv.ParseUint(match[1], 10, 32) + if err != nil { + return 0, fmt.Errorf("failed to parse keyspace group ID: %v", err) + } + return uint32(id), nil +} diff --git a/pkg/tso/util_test.go b/pkg/tso/util_test.go new file mode 100644 index 00000000000..8e4971797d5 --- /dev/null +++ b/pkg/tso/util_test.go @@ -0,0 +1,93 @@ +// Copyright 2023 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tso + +import ( + "testing" + + "github.com/stretchr/testify/require" + "github.com/tikv/pd/pkg/storage/endpoint" +) + +func TestExtractKeyspaceGroupIDFromKeyspaceGroupMembershipPath(t *testing.T) { + re := require.New(t) + + compiledRegexp := endpoint.GetCompiledKeyspaceGroupIDRegexp() + + rightCases := []struct { + path string + id uint32 + }{ + {path: "/pd/{cluster_id}/tso/keyspace_groups/membership/00000", id: 0}, + {path: "/pd/{cluster_id}/tso/keyspace_groups/membership/00001", id: 1}, + {path: "/pd/{cluster_id}/tso/keyspace_groups/membership/12345", id: 12345}, + {path: "/pd/{cluster_id}/tso/keyspace_groups/membership/99999", id: 99999}, + {path: "tso/keyspace_groups/membership/00000", id: 0}, + {path: "tso/keyspace_groups/membership/00001", id: 1}, + {path: "tso/keyspace_groups/membership/12345", id: 12345}, + {path: "tso/keyspace_groups/membership/99999", id: 99999}, + } + + for _, tt := range rightCases { + id, err := ExtractKeyspaceGroupIDFromPath(compiledRegexp, tt.path) + re.Equal(tt.id, id) + re.NoError(err) + } + + wrongCases := []struct { + path string + }{ + {path: ""}, + {path: "00001"}, + {path: "xxx/keyspace_groups/membership/00001"}, + {path: "tso/xxxxxxxxxxxxxxx/membership/00001"}, + {path: "tso/keyspace_groups/xxxxxxxxxx/00001"}, + {path: "/pd/{cluster_id}/tso/keyspace_groups/xxxxxxxxxx/00001"}, + {path: "/pd/{cluster_id}/xxx/keyspace_groups/membership/00001"}, + {path: "/pd/{cluster_id}/tso/xxxxxxxxxxxxxxx/membership/00001"}, + {path: "/pd/{cluster_id}/tso/keyspace_groups/membership/"}, + {path: "/pd/{cluster_id}/tso/keyspace_groups/membership/0"}, + {path: "/pd/{cluster_id}/tso/keyspace_groups/membership/0001"}, + {path: "/pd/{cluster_id}/tso/keyspace_groups/membership/123456"}, + {path: "/pd/{cluster_id}/tso/keyspace_groups/membership/1234a"}, + {path: "/pd/{cluster_id}/tso/keyspace_groups/membership/12345a"}, + } + + for _, tt := range wrongCases { + _, err := ExtractKeyspaceGroupIDFromPath(compiledRegexp, tt.path) + re.Error(err) + } +} + +func TestExtractKeyspaceGroupIDFromKeyspaceGroupPrimaryPath(t *testing.T) { + re := require.New(t) + + compiledRegexp := endpoint.GetCompiledNonDefaultIDRegexp(uint64(111)) + + rightCases := []struct { + path string + id uint32 + }{ + {path: "/ms/111/tso/keyspace_groups/election/00001/primary", id: 1}, + {path: "/ms/111/tso/keyspace_groups/election/12345/primary", id: 12345}, + {path: "/ms/111/tso/keyspace_groups/election/99999/primary", id: 99999}, + } + + for _, tt := range rightCases { + id, err := ExtractKeyspaceGroupIDFromPath(compiledRegexp, tt.path) + re.Equal(tt.id, id) + re.NoError(err) + } +} diff --git a/server/cluster/unsafe_recovery_controller.go b/pkg/unsaferecovery/unsafe_recovery_controller.go similarity index 83% rename from server/cluster/unsafe_recovery_controller.go rename to pkg/unsaferecovery/unsafe_recovery_controller.go index d54a331048e..b3716d145d1 100644 --- a/server/cluster/unsafe_recovery_controller.go +++ b/pkg/unsaferecovery/unsafe_recovery_controller.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package cluster +package unsaferecovery import ( "bytes" @@ -33,22 +33,24 @@ import ( "github.com/tikv/pd/pkg/errs" "github.com/tikv/pd/pkg/utils/logutil" "github.com/tikv/pd/pkg/utils/syncutil" + "github.com/tikv/pd/server/config" "go.uber.org/zap" ) -type unsafeRecoveryStage int +// stage is the stage of unsafe recovery. +type stage int const ( storeRequestInterval = time.Second * 40 ) -// Stage transition graph: for more details, please check `unsafeRecoveryController.HandleStoreHeartbeat()` +// Stage transition graph: for more details, please check `Controller.HandleStoreHeartbeat()` // // +-----------+ +-----------+ // +-----------+ | | | | -// | | | collect | | tombstone | -// | idle |------>| Report |-----+---->| tiflash |-----+ -// | | | | | | learner | | +// | | | Collect | | Tombstone | +// | Idle |------>| Report |-----+---->| Tiflash |-----+ +// | | | | | | Learner | | // +-----------+ +-----------+ | | | | // | +-----------+ | // | | | @@ -56,7 +58,7 @@ const ( // | v | // | +-----------+ | // | | | | -// | | force | | +// | | Force | | // | | LeaderFor |-----+ // | |CommitMerge| | // | | | | @@ -66,8 +68,8 @@ const ( // | v | // | +-----------+ | +-----------+ // | | | | | | +-----------+ -// | | force | | | exitForce | | | -// | | Leader |-----+---->| Leader |------->| failed | +// | | Force | | | exitForce | | | +// | | Leader |-----+---->| Leader |------->| Failed | // | | | | | | | | // | +-----------+ | +-----------+ +-----------+ // | | | @@ -75,7 +77,7 @@ const ( // | v | // | +-----------+ | // | | | | -// | | demote | | +// | | Demote | | // +-----| Voter |-----| // | | | // +-----------+ | @@ -84,28 +86,38 @@ const ( // v | // +-----------+ +-----------+ | // +-----------+ | | | | | -// | | | exitForce | | create | | -// | finished |<------| Leader |<----------| Region |-----+ +// | | | ExitForce | | Create | | +// | Finished |<------| Leader |<----------| Region |-----+ // | | | | | | // +-----------+ +-----------+ +-----------+ const ( - idle unsafeRecoveryStage = iota - collectReport - tombstoneTiFlashLearner - forceLeaderForCommitMerge - forceLeader - demoteFailedVoter - createEmptyRegion - exitForceLeader - finished - failed + Idle stage = iota + CollectReport + TombstoneTiFlashLearner + ForceLeaderForCommitMerge + ForceLeader + DemoteFailedVoter + CreateEmptyRegion + ExitForceLeader + Finished + Failed ) -type unsafeRecoveryController struct { +type cluster interface { + core.StoreSetInformer + + DropCacheAllRegion() + AllocID() (uint64, error) + BuryStore(storeID uint64, forceBury bool) error + GetPersistOptions() *config.PersistOptions +} + +// Controller is used to control the unsafe recovery process. +type Controller struct { syncutil.RWMutex - cluster *RaftCluster - stage unsafeRecoveryStage + cluster cluster + stage stage // the round of recovery, which is an increasing number to identify the reports of each round step uint64 failedStores map[uint64]struct{} @@ -120,8 +132,9 @@ type unsafeRecoveryController struct { storeRecoveryPlans map[uint64]*pdpb.RecoveryPlan // accumulated output for the whole recovery process - output []StageOutput - affectedTableIDs map[int64]struct{} + output []StageOutput + // exposed to the outside for testing + AffectedTableIDs map[int64]struct{} affectedMetaRegions map[uint64]struct{} err error } @@ -134,16 +147,17 @@ type StageOutput struct { Details []string `json:"details,omitempty"` } -func newUnsafeRecoveryController(cluster *RaftCluster) *unsafeRecoveryController { - u := &unsafeRecoveryController{ +// NewController creates a new Controller. +func NewController(cluster cluster) *Controller { + u := &Controller{ cluster: cluster, } u.reset() return u } -func (u *unsafeRecoveryController) reset() { - u.stage = idle +func (u *Controller) reset() { + u.stage = Idle u.step = 0 u.failedStores = make(map[uint64]struct{}) u.storeReports = make(map[uint64]*pdpb.StoreReport) @@ -151,29 +165,29 @@ func (u *unsafeRecoveryController) reset() { u.storePlanExpires = make(map[uint64]time.Time) u.storeRecoveryPlans = make(map[uint64]*pdpb.RecoveryPlan) u.output = make([]StageOutput, 0) - u.affectedTableIDs = make(map[int64]struct{}, 0) + u.AffectedTableIDs = make(map[int64]struct{}, 0) u.affectedMetaRegions = make(map[uint64]struct{}, 0) u.err = nil } // IsRunning returns whether there is ongoing unsafe recovery process. If yes, further unsafe // recovery requests, schedulers, checkers, AskSplit and AskBatchSplit requests are blocked. -func (u *unsafeRecoveryController) IsRunning() bool { +func (u *Controller) IsRunning() bool { u.RLock() defer u.RUnlock() - return u.isRunningLocked() + return isRunning(u.stage) } -func (u *unsafeRecoveryController) isRunningLocked() bool { - return u.stage != idle && u.stage != finished && u.stage != failed +func isRunning(s stage) bool { + return s != Idle && s != Finished && s != Failed } -// RemoveFailedStores removes failed stores from the cluster. -func (u *unsafeRecoveryController) RemoveFailedStores(failedStores map[uint64]struct{}, timeout uint64, autoDetect bool) error { +// RemoveFailedStores removes Failed stores from the cluster. +func (u *Controller) RemoveFailedStores(failedStores map[uint64]struct{}, timeout uint64, autoDetect bool) error { u.Lock() defer u.Unlock() - if u.isRunningLocked() { + if isRunning(u.stage) { return errs.ErrUnsafeRecoveryIsRunning.FastGenByArgs() } @@ -213,29 +227,29 @@ func (u *unsafeRecoveryController) RemoveFailedStores(failedStores map[uint64]st u.timeout = time.Now().Add(time.Duration(timeout) * time.Second) u.failedStores = failedStores u.autoDetect = autoDetect - u.changeStage(collectReport) + u.changeStage(CollectReport) return nil } // Show returns the current status of ongoing unsafe recover operation. -func (u *unsafeRecoveryController) Show() []StageOutput { +func (u *Controller) Show() []StageOutput { u.Lock() defer u.Unlock() - if u.stage == idle { + if u.stage == Idle { return []StageOutput{{Info: "No on-going recovery."}} } if err := u.checkTimeout(); err != nil { - u.HandleErr(err) + u.handleErr(err) } status := u.output - if u.stage != finished && u.stage != failed { + if u.stage != Finished && u.stage != Failed { status = append(status, u.getReportStatus()) } return status } -func (u *unsafeRecoveryController) getReportStatus() StageOutput { +func (u *Controller) getReportStatus() StageOutput { var status StageOutput status.Time = time.Now().Format("2006-01-02 15:04:05.000") if u.numStoresReported != len(u.storeReports) { @@ -262,8 +276,8 @@ func (u *unsafeRecoveryController) getReportStatus() StageOutput { return status } -func (u *unsafeRecoveryController) checkTimeout() error { - if u.stage == finished || u.stage == failed { +func (u *Controller) checkTimeout() error { + if u.stage == Finished || u.stage == Failed { return nil } @@ -273,36 +287,37 @@ func (u *unsafeRecoveryController) checkTimeout() error { return nil } -func (u *unsafeRecoveryController) HandleErr(err error) bool { +// handleErr handles the error occurred during the unsafe recovery process. +func (u *Controller) handleErr(err error) bool { // Keep the earliest error. if u.err == nil { u.err = err } - if u.stage == exitForceLeader { - // We already tried to exit force leader, and it still failed. - // We turn into failed stage directly. TiKV will step down force leader + if u.stage == ExitForceLeader { + // We already tried to exit force leader, and it still Failed. + // We turn into Failed stage directly. TiKV will step down force leader // automatically after being for a long time. - u.changeStage(failed) + u.changeStage(Failed) return true } // When encountering an error for the first time, we will try to exit force - // leader before turning into failed stage to avoid the leaking force leaders + // leader before turning into Failed stage to avoid the leaking force leaders // blocks reads and writes. u.storePlanExpires = make(map[uint64]time.Time) u.storeRecoveryPlans = make(map[uint64]*pdpb.RecoveryPlan) u.timeout = time.Now().Add(storeRequestInterval * 2) // empty recovery plan would trigger exit force leader - u.changeStage(exitForceLeader) + u.changeStage(ExitForceLeader) return false } // HandleStoreHeartbeat handles the store heartbeat requests and checks whether the stores need to // send detailed report back. -func (u *unsafeRecoveryController) HandleStoreHeartbeat(heartbeat *pdpb.StoreHeartbeatRequest, resp *pdpb.StoreHeartbeatResponse) { +func (u *Controller) HandleStoreHeartbeat(heartbeat *pdpb.StoreHeartbeatRequest, resp *pdpb.StoreHeartbeatResponse) { u.Lock() defer u.Unlock() - if !u.isRunningLocked() { + if !isRunning(u.stage) { // no recovery in progress, do nothing return } @@ -312,7 +327,7 @@ func (u *unsafeRecoveryController) HandleStoreHeartbeat(heartbeat *pdpb.StoreHea return false, err } - allCollected, err := u.collectReport(heartbeat) + allCollected, err := u.CollectReport(heartbeat) if err != nil { return false, err } @@ -328,13 +343,13 @@ func (u *unsafeRecoveryController) HandleStoreHeartbeat(heartbeat *pdpb.StoreHea return false, nil }() - if done || (err != nil && u.HandleErr(err)) { + if done || (err != nil && u.handleErr(err)) { return } u.dispatchPlan(heartbeat, resp) } -func (u *unsafeRecoveryController) generatePlan(newestRegionTree *regionTree, peersMap map[uint64][]*regionItem) (bool, error) { +func (u *Controller) generatePlan(newestRegionTree *regionTree, peersMap map[uint64][]*regionItem) (bool, error) { // clean up previous plan u.storePlanExpires = make(map[uint64]time.Time) u.storeRecoveryPlans = make(map[uint64]*pdpb.RecoveryPlan) @@ -345,57 +360,57 @@ func (u *unsafeRecoveryController) generatePlan(newestRegionTree *regionTree, pe var err error for { switch stage { - case collectReport: + case CollectReport: fallthrough - case tombstoneTiFlashLearner: + case TombstoneTiFlashLearner: if hasPlan, err = u.generateTombstoneTiFlashLearnerPlan(newestRegionTree, peersMap); hasPlan && err == nil { - u.changeStage(tombstoneTiFlashLearner) + u.changeStage(TombstoneTiFlashLearner) break } if err != nil { break } fallthrough - case forceLeaderForCommitMerge: + case ForceLeaderForCommitMerge: if hasPlan, err = u.generateForceLeaderPlan(newestRegionTree, peersMap, true); hasPlan && err == nil { - u.changeStage(forceLeaderForCommitMerge) + u.changeStage(ForceLeaderForCommitMerge) break } if err != nil { break } fallthrough - case forceLeader: + case ForceLeader: if hasPlan, err = u.generateForceLeaderPlan(newestRegionTree, peersMap, false); hasPlan && err == nil { - u.changeStage(forceLeader) + u.changeStage(ForceLeader) break } if err != nil { break } fallthrough - case demoteFailedVoter: + case DemoteFailedVoter: if hasPlan = u.generateDemoteFailedVoterPlan(newestRegionTree, peersMap); hasPlan { - u.changeStage(demoteFailedVoter) + u.changeStage(DemoteFailedVoter) break } else if !reCheck { reCheck = true - stage = tombstoneTiFlashLearner + stage = TombstoneTiFlashLearner continue } fallthrough - case createEmptyRegion: + case CreateEmptyRegion: if hasPlan, err = u.generateCreateEmptyRegionPlan(newestRegionTree, peersMap); hasPlan && err == nil { - u.changeStage(createEmptyRegion) + u.changeStage(CreateEmptyRegion) break } if err != nil { break } fallthrough - case exitForceLeader: + case ExitForceLeader: if hasPlan = u.generateExitForceLeaderPlan(); hasPlan { - u.changeStage(exitForceLeader) + u.changeStage(ExitForceLeader) } default: panic("unreachable") @@ -405,9 +420,9 @@ func (u *unsafeRecoveryController) generatePlan(newestRegionTree *regionTree, pe if err == nil && !hasPlan { if u.err != nil { - u.changeStage(failed) + u.changeStage(Failed) } else { - u.changeStage(finished) + u.changeStage(Finished) } return true, nil } @@ -415,7 +430,7 @@ func (u *unsafeRecoveryController) generatePlan(newestRegionTree *regionTree, pe } // It dispatches recovery plan if any. -func (u *unsafeRecoveryController) dispatchPlan(heartbeat *pdpb.StoreHeartbeatRequest, resp *pdpb.StoreHeartbeatResponse) { +func (u *Controller) dispatchPlan(heartbeat *pdpb.StoreHeartbeatRequest, resp *pdpb.StoreHeartbeatResponse) { storeID := heartbeat.Stats.StoreId now := time.Now() @@ -436,11 +451,11 @@ func (u *unsafeRecoveryController) dispatchPlan(heartbeat *pdpb.StoreHeartbeatRe } } -// It collects and checks if store reports have been fully collected. -func (u *unsafeRecoveryController) collectReport(heartbeat *pdpb.StoreHeartbeatRequest) (bool, error) { +// CollectReport collects and checks if store reports have been fully collected. +func (u *Controller) CollectReport(heartbeat *pdpb.StoreHeartbeatRequest) (bool, error) { storeID := heartbeat.Stats.StoreId if _, isFailedStore := u.failedStores[storeID]; isFailedStore { - return false, errors.Errorf("Receive heartbeat from failed store %d", storeID) + return false, errors.Errorf("Receive heartbeat from Failed store %d", storeID) } if heartbeat.StoreReport == nil { @@ -467,25 +482,30 @@ func (u *unsafeRecoveryController) collectReport(heartbeat *pdpb.StoreHeartbeatR return false, nil } -// Gets the stage of the current unsafe recovery. -func (u *unsafeRecoveryController) GetStage() unsafeRecoveryStage { +// GetStage gets the stage of the current unsafe recovery. +func (u *Controller) GetStage() stage { u.RLock() defer u.RUnlock() return u.stage } -func (u *unsafeRecoveryController) changeStage(stage unsafeRecoveryStage) { +func (u *Controller) changeStage(stage stage) { u.stage = stage + // Halt and resume the scheduling once the running state changed. + running := isRunning(stage) + if opt := u.cluster.GetPersistOptions(); opt.IsSchedulingHalted() != running { + opt.SetHaltScheduling(running, "online-unsafe-recovery") + } var output StageOutput output.Time = time.Now().Format("2006-01-02 15:04:05.000") switch u.stage { - case idle: - case collectReport: + case Idle: + case CollectReport: // TODO: clean up existing operators output.Info = "Unsafe recovery enters collect report stage" if u.autoDetect { - output.Details = append(output.Details, "auto detect mode with no specified failed stores") + output.Details = append(output.Details, "auto detect mode with no specified Failed stores") } else { stores := "" count := 0 @@ -496,40 +516,40 @@ func (u *unsafeRecoveryController) changeStage(stage unsafeRecoveryStage) { stores += ", " } } - output.Details = append(output.Details, fmt.Sprintf("failed stores %s", stores)) + output.Details = append(output.Details, fmt.Sprintf("Failed stores %s", stores)) } - case tombstoneTiFlashLearner: + case TombstoneTiFlashLearner: output.Info = "Unsafe recovery enters tombstone TiFlash learner stage" output.Actions = u.getTombstoneTiFlashLearnerDigest() - case forceLeaderForCommitMerge: + case ForceLeaderForCommitMerge: output.Info = "Unsafe recovery enters force leader for commit merge stage" output.Actions = u.getForceLeaderPlanDigest() - case forceLeader: + case ForceLeader: output.Info = "Unsafe recovery enters force leader stage" output.Actions = u.getForceLeaderPlanDigest() - case demoteFailedVoter: - output.Info = "Unsafe recovery enters demote failed voter stage" + case DemoteFailedVoter: + output.Info = "Unsafe recovery enters demote Failed voter stage" output.Actions = u.getDemoteFailedVoterPlanDigest() - case createEmptyRegion: + case CreateEmptyRegion: output.Info = "Unsafe recovery enters create empty region stage" output.Actions = u.getCreateEmptyRegionPlanDigest() - case exitForceLeader: + case ExitForceLeader: output.Info = "Unsafe recovery enters exit force leader stage" if u.err != nil { output.Details = append(output.Details, fmt.Sprintf("triggered by error: %v", u.err.Error())) } - case finished: + case Finished: if u.step > 1 { // == 1 means no operation has done, no need to invalid cache u.cluster.DropCacheAllRegion() } - output.Info = "Unsafe recovery finished" + output.Info = "Unsafe recovery Finished" output.Details = u.getAffectedTableDigest() u.storePlanExpires = make(map[uint64]time.Time) u.storeRecoveryPlans = make(map[uint64]*pdpb.RecoveryPlan) - case failed: - output.Info = fmt.Sprintf("Unsafe recovery failed: %v", u.err) + case Failed: + output.Info = fmt.Sprintf("Unsafe recovery Failed: %v", u.err) output.Details = u.getAffectedTableDigest() if u.numStoresReported != len(u.storeReports) { // in collecting reports, print out which stores haven't reported yet @@ -556,7 +576,7 @@ func (u *unsafeRecoveryController) changeStage(stage unsafeRecoveryStage) { u.step += 1 } -func (u *unsafeRecoveryController) getForceLeaderPlanDigest() map[string][]string { +func (u *Controller) getForceLeaderPlanDigest() map[string][]string { outputs := make(map[string][]string) for storeID, plan := range u.storeRecoveryPlans { forceLeaders := plan.GetForceLeader() @@ -574,7 +594,7 @@ func (u *unsafeRecoveryController) getForceLeaderPlanDigest() map[string][]strin return outputs } -func (u *unsafeRecoveryController) getDemoteFailedVoterPlanDigest() map[string][]string { +func (u *Controller) getDemoteFailedVoterPlanDigest() map[string][]string { outputs := make(map[string][]string) for storeID, plan := range u.storeRecoveryPlans { if len(plan.GetDemotes()) == 0 && len(plan.GetTombstones()) == 0 { @@ -596,7 +616,7 @@ func (u *unsafeRecoveryController) getDemoteFailedVoterPlanDigest() map[string][ return outputs } -func (u *unsafeRecoveryController) getTombstoneTiFlashLearnerDigest() map[string][]string { +func (u *Controller) getTombstoneTiFlashLearnerDigest() map[string][]string { outputs := make(map[string][]string) for storeID, plan := range u.storeRecoveryPlans { if len(plan.GetTombstones()) == 0 { @@ -611,7 +631,7 @@ func (u *unsafeRecoveryController) getTombstoneTiFlashLearnerDigest() map[string return outputs } -func (u *unsafeRecoveryController) getCreateEmptyRegionPlanDigest() map[string][]string { +func (u *Controller) getCreateEmptyRegionPlanDigest() map[string][]string { outputs := make(map[string][]string) for storeID, plan := range u.storeRecoveryPlans { if plan.GetCreates() == nil { @@ -630,7 +650,7 @@ func (u *unsafeRecoveryController) getCreateEmptyRegionPlanDigest() map[string][ return outputs } -func (u *unsafeRecoveryController) getAffectedTableDigest() []string { +func (u *Controller) getAffectedTableDigest() []string { var details []string if len(u.affectedMetaRegions) != 0 { regions := "" @@ -639,9 +659,9 @@ func (u *unsafeRecoveryController) getAffectedTableDigest() []string { } details = append(details, "affected meta regions: "+strings.Trim(regions, ", ")) } - if len(u.affectedTableIDs) != 0 { + if len(u.AffectedTableIDs) != 0 { tables := "" - for t := range u.affectedTableIDs { + for t := range u.AffectedTableIDs { tables += fmt.Sprintf("%d, ", t) } details = append(details, "affected table ids: "+strings.Trim(tables, ", ")) @@ -649,16 +669,16 @@ func (u *unsafeRecoveryController) getAffectedTableDigest() []string { return details } -func (u *unsafeRecoveryController) recordAffectedRegion(region *metapb.Region) { +func (u *Controller) recordAffectedRegion(region *metapb.Region) { isMeta, tableID := codec.Key(region.StartKey).MetaOrTable() if isMeta { u.affectedMetaRegions[region.GetId()] = struct{}{} } else if tableID != 0 { - u.affectedTableIDs[tableID] = struct{}{} + u.AffectedTableIDs[tableID] = struct{}{} } } -func (u *unsafeRecoveryController) isFailed(peer *metapb.Peer) bool { +func (u *Controller) isFailed(peer *metapb.Peer) bool { _, isFailed := u.failedStores[peer.StoreId] _, isLive := u.storeReports[peer.StoreId] if isFailed || (u.autoDetect && !isLive) { @@ -667,7 +687,7 @@ func (u *unsafeRecoveryController) isFailed(peer *metapb.Peer) bool { return false } -func (u *unsafeRecoveryController) canElectLeader(region *metapb.Region, onlyIncoming bool) bool { +func (u *Controller) canElectLeader(region *metapb.Region, onlyIncoming bool) bool { hasQuorum := func(voters []*metapb.Peer) bool { numFailedVoters := 0 numLiveVoters := 0 @@ -698,7 +718,7 @@ func (u *unsafeRecoveryController) canElectLeader(region *metapb.Region, onlyInc return hasQuorum(incomingVoters) && (onlyIncoming || hasQuorum(outgoingVoters)) } -func (u *unsafeRecoveryController) getFailedPeers(region *metapb.Region) []*metapb.Peer { +func (u *Controller) getFailedPeers(region *metapb.Region) []*metapb.Peer { // if it can form a quorum after exiting the joint state, then no need to demotes any peer if u.canElectLeader(region, true) { return nil @@ -706,9 +726,6 @@ func (u *unsafeRecoveryController) getFailedPeers(region *metapb.Region) []*meta var failedPeers []*metapb.Peer for _, peer := range region.Peers { - if peer.Role == metapb.PeerRole_Learner || peer.Role == metapb.PeerRole_DemotingVoter { - continue - } if u.isFailed(peer) { failedPeers = append(failedPeers, peer) } @@ -747,7 +764,7 @@ func (r *regionItem) IsEpochStale(other *regionItem) bool { return re.GetVersion() < oe.GetVersion() || (re.GetVersion() == oe.GetVersion() && re.GetConfVer() < oe.GetConfVer()) } -func (r *regionItem) IsRaftStale(origin *regionItem, u *unsafeRecoveryController) bool { +func (r *regionItem) IsRaftStale(origin *regionItem, u *Controller) bool { cmps := []func(a, b *regionItem) int{ func(a, b *regionItem) int { return int(a.report.GetRaftState().GetHardState().GetTerm()) - int(b.report.GetRaftState().GetHardState().GetTerm()) @@ -877,14 +894,14 @@ func (t *regionTree) insert(item *regionItem) (bool, error) { return true, nil } -func (u *unsafeRecoveryController) getRecoveryPlan(storeID uint64) *pdpb.RecoveryPlan { +func (u *Controller) getRecoveryPlan(storeID uint64) *pdpb.RecoveryPlan { if _, exists := u.storeRecoveryPlans[storeID]; !exists { u.storeRecoveryPlans[storeID] = &pdpb.RecoveryPlan{} } return u.storeRecoveryPlans[storeID] } -func (u *unsafeRecoveryController) buildUpFromReports() (*regionTree, map[uint64][]*regionItem, error) { +func (u *Controller) buildUpFromReports() (*regionTree, map[uint64][]*regionItem, error) { peersMap := make(map[uint64][]*regionItem) // Go through all the peer reports to build up the newest region tree for storeID, storeReport := range u.storeReports { @@ -925,7 +942,7 @@ func (u *unsafeRecoveryController) buildUpFromReports() (*regionTree, map[uint64 return newestRegionTree, peersMap, nil } -func (u *unsafeRecoveryController) selectLeader(peersMap map[uint64][]*regionItem, region *metapb.Region) *regionItem { +func (u *Controller) selectLeader(peersMap map[uint64][]*regionItem, region *metapb.Region) *regionItem { var leader *regionItem for _, peer := range peersMap[region.GetId()] { if leader == nil || leader.IsRaftStale(peer, u) { @@ -935,7 +952,7 @@ func (u *unsafeRecoveryController) selectLeader(peersMap map[uint64][]*regionIte return leader } -func (u *unsafeRecoveryController) generateTombstoneTiFlashLearnerPlan(newestRegionTree *regionTree, peersMap map[uint64][]*regionItem) (bool, error) { +func (u *Controller) generateTombstoneTiFlashLearnerPlan(newestRegionTree *regionTree, peersMap map[uint64][]*regionItem) (bool, error) { if u.err != nil { return false, nil } @@ -964,7 +981,7 @@ func (u *unsafeRecoveryController) generateTombstoneTiFlashLearnerPlan(newestReg return hasPlan, err } -func (u *unsafeRecoveryController) generateForceLeaderPlan(newestRegionTree *regionTree, peersMap map[uint64][]*regionItem, forCommitMerge bool) (bool, error) { +func (u *Controller) generateForceLeaderPlan(newestRegionTree *regionTree, peersMap map[uint64][]*regionItem, forCommitMerge bool) (bool, error) { if u.err != nil { return false, nil } @@ -981,7 +998,7 @@ func (u *unsafeRecoveryController) generateForceLeaderPlan(newestRegionTree *reg var err error // Check the regions in newest Region Tree to see if it can still elect leader - // considering the failed stores + // considering the Failed stores newestRegionTree.tree.Ascend(func(item *regionItem) bool { report := item.report region := item.Region() @@ -1014,7 +1031,7 @@ func (u *unsafeRecoveryController) generateForceLeaderPlan(newestRegionTree *reg } } if u.autoDetect { - // For auto detect, the failedStores is empty. So need to add the detected failed store to the list + // For auto detect, the failedStores is empty. So need to add the detected Failed store to the list for _, peer := range u.getFailedPeers(leader.Region()) { found := false for _, store := range storeRecoveryPlan.ForceLeader.FailedStores { @@ -1049,7 +1066,7 @@ func (u *unsafeRecoveryController) generateForceLeaderPlan(newestRegionTree *reg return hasPlan, err } -func (u *unsafeRecoveryController) generateDemoteFailedVoterPlan(newestRegionTree *regionTree, peersMap map[uint64][]*regionItem) bool { +func (u *Controller) generateDemoteFailedVoterPlan(newestRegionTree *regionTree, peersMap map[uint64][]*regionItem) bool { if u.err != nil { return false } @@ -1067,7 +1084,7 @@ func (u *unsafeRecoveryController) generateDemoteFailedVoterPlan(newestRegionTre } // Check the regions in newest Region Tree to see if it can still elect leader - // considering the failed stores + // considering the Failed stores newestRegionTree.tree.Ascend(func(item *regionItem) bool { region := item.Region() if !u.canElectLeader(region, false) { @@ -1107,18 +1124,18 @@ func (u *unsafeRecoveryController) generateDemoteFailedVoterPlan(newestRegionTre return hasPlan } -func (u *unsafeRecoveryController) generateCreateEmptyRegionPlan(newestRegionTree *regionTree, peersMap map[uint64][]*regionItem) (bool, error) { +func (u *Controller) generateCreateEmptyRegionPlan(newestRegionTree *regionTree, peersMap map[uint64][]*regionItem) (bool, error) { if u.err != nil { return false, nil } hasPlan := false createRegion := func(startKey, endKey []byte, storeID uint64) (*metapb.Region, error) { - regionID, err := u.cluster.GetAllocator().Alloc() + regionID, err := u.cluster.AllocID() if err != nil { return nil, err } - peerID, err := u.cluster.GetAllocator().Alloc() + peerID, err := u.cluster.AllocID() if err != nil { return nil, err } @@ -1214,7 +1231,7 @@ func (u *unsafeRecoveryController) generateCreateEmptyRegionPlan(newestRegionTre return hasPlan, nil } -func (u *unsafeRecoveryController) generateExitForceLeaderPlan() bool { +func (u *Controller) generateExitForceLeaderPlan() bool { hasPlan := false for storeID, storeReport := range u.storeReports { for _, peerReport := range storeReport.PeerReports { diff --git a/server/cluster/unsafe_recovery_controller_test.go b/pkg/unsaferecovery/unsafe_recovery_controller_test.go similarity index 83% rename from server/cluster/unsafe_recovery_controller_test.go rename to pkg/unsaferecovery/unsafe_recovery_controller_test.go index 6ef20fdb6b7..44c4e4a7b4d 100644 --- a/server/cluster/unsafe_recovery_controller_test.go +++ b/pkg/unsaferecovery/unsafe_recovery_controller_test.go @@ -12,10 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. -package cluster +// change the package to avoid import cycle +package unsaferecovery import ( "context" + "fmt" "testing" "time" @@ -26,9 +28,10 @@ import ( "github.com/stretchr/testify/require" "github.com/tikv/pd/pkg/codec" "github.com/tikv/pd/pkg/core" - "github.com/tikv/pd/pkg/mock/mockid" + "github.com/tikv/pd/pkg/mock/mockcluster" + "github.com/tikv/pd/pkg/mock/mockconfig" + "github.com/tikv/pd/pkg/schedule" "github.com/tikv/pd/pkg/schedule/hbstream" - "github.com/tikv/pd/pkg/storage" ) func newStoreHeartbeat(storeID uint64, report *pdpb.StoreReport) *pdpb.StoreHeartbeatRequest { @@ -162,7 +165,7 @@ func applyRecoveryPlan(re *require.Assertions, storeID uint64, storeReports map[ } } -func advanceUntilFinished(re *require.Assertions, recoveryController *unsafeRecoveryController, reports map[uint64]*pdpb.StoreReport) { +func advanceUntilFinished(re *require.Assertions, recoveryController *Controller, reports map[uint64]*pdpb.StoreReport) { retry := 0 for { @@ -173,10 +176,10 @@ func advanceUntilFinished(re *require.Assertions, recoveryController *unsafeReco recoveryController.HandleStoreHeartbeat(req, resp) applyRecoveryPlan(re, storeID, reports, resp) } - if recoveryController.GetStage() == finished { + if recoveryController.GetStage() == Finished { break - } else if recoveryController.GetStage() == failed { - panic("failed to recovery") + } else if recoveryController.GetStage() == Failed { + panic("Failed to recovery") } else if retry >= 10 { panic("retry timeout") } @@ -189,14 +192,14 @@ func TestFinished(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - _, opt, _ := newTestScheduleConfig() - cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend(), core.NewBasicCluster()) - cluster.coordinator = newCoordinator(ctx, cluster, hbstream.NewTestHeartbeatStreams(ctx, cluster.meta.GetId(), cluster, true)) - cluster.coordinator.run() + opts := mockconfig.NewTestOptions() + cluster := mockcluster.NewCluster(ctx, opts) + coordinator := schedule.NewCoordinator(ctx, cluster, hbstream.NewTestHeartbeatStreams(ctx, cluster.ID, cluster, true)) + coordinator.Run() for _, store := range newTestStores(3, "6.0.0") { - re.NoError(cluster.PutStore(store.GetMeta())) + cluster.PutStore(store) } - recoveryController := newUnsafeRecoveryController(cluster) + recoveryController := NewController(cluster) re.NoError(recoveryController.RemoveFailedStores(map[uint64]struct{}{ 2: {}, 3: {}, @@ -214,7 +217,7 @@ func TestFinished(t *testing.T) { {Id: 11, StoreId: 1}, {Id: 21, StoreId: 2}, {Id: 31, StoreId: 3}}}}}, }}, } - re.Equal(collectReport, recoveryController.GetStage()) + re.Equal(CollectReport, recoveryController.GetStage()) for storeID := range reports { req := newStoreHeartbeat(storeID, nil) resp := &pdpb.StoreHeartbeatResponse{} @@ -240,7 +243,7 @@ func TestFinished(t *testing.T) { re.NotNil(resp.RecoveryPlan.ForceLeader.FailedStores) applyRecoveryPlan(re, storeID, reports, resp) } - re.Equal(forceLeader, recoveryController.GetStage()) + re.Equal(ForceLeader, recoveryController.GetStage()) for storeID, report := range reports { req := newStoreHeartbeat(storeID, report) @@ -251,7 +254,7 @@ func TestFinished(t *testing.T) { re.Len(resp.RecoveryPlan.Demotes, 1) applyRecoveryPlan(re, storeID, reports, resp) } - re.Equal(demoteFailedVoter, recoveryController.GetStage()) + re.Equal(DemoteFailedVoter, recoveryController.GetStage()) for storeID, report := range reports { req := newStoreHeartbeat(storeID, report) req.StoreReport = report @@ -261,7 +264,7 @@ func TestFinished(t *testing.T) { // remove the two failed peers applyRecoveryPlan(re, storeID, reports, resp) } - re.Equal(finished, recoveryController.GetStage()) + re.Equal(Finished, recoveryController.GetStage()) } func TestFailed(t *testing.T) { @@ -269,14 +272,14 @@ func TestFailed(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - _, opt, _ := newTestScheduleConfig() - cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend(), core.NewBasicCluster()) - cluster.coordinator = newCoordinator(ctx, cluster, hbstream.NewTestHeartbeatStreams(ctx, cluster.meta.GetId(), cluster, true)) - cluster.coordinator.run() + opts := mockconfig.NewTestOptions() + cluster := mockcluster.NewCluster(ctx, opts) + coordinator := schedule.NewCoordinator(ctx, cluster, hbstream.NewTestHeartbeatStreams(ctx, cluster.ID, cluster, true)) + coordinator.Run() for _, store := range newTestStores(3, "6.0.0") { - re.NoError(cluster.PutStore(store.GetMeta())) + cluster.PutStore(store) } - recoveryController := newUnsafeRecoveryController(cluster) + recoveryController := NewController(cluster) re.NoError(recoveryController.RemoveFailedStores(map[uint64]struct{}{ 2: {}, 3: {}, @@ -294,7 +297,7 @@ func TestFailed(t *testing.T) { {Id: 11, StoreId: 1}, {Id: 21, StoreId: 2}, {Id: 31, StoreId: 3}}}}}, }}, } - re.Equal(collectReport, recoveryController.GetStage()) + re.Equal(CollectReport, recoveryController.GetStage()) // require peer report for storeID := range reports { req := newStoreHeartbeat(storeID, nil) @@ -319,7 +322,7 @@ func TestFailed(t *testing.T) { re.NotNil(resp.RecoveryPlan.ForceLeader.FailedStores) applyRecoveryPlan(re, storeID, reports, resp) } - re.Equal(forceLeader, recoveryController.GetStage()) + re.Equal(ForceLeader, recoveryController.GetStage()) for storeID, report := range reports { req := newStoreHeartbeat(storeID, report) @@ -330,13 +333,13 @@ func TestFailed(t *testing.T) { re.Len(resp.RecoveryPlan.Demotes, 1) applyRecoveryPlan(re, storeID, reports, resp) } - re.Equal(demoteFailedVoter, recoveryController.GetStage()) + re.Equal(DemoteFailedVoter, recoveryController.GetStage()) - // received heartbeat from failed store, abort + // received heartbeat from Failed store, abort req := newStoreHeartbeat(2, nil) resp := &pdpb.StoreHeartbeatResponse{} recoveryController.HandleStoreHeartbeat(req, resp) - re.Equal(exitForceLeader, recoveryController.GetStage()) + re.Equal(ExitForceLeader, recoveryController.GetStage()) for storeID, report := range reports { req := newStoreHeartbeat(storeID, report) @@ -354,7 +357,7 @@ func TestFailed(t *testing.T) { recoveryController.HandleStoreHeartbeat(req, resp) applyRecoveryPlan(re, storeID, reports, resp) } - re.Equal(failed, recoveryController.GetStage()) + re.Equal(Failed, recoveryController.GetStage()) } func TestForceLeaderFail(t *testing.T) { @@ -362,14 +365,14 @@ func TestForceLeaderFail(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - _, opt, _ := newTestScheduleConfig() - cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend(), core.NewBasicCluster()) - cluster.coordinator = newCoordinator(ctx, cluster, hbstream.NewTestHeartbeatStreams(ctx, cluster.meta.GetId(), cluster, true)) - cluster.coordinator.run() + opts := mockconfig.NewTestOptions() + cluster := mockcluster.NewCluster(ctx, opts) + coordinator := schedule.NewCoordinator(ctx, cluster, hbstream.NewTestHeartbeatStreams(ctx, cluster.ID, cluster, true)) + coordinator.Run() for _, store := range newTestStores(4, "6.0.0") { - re.NoError(cluster.PutStore(store.GetMeta())) + cluster.PutStore(store) } - recoveryController := newUnsafeRecoveryController(cluster) + recoveryController := NewController(cluster) re.NoError(recoveryController.RemoveFailedStores(map[uint64]struct{}{ 3: {}, 4: {}, @@ -414,7 +417,7 @@ func TestForceLeaderFail(t *testing.T) { resp2 := &pdpb.StoreHeartbeatResponse{} req2.StoreReport.Step = 1 recoveryController.HandleStoreHeartbeat(req2, resp2) - re.Equal(forceLeader, recoveryController.GetStage()) + re.Equal(ForceLeader, recoveryController.GetStage()) recoveryController.HandleStoreHeartbeat(req1, resp1) // force leader on store 1 succeed @@ -426,7 +429,7 @@ func TestForceLeaderFail(t *testing.T) { // force leader should retry on store 2 recoveryController.HandleStoreHeartbeat(req1, resp1) recoveryController.HandleStoreHeartbeat(req2, resp2) - re.Equal(forceLeader, recoveryController.GetStage()) + re.Equal(ForceLeader, recoveryController.GetStage()) recoveryController.HandleStoreHeartbeat(req1, resp1) // force leader succeed this time @@ -434,7 +437,7 @@ func TestForceLeaderFail(t *testing.T) { applyRecoveryPlan(re, 2, reports, resp2) recoveryController.HandleStoreHeartbeat(req1, resp1) recoveryController.HandleStoreHeartbeat(req2, resp2) - re.Equal(demoteFailedVoter, recoveryController.GetStage()) + re.Equal(DemoteFailedVoter, recoveryController.GetStage()) } func TestAffectedTableID(t *testing.T) { @@ -442,14 +445,14 @@ func TestAffectedTableID(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - _, opt, _ := newTestScheduleConfig() - cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend(), core.NewBasicCluster()) - cluster.coordinator = newCoordinator(ctx, cluster, hbstream.NewTestHeartbeatStreams(ctx, cluster.meta.GetId(), cluster, true)) - cluster.coordinator.run() + opts := mockconfig.NewTestOptions() + cluster := mockcluster.NewCluster(ctx, opts) + coordinator := schedule.NewCoordinator(ctx, cluster, hbstream.NewTestHeartbeatStreams(ctx, cluster.ID, cluster, true)) + coordinator.Run() for _, store := range newTestStores(3, "6.0.0") { - re.NoError(cluster.PutStore(store.GetMeta())) + cluster.PutStore(store) } - recoveryController := newUnsafeRecoveryController(cluster) + recoveryController := NewController(cluster) re.NoError(recoveryController.RemoveFailedStores(map[uint64]struct{}{ 2: {}, 3: {}, @@ -473,8 +476,8 @@ func TestAffectedTableID(t *testing.T) { advanceUntilFinished(re, recoveryController, reports) - re.Len(recoveryController.affectedTableIDs, 1) - _, exists := recoveryController.affectedTableIDs[6] + re.Len(recoveryController.AffectedTableIDs, 1) + _, exists := recoveryController.AffectedTableIDs[6] re.True(exists) } @@ -483,14 +486,14 @@ func TestForceLeaderForCommitMerge(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - _, opt, _ := newTestScheduleConfig() - cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend(), core.NewBasicCluster()) - cluster.coordinator = newCoordinator(ctx, cluster, hbstream.NewTestHeartbeatStreams(ctx, cluster.meta.GetId(), cluster, true)) - cluster.coordinator.run() + opts := mockconfig.NewTestOptions() + cluster := mockcluster.NewCluster(ctx, opts) + coordinator := schedule.NewCoordinator(ctx, cluster, hbstream.NewTestHeartbeatStreams(ctx, cluster.ID, cluster, true)) + coordinator.Run() for _, store := range newTestStores(3, "6.0.0") { - re.NoError(cluster.PutStore(store.GetMeta())) + cluster.PutStore(store) } - recoveryController := newUnsafeRecoveryController(cluster) + recoveryController := NewController(cluster) re.NoError(recoveryController.RemoveFailedStores(map[uint64]struct{}{ 2: {}, 3: {}, @@ -529,7 +532,7 @@ func TestForceLeaderForCommitMerge(t *testing.T) { resp := &pdpb.StoreHeartbeatResponse{} req.StoreReport.Step = 1 recoveryController.HandleStoreHeartbeat(req, resp) - re.Equal(forceLeaderForCommitMerge, recoveryController.GetStage()) + re.Equal(ForceLeaderForCommitMerge, recoveryController.GetStage()) // force leader on regions of commit merge first re.NotNil(resp.RecoveryPlan) @@ -540,7 +543,7 @@ func TestForceLeaderForCommitMerge(t *testing.T) { applyRecoveryPlan(re, 1, reports, resp) recoveryController.HandleStoreHeartbeat(req, resp) - re.Equal(forceLeader, recoveryController.GetStage()) + re.Equal(ForceLeader, recoveryController.GetStage()) // force leader on the rest regions re.NotNil(resp.RecoveryPlan) @@ -551,7 +554,7 @@ func TestForceLeaderForCommitMerge(t *testing.T) { applyRecoveryPlan(re, 1, reports, resp) recoveryController.HandleStoreHeartbeat(req, resp) - re.Equal(demoteFailedVoter, recoveryController.GetStage()) + re.Equal(DemoteFailedVoter, recoveryController.GetStage()) } func TestAutoDetectMode(t *testing.T) { @@ -559,14 +562,14 @@ func TestAutoDetectMode(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - _, opt, _ := newTestScheduleConfig() - cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend(), core.NewBasicCluster()) - cluster.coordinator = newCoordinator(ctx, cluster, hbstream.NewTestHeartbeatStreams(ctx, cluster.meta.GetId(), cluster, true)) - cluster.coordinator.run() + opts := mockconfig.NewTestOptions() + cluster := mockcluster.NewCluster(ctx, opts) + coordinator := schedule.NewCoordinator(ctx, cluster, hbstream.NewTestHeartbeatStreams(ctx, cluster.ID, cluster, true)) + coordinator.Run() for _, store := range newTestStores(1, "6.0.0") { - re.NoError(cluster.PutStore(store.GetMeta())) + cluster.PutStore(store) } - recoveryController := newUnsafeRecoveryController(cluster) + recoveryController := NewController(cluster) re.NoError(recoveryController.RemoveFailedStores(nil, 60, true)) reports := map[uint64]*pdpb.StoreReport{ @@ -606,19 +609,61 @@ func TestAutoDetectMode(t *testing.T) { } } +// Failed learner replica store should be considered by auto-detect mode. +func TestAutoDetectWithOneLearner(t *testing.T) { + re := require.New(t) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + opts := mockconfig.NewTestOptions() + cluster := mockcluster.NewCluster(ctx, opts) + coordinator := schedule.NewCoordinator(ctx, cluster, hbstream.NewTestHeartbeatStreams(ctx, cluster.ID, cluster, true)) + coordinator.Run() + for _, store := range newTestStores(1, "6.0.0") { + cluster.PutStore(store) + } + recoveryController := NewController(cluster) + re.NoError(recoveryController.RemoveFailedStores(nil, 60, true)) + + storeReport := pdpb.StoreReport{ + PeerReports: []*pdpb.PeerReport{ + { + RaftState: &raft_serverpb.RaftLocalState{LastIndex: 10, HardState: &eraftpb.HardState{Term: 1, Commit: 10}}, + RegionState: &raft_serverpb.RegionLocalState{ + Region: &metapb.Region{ + Id: 1001, + RegionEpoch: &metapb.RegionEpoch{ConfVer: 7, Version: 10}, + Peers: []*metapb.Peer{ + {Id: 11, StoreId: 1}, {Id: 12, StoreId: 2}, {Id: 13, StoreId: 3, Role: metapb.PeerRole_Learner}}}}}, + }, + } + req := newStoreHeartbeat(1, &storeReport) + req.StoreReport.Step = 1 + resp := &pdpb.StoreHeartbeatResponse{} + recoveryController.HandleStoreHeartbeat(req, resp) + hasStore3AsFailedStore := false + for _, failedStore := range resp.RecoveryPlan.ForceLeader.FailedStores { + if failedStore == 3 { + hasStore3AsFailedStore = true + break + } + } + re.True(hasStore3AsFailedStore) +} + func TestOneLearner(t *testing.T) { re := require.New(t) ctx, cancel := context.WithCancel(context.Background()) defer cancel() - _, opt, _ := newTestScheduleConfig() - cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend(), core.NewBasicCluster()) - cluster.coordinator = newCoordinator(ctx, cluster, hbstream.NewTestHeartbeatStreams(ctx, cluster.meta.GetId(), cluster, true)) - cluster.coordinator.run() + opts := mockconfig.NewTestOptions() + cluster := mockcluster.NewCluster(ctx, opts) + coordinator := schedule.NewCoordinator(ctx, cluster, hbstream.NewTestHeartbeatStreams(ctx, cluster.ID, cluster, true)) + coordinator.Run() for _, store := range newTestStores(3, "6.0.0") { - re.NoError(cluster.PutStore(store.GetMeta())) + cluster.PutStore(store) } - recoveryController := newUnsafeRecoveryController(cluster) + recoveryController := NewController(cluster) re.NoError(recoveryController.RemoveFailedStores(map[uint64]struct{}{ 2: {}, 3: {}, @@ -666,17 +711,17 @@ func TestTiflashLearnerPeer(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - _, opt, _ := newTestScheduleConfig() - cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend(), core.NewBasicCluster()) - cluster.coordinator = newCoordinator(ctx, cluster, hbstream.NewTestHeartbeatStreams(ctx, cluster.meta.GetId(), cluster, true)) - cluster.coordinator.run() + opts := mockconfig.NewTestOptions() + cluster := mockcluster.NewCluster(ctx, opts) + coordinator := schedule.NewCoordinator(ctx, cluster, hbstream.NewTestHeartbeatStreams(ctx, cluster.ID, cluster, true)) + coordinator.Run() for _, store := range newTestStores(5, "6.0.0") { if store.GetID() == 3 { store.GetMeta().Labels = []*metapb.StoreLabel{{Key: "engine", Value: "tiflash"}} } - re.NoError(cluster.PutStore(store.GetMeta())) + cluster.PutStore(store) } - recoveryController := newUnsafeRecoveryController(cluster) + recoveryController := NewController(cluster) re.NoError(recoveryController.RemoveFailedStores(map[uint64]struct{}{ 4: {}, 5: {}, @@ -841,14 +886,14 @@ func TestUninitializedPeer(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - _, opt, _ := newTestScheduleConfig() - cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend(), core.NewBasicCluster()) - cluster.coordinator = newCoordinator(ctx, cluster, hbstream.NewTestHeartbeatStreams(ctx, cluster.meta.GetId(), cluster, true)) - cluster.coordinator.run() + opts := mockconfig.NewTestOptions() + cluster := mockcluster.NewCluster(ctx, opts) + coordinator := schedule.NewCoordinator(ctx, cluster, hbstream.NewTestHeartbeatStreams(ctx, cluster.ID, cluster, true)) + coordinator.Run() for _, store := range newTestStores(3, "6.0.0") { - re.NoError(cluster.PutStore(store.GetMeta())) + cluster.PutStore(store) } - recoveryController := newUnsafeRecoveryController(cluster) + recoveryController := NewController(cluster) re.NoError(recoveryController.RemoveFailedStores(map[uint64]struct{}{ 2: {}, 3: {}, @@ -897,14 +942,14 @@ func TestJointState(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - _, opt, _ := newTestScheduleConfig() - cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend(), core.NewBasicCluster()) - cluster.coordinator = newCoordinator(ctx, cluster, hbstream.NewTestHeartbeatStreams(ctx, cluster.meta.GetId(), cluster, true)) - cluster.coordinator.run() + opts := mockconfig.NewTestOptions() + cluster := mockcluster.NewCluster(ctx, opts) + coordinator := schedule.NewCoordinator(ctx, cluster, hbstream.NewTestHeartbeatStreams(ctx, cluster.ID, cluster, true)) + coordinator.Run() for _, store := range newTestStores(5, "6.0.0") { - re.NoError(cluster.PutStore(store.GetMeta())) + cluster.PutStore(store) } - recoveryController := newUnsafeRecoveryController(cluster) + recoveryController := NewController(cluster) re.NoError(recoveryController.RemoveFailedStores(map[uint64]struct{}{ 4: {}, 5: {}, @@ -1090,14 +1135,14 @@ func TestExecutionTimeout(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - _, opt, _ := newTestScheduleConfig() - cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend(), core.NewBasicCluster()) - cluster.coordinator = newCoordinator(ctx, cluster, hbstream.NewTestHeartbeatStreams(ctx, cluster.meta.GetId(), cluster, true)) - cluster.coordinator.run() + opts := mockconfig.NewTestOptions() + cluster := mockcluster.NewCluster(ctx, opts) + coordinator := schedule.NewCoordinator(ctx, cluster, hbstream.NewTestHeartbeatStreams(ctx, cluster.ID, cluster, true)) + coordinator.Run() for _, store := range newTestStores(3, "6.0.0") { - re.NoError(cluster.PutStore(store.GetMeta())) + cluster.PutStore(store) } - recoveryController := newUnsafeRecoveryController(cluster) + recoveryController := NewController(cluster) re.NoError(recoveryController.RemoveFailedStores(map[uint64]struct{}{ 2: {}, 3: {}, @@ -1107,10 +1152,10 @@ func TestExecutionTimeout(t *testing.T) { req := newStoreHeartbeat(1, nil) resp := &pdpb.StoreHeartbeatResponse{} recoveryController.HandleStoreHeartbeat(req, resp) - re.Equal(exitForceLeader, recoveryController.GetStage()) + re.Equal(ExitForceLeader, recoveryController.GetStage()) req.StoreReport = &pdpb.StoreReport{Step: 2} recoveryController.HandleStoreHeartbeat(req, resp) - re.Equal(failed, recoveryController.GetStage()) + re.Equal(Failed, recoveryController.GetStage()) output := recoveryController.Show() re.Equal(len(output), 3) @@ -1122,14 +1167,14 @@ func TestNoHeartbeatTimeout(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - _, opt, _ := newTestScheduleConfig() - cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend(), core.NewBasicCluster()) - cluster.coordinator = newCoordinator(ctx, cluster, hbstream.NewTestHeartbeatStreams(ctx, cluster.meta.GetId(), cluster, true)) - cluster.coordinator.run() + opts := mockconfig.NewTestOptions() + cluster := mockcluster.NewCluster(ctx, opts) + coordinator := schedule.NewCoordinator(ctx, cluster, hbstream.NewTestHeartbeatStreams(ctx, cluster.ID, cluster, true)) + coordinator.Run() for _, store := range newTestStores(3, "6.0.0") { - re.NoError(cluster.PutStore(store.GetMeta())) + cluster.PutStore(store) } - recoveryController := newUnsafeRecoveryController(cluster) + recoveryController := NewController(cluster) re.NoError(recoveryController.RemoveFailedStores(map[uint64]struct{}{ 2: {}, 3: {}, @@ -1137,7 +1182,7 @@ func TestNoHeartbeatTimeout(t *testing.T) { time.Sleep(time.Second) recoveryController.Show() - re.Equal(exitForceLeader, recoveryController.GetStage()) + re.Equal(ExitForceLeader, recoveryController.GetStage()) } func TestExitForceLeader(t *testing.T) { @@ -1145,14 +1190,14 @@ func TestExitForceLeader(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - _, opt, _ := newTestScheduleConfig() - cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend(), core.NewBasicCluster()) - cluster.coordinator = newCoordinator(ctx, cluster, hbstream.NewTestHeartbeatStreams(ctx, cluster.meta.GetId(), cluster, true)) - cluster.coordinator.run() + opts := mockconfig.NewTestOptions() + cluster := mockcluster.NewCluster(ctx, opts) + coordinator := schedule.NewCoordinator(ctx, cluster, hbstream.NewTestHeartbeatStreams(ctx, cluster.ID, cluster, true)) + coordinator.Run() for _, store := range newTestStores(3, "6.0.0") { - re.NoError(cluster.PutStore(store.GetMeta())) + cluster.PutStore(store) } - recoveryController := newUnsafeRecoveryController(cluster) + recoveryController := NewController(cluster) re.NoError(recoveryController.RemoveFailedStores(map[uint64]struct{}{ 2: {}, 3: {}, @@ -1183,7 +1228,7 @@ func TestExitForceLeader(t *testing.T) { recoveryController.HandleStoreHeartbeat(req, resp) applyRecoveryPlan(re, storeID, reports, resp) } - re.Equal(exitForceLeader, recoveryController.GetStage()) + re.Equal(ExitForceLeader, recoveryController.GetStage()) for storeID, report := range reports { req := newStoreHeartbeat(storeID, report) @@ -1192,7 +1237,7 @@ func TestExitForceLeader(t *testing.T) { recoveryController.HandleStoreHeartbeat(req, resp) applyRecoveryPlan(re, storeID, reports, resp) } - re.Equal(finished, recoveryController.GetStage()) + re.Equal(Finished, recoveryController.GetStage()) expects := map[uint64]*pdpb.StoreReport{ 1: { @@ -1223,14 +1268,14 @@ func TestStep(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - _, opt, _ := newTestScheduleConfig() - cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend(), core.NewBasicCluster()) - cluster.coordinator = newCoordinator(ctx, cluster, hbstream.NewTestHeartbeatStreams(ctx, cluster.meta.GetId(), cluster, true)) - cluster.coordinator.run() + opts := mockconfig.NewTestOptions() + cluster := mockcluster.NewCluster(ctx, opts) + coordinator := schedule.NewCoordinator(ctx, cluster, hbstream.NewTestHeartbeatStreams(ctx, cluster.ID, cluster, true)) + coordinator.Run() for _, store := range newTestStores(3, "6.0.0") { - re.NoError(cluster.PutStore(store.GetMeta())) + cluster.PutStore(store) } - recoveryController := newUnsafeRecoveryController(cluster) + recoveryController := NewController(cluster) re.NoError(recoveryController.RemoveFailedStores(map[uint64]struct{}{ 2: {}, 3: {}, @@ -1255,22 +1300,22 @@ func TestStep(t *testing.T) { resp := &pdpb.StoreHeartbeatResponse{} recoveryController.HandleStoreHeartbeat(req, resp) // step is not set, ignore - re.Equal(collectReport, recoveryController.GetStage()) + re.Equal(CollectReport, recoveryController.GetStage()) // valid store report req.StoreReport.Step = 1 recoveryController.HandleStoreHeartbeat(req, resp) - re.Equal(forceLeader, recoveryController.GetStage()) + re.Equal(ForceLeader, recoveryController.GetStage()) // duplicate report with same step, ignore recoveryController.HandleStoreHeartbeat(req, resp) - re.Equal(forceLeader, recoveryController.GetStage()) + re.Equal(ForceLeader, recoveryController.GetStage()) applyRecoveryPlan(re, 1, reports, resp) recoveryController.HandleStoreHeartbeat(req, resp) - re.Equal(demoteFailedVoter, recoveryController.GetStage()) + re.Equal(DemoteFailedVoter, recoveryController.GetStage()) applyRecoveryPlan(re, 1, reports, resp) recoveryController.HandleStoreHeartbeat(req, resp) - re.Equal(finished, recoveryController.GetStage()) + re.Equal(Finished, recoveryController.GetStage()) } func TestOnHealthyRegions(t *testing.T) { @@ -1278,14 +1323,14 @@ func TestOnHealthyRegions(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - _, opt, _ := newTestScheduleConfig() - cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend(), core.NewBasicCluster()) - cluster.coordinator = newCoordinator(ctx, cluster, hbstream.NewTestHeartbeatStreams(ctx, cluster.meta.GetId(), cluster, true)) - cluster.coordinator.run() + opts := mockconfig.NewTestOptions() + cluster := mockcluster.NewCluster(ctx, opts) + coordinator := schedule.NewCoordinator(ctx, cluster, hbstream.NewTestHeartbeatStreams(ctx, cluster.ID, cluster, true)) + coordinator.Run() for _, store := range newTestStores(5, "6.0.0") { - re.NoError(cluster.PutStore(store.GetMeta())) + cluster.PutStore(store) } - recoveryController := newUnsafeRecoveryController(cluster) + recoveryController := NewController(cluster) re.NoError(recoveryController.RemoveFailedStores(map[uint64]struct{}{ 4: {}, 5: {}, @@ -1323,7 +1368,7 @@ func TestOnHealthyRegions(t *testing.T) { {Id: 11, StoreId: 1}, {Id: 21, StoreId: 2}, {Id: 31, StoreId: 3}}}}}, }}, } - re.Equal(collectReport, recoveryController.GetStage()) + re.Equal(CollectReport, recoveryController.GetStage()) // require peer report for storeID := range reports { req := newStoreHeartbeat(storeID, nil) @@ -1346,7 +1391,7 @@ func TestOnHealthyRegions(t *testing.T) { applyRecoveryPlan(re, storeID, reports, resp) } // nothing to do, finish directly - re.Equal(finished, recoveryController.GetStage()) + re.Equal(Finished, recoveryController.GetStage()) } func TestCreateEmptyRegion(t *testing.T) { @@ -1354,14 +1399,14 @@ func TestCreateEmptyRegion(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - _, opt, _ := newTestScheduleConfig() - cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend(), core.NewBasicCluster()) - cluster.coordinator = newCoordinator(ctx, cluster, hbstream.NewTestHeartbeatStreams(ctx, cluster.meta.GetId(), cluster, true)) - cluster.coordinator.run() + opts := mockconfig.NewTestOptions() + cluster := mockcluster.NewCluster(ctx, opts) + coordinator := schedule.NewCoordinator(ctx, cluster, hbstream.NewTestHeartbeatStreams(ctx, cluster.ID, cluster, true)) + coordinator.Run() for _, store := range newTestStores(3, "6.0.0") { - re.NoError(cluster.PutStore(store.GetMeta())) + cluster.PutStore(store) } - recoveryController := newUnsafeRecoveryController(cluster) + recoveryController := NewController(cluster) re.NoError(recoveryController.RemoveFailedStores(map[uint64]struct{}{ 2: {}, 3: {}, @@ -1463,14 +1508,14 @@ func TestRangeOverlap1(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - _, opt, _ := newTestScheduleConfig() - cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend(), core.NewBasicCluster()) - cluster.coordinator = newCoordinator(ctx, cluster, hbstream.NewTestHeartbeatStreams(ctx, cluster.meta.GetId(), cluster, true)) - cluster.coordinator.run() + opts := mockconfig.NewTestOptions() + cluster := mockcluster.NewCluster(ctx, opts) + coordinator := schedule.NewCoordinator(ctx, cluster, hbstream.NewTestHeartbeatStreams(ctx, cluster.ID, cluster, true)) + coordinator.Run() for _, store := range newTestStores(5, "6.0.0") { - re.NoError(cluster.PutStore(store.GetMeta())) + cluster.PutStore(store) } - recoveryController := newUnsafeRecoveryController(cluster) + recoveryController := NewController(cluster) re.NoError(recoveryController.RemoveFailedStores(map[uint64]struct{}{ 4: {}, 5: {}, @@ -1558,14 +1603,14 @@ func TestRangeOverlap2(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - _, opt, _ := newTestScheduleConfig() - cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend(), core.NewBasicCluster()) - cluster.coordinator = newCoordinator(ctx, cluster, hbstream.NewTestHeartbeatStreams(ctx, cluster.meta.GetId(), cluster, true)) - cluster.coordinator.run() + opts := mockconfig.NewTestOptions() + cluster := mockcluster.NewCluster(ctx, opts) + coordinator := schedule.NewCoordinator(ctx, cluster, hbstream.NewTestHeartbeatStreams(ctx, cluster.ID, cluster, true)) + coordinator.Run() for _, store := range newTestStores(5, "6.0.0") { - re.NoError(cluster.PutStore(store.GetMeta())) + cluster.PutStore(store) } - recoveryController := newUnsafeRecoveryController(cluster) + recoveryController := NewController(cluster) re.NoError(recoveryController.RemoveFailedStores(map[uint64]struct{}{ 4: {}, 5: {}, @@ -1652,17 +1697,16 @@ func TestRemoveFailedStores(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - _, opt, _ := newTestScheduleConfig() - cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend(), core.NewBasicCluster()) - cluster.coordinator = newCoordinator(ctx, cluster, hbstream.NewTestHeartbeatStreams(ctx, cluster.meta.GetId(), cluster, true)) - cluster.coordinator.run() + opts := mockconfig.NewTestOptions() + cluster := mockcluster.NewCluster(ctx, opts) + coordinator := schedule.NewCoordinator(ctx, cluster, hbstream.NewTestHeartbeatStreams(ctx, cluster.ID, cluster, true)) + coordinator.Run() stores := newTestStores(2, "5.3.0") stores[1] = stores[1].Clone(core.SetLastHeartbeatTS(time.Now())) for _, store := range stores { - re.NoError(cluster.PutStore(store.GetMeta())) + cluster.PutStore(store) } - recoveryController := newUnsafeRecoveryController(cluster) - + recoveryController := NewController(cluster) // Store 3 doesn't exist, reject to remove. re.Error(recoveryController.RemoveFailedStores(map[uint64]struct{}{ 1: {}, @@ -1673,8 +1717,9 @@ func TestRemoveFailedStores(t *testing.T) { 1: {}, }, 60, false)) re.True(cluster.GetStore(uint64(1)).IsRemoved()) - for _, s := range cluster.GetSchedulers() { - paused, err := cluster.IsSchedulerAllowed(s) + schedulers := coordinator.GetSchedulersController() + for _, s := range schedulers.GetSchedulerNames() { + paused, err := schedulers.IsSchedulerAllowed(s) if s != "split-bucket-scheduler" { re.NoError(err) re.True(paused) @@ -1688,34 +1733,26 @@ func TestRemoveFailedStores(t *testing.T) { }, 60, false)) } -func TestSplitPaused(t *testing.T) { +func TestRunning(t *testing.T) { re := require.New(t) ctx, cancel := context.WithCancel(context.Background()) defer cancel() - _, opt, _ := newTestScheduleConfig() - cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend(), core.NewBasicCluster()) - recoveryController := newUnsafeRecoveryController(cluster) - cluster.Lock() - cluster.unsafeRecoveryController = recoveryController - cluster.coordinator = newCoordinator(ctx, cluster, hbstream.NewTestHeartbeatStreams(ctx, cluster.meta.GetId(), cluster, true)) - cluster.Unlock() - cluster.coordinator.run() + opts := mockconfig.NewTestOptions() + cluster := mockcluster.NewCluster(ctx, opts) + coordinator := schedule.NewCoordinator(ctx, cluster, hbstream.NewTestHeartbeatStreams(ctx, cluster.ID, cluster, true)) + coordinator.Run() stores := newTestStores(2, "5.3.0") stores[1] = stores[1].Clone(core.SetLastHeartbeatTS(time.Now())) for _, store := range stores { - re.NoError(cluster.PutStore(store.GetMeta())) + cluster.PutStore(store) } failedStores := map[uint64]struct{}{ 1: {}, } + recoveryController := NewController(cluster) re.NoError(recoveryController.RemoveFailedStores(failedStores, 60, false)) - askSplitReq := &pdpb.AskSplitRequest{} - _, err := cluster.HandleAskSplit(askSplitReq) - re.Equal("[PD:unsaferecovery:ErrUnsafeRecoveryIsRunning]unsafe recovery is running", err.Error()) - askBatchSplitReq := &pdpb.AskBatchSplitRequest{} - _, err = cluster.HandleAskBatchSplit(askBatchSplitReq) - re.Equal("[PD:unsaferecovery:ErrUnsafeRecoveryIsRunning]unsafe recovery is running", err.Error()) + re.True(recoveryController.IsRunning()) } func TestEpochComparsion(t *testing.T) { @@ -1723,14 +1760,14 @@ func TestEpochComparsion(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - _, opt, _ := newTestScheduleConfig() - cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend(), core.NewBasicCluster()) - cluster.coordinator = newCoordinator(ctx, cluster, hbstream.NewTestHeartbeatStreams(ctx, cluster.meta.GetId(), cluster, true)) - cluster.coordinator.run() + opts := mockconfig.NewTestOptions() + cluster := mockcluster.NewCluster(ctx, opts) + coordinator := schedule.NewCoordinator(ctx, cluster, hbstream.NewTestHeartbeatStreams(ctx, cluster.ID, cluster, true)) + coordinator.Run() for _, store := range newTestStores(3, "6.0.0") { - re.Nil(cluster.PutStore(store.GetMeta())) + cluster.PutStore(store) } - recoveryController := newUnsafeRecoveryController(cluster) + recoveryController := NewController(cluster) re.Nil(recoveryController.RemoveFailedStores(map[uint64]struct{}{ 2: {}, 3: {}, @@ -1796,3 +1833,26 @@ func TestEpochComparsion(t *testing.T) { } } } + +// TODO: remove them +// Create n stores (0..n). +func newTestStores(n uint64, version string) []*core.StoreInfo { + stores := make([]*core.StoreInfo, 0, n) + for i := uint64(1); i <= n; i++ { + store := &metapb.Store{ + Id: i, + Address: fmt.Sprintf("127.0.0.1:%d", i), + StatusAddress: fmt.Sprintf("127.0.0.1:%d", i), + State: metapb.StoreState_Up, + Version: version, + DeployPath: getTestDeployPath(i), + NodeState: metapb.NodeState_Serving, + } + stores = append(stores, core.NewStoreInfo(store)) + } + return stores +} + +func getTestDeployPath(storeID uint64) string { + return fmt.Sprintf("test/store%d", storeID) +} diff --git a/pkg/utils/apiutil/multiservicesapi/middleware.go b/pkg/utils/apiutil/multiservicesapi/middleware.go index 146f4c172cb..ed34ecc6afb 100644 --- a/pkg/utils/apiutil/multiservicesapi/middleware.go +++ b/pkg/utils/apiutil/multiservicesapi/middleware.go @@ -26,16 +26,19 @@ import ( "go.uber.org/zap" ) -// HTTP headers. const ( + // ServiceAllowDirectHandle is the header key to allow direct handle. ServiceAllowDirectHandle = "service-allow-direct-handle" - ServiceRedirectorHeader = "service-redirector" + // ServiceRedirectorHeader is the header key to indicate the request is redirected. + ServiceRedirectorHeader = "service-redirector" + // ServiceContextKey is the key to get service server from gin.Context. + ServiceContextKey = "service" ) // ServiceRedirector is a middleware to redirect the request to the right place. func ServiceRedirector() gin.HandlerFunc { return func(c *gin.Context) { - svr := c.MustGet("service").(bs.Server) + svr := c.MustGet(ServiceContextKey).(bs.Server) allowDirectHandle := len(c.Request.Header.Get(ServiceAllowDirectHandle)) > 0 isServing := svr.IsServing() if allowDirectHandle || isServing { @@ -52,14 +55,13 @@ func ServiceRedirector() gin.HandlerFunc { c.Request.Header.Set(ServiceRedirectorHeader, svr.Name()) - primary := svr.GetPrimary() - if primary == nil { + listenUrls := svr.GetLeaderListenUrls() + if listenUrls == nil { c.AbortWithStatusJSON(http.StatusServiceUnavailable, errs.ErrLeaderNil.FastGenByArgs().Error()) return } - clientUrls := primary.GetClientUrls() - urls := make([]url.URL, 0, len(clientUrls)) - for _, item := range clientUrls { + urls := make([]url.URL, 0, len(listenUrls)) + for _, item := range listenUrls { u, err := url.Parse(item) if err != nil { c.AbortWithStatusJSON(http.StatusInternalServerError, errs.ErrURLParse.Wrap(err).GenWithStackByCause().Error()) diff --git a/pkg/utils/apiutil/serverapi/middleware.go b/pkg/utils/apiutil/serverapi/middleware.go index c650177de9e..653ede75e7a 100644 --- a/pkg/utils/apiutil/serverapi/middleware.go +++ b/pkg/utils/apiutil/serverapi/middleware.go @@ -30,6 +30,7 @@ import ( const ( PDRedirectorHeader = "PD-Redirector" PDAllowFollowerHandle = "PD-Allow-follower-handle" + ForwardedForHeader = "X-Forwarded-For" ) type runtimeServiceValidator struct { @@ -73,17 +74,65 @@ func IsServiceAllowed(s *server.Server, group apiutil.APIServiceGroup) bool { type redirector struct { s *server.Server + + microserviceRedirectRules []*microserviceRedirectRule +} + +type microserviceRedirectRule struct { + matchPath string + targetPath string + targetServiceName string } // NewRedirector redirects request to the leader if needs to be handled in the leader. -func NewRedirector(s *server.Server) negroni.Handler { - return &redirector{s: s} +func NewRedirector(s *server.Server, opts ...RedirectorOption) negroni.Handler { + r := &redirector{s: s} + for _, opt := range opts { + opt(r) + } + return r +} + +// RedirectorOption defines the option of redirector +type RedirectorOption func(*redirector) + +// MicroserviceRedirectRule new a microservice redirect rule option +func MicroserviceRedirectRule(matchPath, targetPath, targetServiceName string) RedirectorOption { + return func(s *redirector) { + s.microserviceRedirectRules = append(s.microserviceRedirectRules, µserviceRedirectRule{ + matchPath, + targetPath, + targetServiceName, + }) + } +} + +func (h *redirector) matchMicroServiceRedirectRules(r *http.Request) (bool, string) { + if !h.s.IsAPIServiceMode() { + return false, "" + } + if len(h.microserviceRedirectRules) == 0 { + return false, "" + } + for _, rule := range h.microserviceRedirectRules { + if rule.matchPath == r.URL.Path { + addr, ok := h.s.GetServicePrimaryAddr(r.Context(), rule.targetServiceName) + if !ok || addr == "" { + log.Warn("failed to get the service primary addr when try match redirect rules", + zap.String("path", r.URL.Path)) + } + r.URL.Path = rule.targetPath + return true, addr + } + } + return false, "" } func (h *redirector) ServeHTTP(w http.ResponseWriter, r *http.Request, next http.HandlerFunc) { + matchedFlag, targetAddr := h.matchMicroServiceRedirectRules(r) allowFollowerHandle := len(r.Header.Get(PDAllowFollowerHandle)) > 0 isLeader := h.s.GetMember().IsLeader() - if !h.s.IsClosed() && (allowFollowerHandle || isLeader) { + if !h.s.IsClosed() && (allowFollowerHandle || isLeader) && !matchedFlag { next(w, r) return } @@ -96,13 +145,23 @@ func (h *redirector) ServeHTTP(w http.ResponseWriter, r *http.Request, next http } r.Header.Set(PDRedirectorHeader, h.s.Name()) + r.Header.Add(ForwardedForHeader, r.RemoteAddr) - leader := h.s.GetMember().GetLeader() - if leader == nil { - http.Error(w, "no leader", http.StatusServiceUnavailable) - return + var clientUrls []string + if matchedFlag { + if len(targetAddr) == 0 { + http.Error(w, apiutil.ErrRedirectFailed, http.StatusInternalServerError) + return + } + clientUrls = append(clientUrls, targetAddr) + } else { + leader := h.s.GetMember().GetLeader() + if leader == nil { + http.Error(w, "no leader", http.StatusServiceUnavailable) + return + } + clientUrls = leader.GetClientUrls() } - clientUrls := leader.GetClientUrls() urls := make([]url.URL, 0, len(clientUrls)) for _, item := range clientUrls { u, err := url.Parse(item) diff --git a/pkg/utils/configutil/configutil.go b/pkg/utils/configutil/configutil.go index f8560bdf238..4349721ba78 100644 --- a/pkg/utils/configutil/configutil.go +++ b/pkg/utils/configutil/configutil.go @@ -101,15 +101,15 @@ func ConfigFromFile(c interface{}, path string) (*toml.MetaData, error) { return &meta, errors.WithStack(err) } -// AdjustCommandlineString adjusts the value of a string variable from command line flags. -func AdjustCommandlineString(flagSet *pflag.FlagSet, v *string, name string) { +// AdjustCommandLineString adjusts the value of a string variable from command line flags. +func AdjustCommandLineString(flagSet *pflag.FlagSet, v *string, name string) { if value, _ := flagSet.GetString(name); value != "" { *v = value } } -// AdjustCommandlineBool adjusts the value of a bool variable from command line flags. -func AdjustCommandlineBool(flagSet *pflag.FlagSet, v *bool, name string) { +// AdjustCommandLineBool adjusts the value of a bool variable from command line flags. +func AdjustCommandLineBool(flagSet *pflag.FlagSet, v *bool, name string) { if value, _ := flagSet.GetBool(name); value { *v = value } diff --git a/pkg/utils/etcdutil/etcdutil.go b/pkg/utils/etcdutil/etcdutil.go index 0c7081d537e..59173222205 100644 --- a/pkg/utils/etcdutil/etcdutil.go +++ b/pkg/utils/etcdutil/etcdutil.go @@ -17,9 +17,11 @@ package etcdutil import ( "context" "crypto/tls" + "fmt" "math/rand" "net/http" "net/url" + "sync" "time" "github.com/gogo/protobuf/proto" @@ -27,9 +29,12 @@ import ( "github.com/pingcap/failpoint" "github.com/pingcap/log" "github.com/tikv/pd/pkg/errs" + "github.com/tikv/pd/pkg/utils/logutil" "github.com/tikv/pd/pkg/utils/typeutil" "go.etcd.io/etcd/clientv3" "go.etcd.io/etcd/etcdserver" + "go.etcd.io/etcd/etcdserver/api/v3rpc/rpctypes" + "go.etcd.io/etcd/mvcc/mvccpb" "go.etcd.io/etcd/pkg/types" "go.uber.org/zap" ) @@ -38,9 +43,6 @@ const ( // defaultEtcdClientTimeout is the default timeout for etcd client. defaultEtcdClientTimeout = 3 * time.Second - // defaultAutoSyncInterval is the interval to sync etcd cluster. - defaultAutoSyncInterval = 60 * time.Second - // defaultDialKeepAliveTime is the time after which client pings the server to see if transport is alive. defaultDialKeepAliveTime = 10 * time.Second @@ -58,6 +60,8 @@ const ( // DefaultSlowRequestTime 1s for the threshold for normal request, for those // longer then 1s, they are considered as slow requests. DefaultSlowRequestTime = time.Second + + healthyPath = "health" ) // CheckClusterID checks etcd cluster ID, returns an error if mismatch. @@ -141,6 +145,20 @@ func EtcdKVGet(c *clientv3.Client, key string, opts ...clientv3.OpOption) (*clie return resp, nil } +// IsHealthy checks if the etcd is healthy. +func IsHealthy(ctx context.Context, client *clientv3.Client) bool { + timeout := DefaultRequestTimeout + failpoint.Inject("fastTick", func() { + timeout = 100 * time.Millisecond + }) + ctx, cancel := context.WithTimeout(clientv3.WithRequireLeader(ctx), timeout) + defer cancel() + _, err := client.Get(ctx, healthyPath) + // permission denied is OK since proposal goes through consensus to get it + // See: https://github.com/etcd-io/etcd/blob/85b640cee793e25f3837c47200089d14a8392dc7/etcdctl/ctlv3/command/ep_command.go#L124 + return err == nil || err == rpctypes.ErrPermissionDenied +} + // GetValue gets value with key from etcd. func GetValue(c *clientv3.Client, key string, opts ...clientv3.OpOption) ([]byte, error) { resp, err := get(c, key, opts...) @@ -193,57 +211,235 @@ func EtcdKVPutWithTTL(ctx context.Context, c *clientv3.Client, key string, value return kv.Put(ctx, key, value, clientv3.WithLease(grantResp.ID)) } -// CreateClients creates etcd v3 client and http client. -func CreateClients(tlsConfig *tls.Config, acUrls []url.URL) (*clientv3.Client, *http.Client, error) { - client, err := createEtcdClient(tlsConfig, acUrls) - if err != nil { - return nil, nil, errs.ErrNewEtcdClient.Wrap(err).GenWithStackByCause() - } - httpClient := &http.Client{ - Transport: &http.Transport{ - DisableKeepAlives: true, - TLSClientConfig: tlsConfig, - }, - } - return client, httpClient, nil -} +const ( + // etcdServerOfflineTimeout is the timeout for an unhealthy etcd endpoint to be offline from healthy checker. + etcdServerOfflineTimeout = 30 * time.Minute + // etcdServerDisconnectedTimeout is the timeout for an unhealthy etcd endpoint to be disconnected from healthy checker. + etcdServerDisconnectedTimeout = 1 * time.Minute +) -func createEtcdClient(tlsConfig *tls.Config, acUrls []url.URL) (*clientv3.Client, error) { - if len(acUrls) == 0 { - return nil, errs.ErrNewEtcdClient.FastGenByArgs("no available etcd address") - } - endpoints := make([]string, 0, len(acUrls)) - for _, u := range acUrls { - endpoints = append(endpoints, u.String()) +func newClient(tlsConfig *tls.Config, endpoints ...string) (*clientv3.Client, error) { + if len(endpoints) == 0 { + return nil, errs.ErrNewEtcdClient.FastGenByArgs("empty etcd endpoints") } lgc := zap.NewProductionConfig() lgc.Encoding = log.ZapEncodingName - autoSyncInterval := defaultAutoSyncInterval - dialKeepAliveTime := defaultDialKeepAliveTime - dialKeepAliveTimeout := defaultDialKeepAliveTimeout - failpoint.Inject("autoSyncInterval", func() { - autoSyncInterval = 10 * time.Millisecond - }) - failpoint.Inject("closeKeepAliveCheck", func() { - autoSyncInterval = 0 - dialKeepAliveTime = 0 - dialKeepAliveTimeout = 0 - }) client, err := clientv3.New(clientv3.Config{ Endpoints: endpoints, DialTimeout: defaultEtcdClientTimeout, - AutoSyncInterval: autoSyncInterval, TLS: tlsConfig, LogConfig: &lgc, - DialKeepAliveTime: dialKeepAliveTime, - DialKeepAliveTimeout: dialKeepAliveTimeout, + DialKeepAliveTime: defaultDialKeepAliveTime, + DialKeepAliveTimeout: defaultDialKeepAliveTimeout, + }) + return client, err +} + +// CreateEtcdClient creates etcd v3 client with detecting endpoints. +func CreateEtcdClient(tlsConfig *tls.Config, acURLs []url.URL) (*clientv3.Client, error) { + urls := make([]string, 0, len(acURLs)) + for _, u := range acURLs { + urls = append(urls, u.String()) + } + client, err := newClient(tlsConfig, urls...) + if err != nil { + return nil, err + } + + tickerInterval := defaultDialKeepAliveTime + failpoint.Inject("fastTick", func() { + tickerInterval = 100 * time.Millisecond }) - if err == nil { - log.Info("create etcd v3 client", zap.Strings("endpoints", endpoints)) + failpoint.Inject("closeTick", func() { + failpoint.Return(client, err) + }) + + checker := &healthyChecker{ + tlsConfig: tlsConfig, } + eps := syncUrls(client) + checker.update(eps) + + // Create a goroutine to check the health of etcd endpoints periodically. + go func(client *clientv3.Client) { + defer logutil.LogPanic() + ticker := time.NewTicker(tickerInterval) + defer ticker.Stop() + lastAvailable := time.Now() + for { + select { + case <-client.Ctx().Done(): + log.Info("[etcd client] etcd client is closed, exit health check goroutine") + checker.Range(func(key, value interface{}) bool { + client := value.(*healthyClient) + client.Close() + return true + }) + return + case <-ticker.C: + usedEps := client.Endpoints() + healthyEps := checker.patrol(client.Ctx()) + if len(healthyEps) == 0 { + // when all endpoints are unhealthy, try to reset endpoints to update connect + // rather than delete them to avoid there is no any endpoint in client. + // Note: reset endpoints will trigger subconn closed, and then trigger reconnect. + // otherwise, the subconn will be retrying in grpc layer and use exponential backoff, + // and it cannot recover as soon as possible. + if time.Since(lastAvailable) > etcdServerDisconnectedTimeout { + log.Info("[etcd client] no available endpoint, try to reset endpoints", zap.Strings("last-endpoints", usedEps)) + client.SetEndpoints([]string{}...) + client.SetEndpoints(usedEps...) + } + } else { + if !typeutil.AreStringSlicesEquivalent(healthyEps, usedEps) { + client.SetEndpoints(healthyEps...) + change := fmt.Sprintf("%d->%d", len(usedEps), len(healthyEps)) + etcdStateGauge.WithLabelValues("endpoints").Set(float64(len(healthyEps))) + log.Info("[etcd client] update endpoints", zap.String("num-change", change), + zap.Strings("last-endpoints", usedEps), zap.Strings("endpoints", client.Endpoints())) + } + lastAvailable = time.Now() + } + } + } + }(client) + + // Notes: use another goroutine to update endpoints to avoid blocking health check in the first goroutine. + go func(client *clientv3.Client) { + defer logutil.LogPanic() + ticker := time.NewTicker(tickerInterval) + defer ticker.Stop() + for { + select { + case <-client.Ctx().Done(): + log.Info("[etcd client] etcd client is closed, exit update endpoint goroutine") + return + case <-ticker.C: + eps := syncUrls(client) + checker.update(eps) + } + } + }(client) + return client, err } +type healthyClient struct { + *clientv3.Client + lastHealth time.Time +} + +type healthyChecker struct { + sync.Map // map[string]*healthyClient + tlsConfig *tls.Config +} + +func (checker *healthyChecker) patrol(ctx context.Context) []string { + // See https://github.com/etcd-io/etcd/blob/85b640cee793e25f3837c47200089d14a8392dc7/etcdctl/ctlv3/command/ep_command.go#L105-L145 + var wg sync.WaitGroup + count := 0 + checker.Range(func(key, value interface{}) bool { + count++ + return true + }) + hch := make(chan string, count) + healthyList := make([]string, 0, count) + checker.Range(func(key, value interface{}) bool { + wg.Add(1) + go func(key, value interface{}) { + defer wg.Done() + defer logutil.LogPanic() + ep := key.(string) + client := value.(*healthyClient) + if IsHealthy(ctx, client.Client) { + hch <- ep + checker.Store(ep, &healthyClient{ + Client: client.Client, + lastHealth: time.Now(), + }) + return + } + }(key, value) + return true + }) + wg.Wait() + close(hch) + for h := range hch { + healthyList = append(healthyList, h) + } + return healthyList +} + +func (checker *healthyChecker) update(eps []string) { + for _, ep := range eps { + // check if client exists, if not, create one, if exists, check if it's offline or disconnected. + if client, ok := checker.Load(ep); ok { + lastHealthy := client.(*healthyClient).lastHealth + if time.Since(lastHealthy) > etcdServerOfflineTimeout { + log.Info("[etcd client] some etcd server maybe offline", zap.String("endpoint", ep)) + checker.Delete(ep) + } + if time.Since(lastHealthy) > etcdServerDisconnectedTimeout { + // try to reset client endpoint to trigger reconnect + client.(*healthyClient).Client.SetEndpoints([]string{}...) + client.(*healthyClient).Client.SetEndpoints(ep) + } + continue + } + checker.addClient(ep, time.Now()) + } +} + +func (checker *healthyChecker) addClient(ep string, lastHealth time.Time) { + client, err := newClient(checker.tlsConfig, ep) + if err != nil { + log.Error("[etcd client] failed to create etcd healthy client", zap.Error(err)) + return + } + checker.Store(ep, &healthyClient{ + Client: client, + lastHealth: lastHealth, + }) +} + +func syncUrls(client *clientv3.Client) []string { + // See https://github.com/etcd-io/etcd/blob/85b640cee793e25f3837c47200089d14a8392dc7/clientv3/client.go#L170-L183 + ctx, cancel := context.WithTimeout(clientv3.WithRequireLeader(client.Ctx()), DefaultRequestTimeout) + defer cancel() + mresp, err := client.MemberList(ctx) + if err != nil { + log.Error("[etcd client] failed to list members", errs.ZapError(err)) + return []string{} + } + var eps []string + for _, m := range mresp.Members { + if len(m.Name) != 0 && !m.IsLearner { + eps = append(eps, m.ClientURLs...) + } + } + return eps +} + +// CreateClients creates etcd v3 client and http client. +func CreateClients(tlsConfig *tls.Config, acUrls []url.URL) (*clientv3.Client, *http.Client, error) { + client, err := CreateEtcdClient(tlsConfig, acUrls) + if err != nil { + return nil, nil, errs.ErrNewEtcdClient.Wrap(err).GenWithStackByCause() + } + httpClient := createHTTPClient(tlsConfig) + return client, httpClient, nil +} + +// createHTTPClient creates a http client with the given tls config. +func createHTTPClient(tlsConfig *tls.Config) *http.Client { + return &http.Client{ + Transport: &http.Transport{ + DisableKeepAlives: true, + TLSClientConfig: tlsConfig, + }, + } +} + // InitClusterID creates a cluster ID for the given key if it hasn't existed. // This function assumes the cluster ID has already existed and always use a // cheaper read to retrieve it; if it doesn't exist, invoke the more expensive @@ -282,8 +478,9 @@ func InitOrGetClusterID(c *clientv3.Client, key string) (uint64, error) { defer cancel() // Generate a random cluster ID. + r := rand.New(rand.NewSource(time.Now().UnixNano())) ts := uint64(time.Now().Unix()) - clusterID := (ts << 32) + uint64(rand.Uint32()) + clusterID := (ts << 32) + uint64(r.Uint32()) value := typeutil.Uint64ToBytes(clusterID) // Multiple servers may try to init the cluster ID at the same time. @@ -315,3 +512,321 @@ func InitOrGetClusterID(c *clientv3.Client, key string) (uint64, error) { return typeutil.BytesToUint64(response.Kvs[0].Value) } + +const ( + defaultLoadDataFromEtcdTimeout = 30 * time.Second + defaultLoadFromEtcdRetryInterval = 200 * time.Millisecond + defaultLoadFromEtcdRetryTimes = int(defaultLoadDataFromEtcdTimeout / defaultLoadFromEtcdRetryInterval) + defaultLoadBatchSize = 400 + defaultWatchChangeRetryInterval = 1 * time.Second + defaultForceLoadMinimalInterval = 200 * time.Millisecond +) + +// LoopWatcher loads data from etcd and sets a watcher for it. +type LoopWatcher struct { + ctx context.Context + wg *sync.WaitGroup + name string + client *clientv3.Client + + // key is the etcd key to watch. + key string + // opts is used to set etcd options. + opts []clientv3.OpOption + + // forceLoadCh is used to force loading data from etcd. + forceLoadCh chan struct{} + // isLoadedCh is used to notify that the data has been loaded from etcd first time. + isLoadedCh chan error + + // putFn is used to handle the put event. + putFn func(*mvccpb.KeyValue) error + // deleteFn is used to handle the delete event. + deleteFn func(*mvccpb.KeyValue) error + // postEventFn is used to call after handling all events. + postEventFn func() error + + // forceLoadMu is used to ensure two force loads have minimal interval. + forceLoadMu sync.RWMutex + // lastTimeForceLoad is used to record the last time force loading data from etcd. + lastTimeForceLoad time.Time + + // loadTimeout is used to set the timeout for loading data from etcd. + loadTimeout time.Duration + // loadRetryTimes is used to set the retry times for loading data from etcd. + loadRetryTimes int + // loadBatchSize is used to set the batch size for loading data from etcd. + loadBatchSize int64 + // watchChangeRetryInterval is used to set the retry interval for watching etcd change. + watchChangeRetryInterval time.Duration + // updateClientCh is used to update the etcd client. + // It's only used for testing. + updateClientCh chan *clientv3.Client +} + +// NewLoopWatcher creates a new LoopWatcher. +func NewLoopWatcher(ctx context.Context, wg *sync.WaitGroup, client *clientv3.Client, name, key string, + putFn, deleteFn func(*mvccpb.KeyValue) error, postEventFn func() error, opts ...clientv3.OpOption) *LoopWatcher { + return &LoopWatcher{ + ctx: ctx, + client: client, + name: name, + key: key, + wg: wg, + forceLoadCh: make(chan struct{}, 1), + isLoadedCh: make(chan error, 1), + updateClientCh: make(chan *clientv3.Client, 1), + putFn: putFn, + deleteFn: deleteFn, + postEventFn: postEventFn, + opts: opts, + lastTimeForceLoad: time.Now(), + loadTimeout: defaultLoadDataFromEtcdTimeout, + loadRetryTimes: defaultLoadFromEtcdRetryTimes, + loadBatchSize: defaultLoadBatchSize, + watchChangeRetryInterval: defaultWatchChangeRetryInterval, + } +} + +// StartWatchLoop starts a loop to watch the key. +func (lw *LoopWatcher) StartWatchLoop() { + lw.wg.Add(1) + go func() { + defer logutil.LogPanic() + defer lw.wg.Done() + + ctx, cancel := context.WithCancel(lw.ctx) + defer cancel() + watchStartRevision := lw.initFromEtcd(ctx) + + log.Info("start to watch loop", zap.String("name", lw.name), zap.String("key", lw.key)) + for { + select { + case <-ctx.Done(): + log.Info("server is closed, exit watch loop", zap.String("name", lw.name), zap.String("key", lw.key)) + return + default: + } + nextRevision, err := lw.watch(ctx, watchStartRevision) + if err != nil { + log.Error("watcher canceled unexpectedly and a new watcher will start after a while for watch loop", + zap.String("name", lw.name), + zap.String("key", lw.key), + zap.Int64("next-revision", nextRevision), + zap.Time("retry-at", time.Now().Add(lw.watchChangeRetryInterval)), + zap.Error(err)) + watchStartRevision = nextRevision + time.Sleep(lw.watchChangeRetryInterval) + failpoint.Inject("updateClient", func() { + lw.client = <-lw.updateClientCh + }) + } + } + }() +} + +func (lw *LoopWatcher) initFromEtcd(ctx context.Context) int64 { + var ( + watchStartRevision int64 + err error + ) + ticker := time.NewTicker(defaultLoadFromEtcdRetryInterval) + defer ticker.Stop() + ctx, cancel := context.WithTimeout(ctx, lw.loadTimeout) + defer cancel() + + for i := 0; i < lw.loadRetryTimes; i++ { + failpoint.Inject("loadTemporaryFail", func(val failpoint.Value) { + if maxFailTimes, ok := val.(int); ok && i < maxFailTimes { + err = errors.New("fail to read from etcd") + failpoint.Continue() + } + }) + failpoint.Inject("delayLoad", func(val failpoint.Value) { + if sleepIntervalSeconds, ok := val.(int); ok && sleepIntervalSeconds > 0 { + time.Sleep(time.Duration(sleepIntervalSeconds) * time.Second) + } + }) + watchStartRevision, err = lw.load(ctx) + if err == nil { + break + } + select { + case <-ctx.Done(): + lw.isLoadedCh <- errors.Errorf("ctx is done before load data from etcd") + return watchStartRevision + case <-ticker.C: + } + } + if err != nil { + log.Warn("meet error when loading in watch loop", zap.String("name", lw.name), zap.String("key", lw.key), zap.Error(err)) + } else { + log.Info("load finished in watch loop", zap.String("name", lw.name), zap.String("key", lw.key)) + } + lw.isLoadedCh <- err + return watchStartRevision +} + +func (lw *LoopWatcher) watch(ctx context.Context, revision int64) (nextRevision int64, err error) { + watcher := clientv3.NewWatcher(lw.client) + defer watcher.Close() + var watchChanCancel context.CancelFunc + defer func() { + if watchChanCancel != nil { + watchChanCancel() + } + }() + for { + if watchChanCancel != nil { + watchChanCancel() + } + // In order to prevent a watch stream being stuck in a partitioned node, + // make sure to wrap context with "WithRequireLeader". + watchChanCtx, cancel := context.WithCancel(clientv3.WithRequireLeader(ctx)) + watchChanCancel = cancel + opts := append(lw.opts, clientv3.WithRev(revision)) + watchChan := watcher.Watch(watchChanCtx, lw.key, opts...) + WatchChanLoop: + select { + case <-ctx.Done(): + return revision, nil + case <-lw.forceLoadCh: + revision, err = lw.load(ctx) + if err != nil { + log.Warn("force load key failed in watch loop", zap.String("name", lw.name), + zap.String("key", lw.key), zap.Error(err)) + } + continue + case wresp := <-watchChan: + if wresp.CompactRevision != 0 { + log.Warn("required revision has been compacted, use the compact revision in watch loop", + zap.Int64("required-revision", revision), + zap.Int64("compact-revision", wresp.CompactRevision)) + revision = wresp.CompactRevision + continue + } else if wresp.Err() != nil { // wresp.Err() contains CompactRevision not equal to 0 + log.Error("watcher is canceled in watch loop", + zap.Int64("revision", revision), + errs.ZapError(errs.ErrEtcdWatcherCancel, wresp.Err())) + return revision, wresp.Err() + } + for _, event := range wresp.Events { + switch event.Type { + case clientv3.EventTypePut: + if err := lw.putFn(event.Kv); err != nil { + log.Error("put failed in watch loop", zap.String("name", lw.name), + zap.String("key", lw.key), zap.Error(err)) + } else { + log.Debug("put in watch loop", zap.String("name", lw.name), + zap.ByteString("key", event.Kv.Key), + zap.ByteString("value", event.Kv.Value)) + } + case clientv3.EventTypeDelete: + if err := lw.deleteFn(event.Kv); err != nil { + log.Error("delete failed in watch loop", zap.String("name", lw.name), + zap.String("key", lw.key), zap.Error(err)) + } else { + log.Debug("delete in watch loop", zap.String("name", lw.name), + zap.ByteString("key", event.Kv.Key)) + } + } + } + if err := lw.postEventFn(); err != nil { + log.Error("run post event failed in watch loop", zap.String("name", lw.name), + zap.String("key", lw.key), zap.Error(err)) + } + revision = wresp.Header.Revision + 1 + goto WatchChanLoop // use goto to avoid to create a new watchChan + } + } +} + +func (lw *LoopWatcher) load(ctx context.Context) (nextRevision int64, err error) { + ctx, cancel := context.WithTimeout(ctx, DefaultRequestTimeout) + defer cancel() + startKey := lw.key + // If limit is 0, it means no limit. + // If limit is not 0, we need to add 1 to limit to get the next key. + limit := lw.loadBatchSize + if limit != 0 { + limit++ + } + for { + // Sort by key to get the next key and we don't need to worry about the performance, + // Because the default sort is just SortByKey and SortAscend + opts := append(lw.opts, clientv3.WithSort(clientv3.SortByKey, clientv3.SortAscend), clientv3.WithLimit(limit)) + resp, err := clientv3.NewKV(lw.client).Get(ctx, startKey, opts...) + if err != nil { + log.Error("load failed in watch loop", zap.String("name", lw.name), + zap.String("key", lw.key), zap.Error(err)) + return 0, err + } + for i, item := range resp.Kvs { + if resp.More && i == len(resp.Kvs)-1 { + // The last key is the start key of the next batch. + // To avoid to get the same key in the next load, we need to skip the last key. + startKey = string(item.Key) + continue + } + err = lw.putFn(item) + if err != nil { + log.Error("put failed in watch loop when loading", zap.String("name", lw.name), zap.String("key", lw.key), zap.Error(err)) + } + } + // Note: if there are no keys in etcd, the resp.More is false. It also means the load is finished. + if !resp.More { + if err := lw.postEventFn(); err != nil { + log.Error("run post event failed in watch loop", zap.String("name", lw.name), + zap.String("key", lw.key), zap.Error(err)) + } + return resp.Header.Revision + 1, err + } + } +} + +// ForceLoad forces to load the key. +func (lw *LoopWatcher) ForceLoad() { + // When NotLeader error happens, a large volume of force load requests will be received here, + // so the minimal interval between two force loads (from etcd) is used to avoid the congestion. + // Two-phase locking is also used to let most of the requests return directly without acquiring + // the write lock and causing the system to choke. + lw.forceLoadMu.RLock() + if time.Since(lw.lastTimeForceLoad) < defaultForceLoadMinimalInterval { + lw.forceLoadMu.RUnlock() + return + } + lw.forceLoadMu.RUnlock() + + lw.forceLoadMu.Lock() + if time.Since(lw.lastTimeForceLoad) < defaultForceLoadMinimalInterval { + lw.forceLoadMu.Unlock() + return + } + lw.lastTimeForceLoad = time.Now() + lw.forceLoadMu.Unlock() + + select { + case lw.forceLoadCh <- struct{}{}: + default: + } +} + +// WaitLoad waits for the result to obtain whether data is loaded. +func (lw *LoopWatcher) WaitLoad() error { + return <-lw.isLoadedCh +} + +// SetLoadRetryTimes sets the retry times when loading data from etcd. +func (lw *LoopWatcher) SetLoadRetryTimes(times int) { + lw.loadRetryTimes = times +} + +// SetLoadTimeout sets the timeout when loading data from etcd. +func (lw *LoopWatcher) SetLoadTimeout(timeout time.Duration) { + lw.loadTimeout = timeout +} + +// SetLoadBatchSize sets the batch size when loading data from etcd. +func (lw *LoopWatcher) SetLoadBatchSize(size int64) { + lw.loadBatchSize = size +} diff --git a/pkg/utils/etcdutil/etcdutil_test.go b/pkg/utils/etcdutil/etcdutil_test.go index 06ac61e264e..31a84d8a0ad 100644 --- a/pkg/utils/etcdutil/etcdutil_test.go +++ b/pkg/utils/etcdutil/etcdutil_test.go @@ -19,21 +19,31 @@ import ( "crypto/tls" "fmt" "io" + "math/rand" "net" "strings" + "sync" "sync/atomic" "testing" "time" "github.com/pingcap/failpoint" "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" "github.com/tikv/pd/pkg/utils/tempurl" + "github.com/tikv/pd/pkg/utils/testutil" "go.etcd.io/etcd/clientv3" "go.etcd.io/etcd/embed" "go.etcd.io/etcd/etcdserver/etcdserverpb" + "go.etcd.io/etcd/mvcc/mvccpb" "go.etcd.io/etcd/pkg/types" + "go.uber.org/goleak" ) +func TestMain(m *testing.M) { + goleak.VerifyTestMain(m, testutil.LeakOptions...) +} + func TestMemberHelpers(t *testing.T) { re := require.New(t) cfg1 := NewTestSingleConfig(t) @@ -47,6 +57,9 @@ func TestMemberHelpers(t *testing.T) { client1, err := clientv3.New(clientv3.Config{ Endpoints: []string{ep1}, }) + defer func() { + client1.Close() + }() re.NoError(err) <-etcd1.Server.ReadyNotify() @@ -66,6 +79,9 @@ func TestMemberHelpers(t *testing.T) { client2, err := clientv3.New(clientv3.Config{ Endpoints: []string{ep2}, }) + defer func() { + client2.Close() + }() re.NoError(err) checkMembers(re, client2, []*embed.Etcd{etcd1, etcd2}) @@ -98,6 +114,9 @@ func TestEtcdKVGet(t *testing.T) { client, err := clientv3.New(clientv3.Config{ Endpoints: []string{ep}, }) + defer func() { + client.Close() + }() re.NoError(err) <-etcd.Server.ReadyNotify() @@ -148,6 +167,9 @@ func TestEtcdKVPutWithTTL(t *testing.T) { client, err := clientv3.New(clientv3.Config{ Endpoints: []string{ep}, }) + defer func() { + client.Close() + }() re.NoError(err) <-etcd.Server.ReadyNotify() @@ -188,6 +210,9 @@ func TestInitClusterID(t *testing.T) { client, err := clientv3.New(clientv3.Config{ Endpoints: []string{ep}, }) + defer func() { + client.Close() + }() re.NoError(err) <-etcd.Server.ReadyNotify() @@ -209,18 +234,21 @@ func TestInitClusterID(t *testing.T) { func TestEtcdClientSync(t *testing.T) { re := require.New(t) - re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/utils/etcdutil/autoSyncInterval", "return(true)")) + re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/utils/etcdutil/fastTick", "return(true)")) // Start a etcd server. cfg1 := NewTestSingleConfig(t) etcd1, err := embed.StartEtcd(cfg1) + defer func() { + etcd1.Close() + }() re.NoError(err) // Create a etcd client with etcd1 as endpoint. - ep1 := cfg1.LCUrls[0].String() - urls, err := types.NewURLs([]string{ep1}) - re.NoError(err) - client1, err := createEtcdClient(nil, urls) + client1, err := CreateEtcdClient(nil, cfg1.LCUrls) + defer func() { + client1.Close() + }() re.NoError(err) <-etcd1.Server.ReadyNotify() @@ -228,36 +256,124 @@ func TestEtcdClientSync(t *testing.T) { etcd2 := checkAddEtcdMember(t, cfg1, client1) defer etcd2.Close() checkMembers(re, client1, []*embed.Etcd{etcd1, etcd2}) + testutil.Eventually(re, func() bool { + // wait for etcd client sync endpoints + return len(client1.Endpoints()) == 2 + }) // Remove the first member and close the etcd1. _, err = RemoveEtcdMember(client1, uint64(etcd1.Server.ID())) re.NoError(err) - time.Sleep(20 * time.Millisecond) // wait for etcd client sync endpoints and client will be connected to etcd2 etcd1.Close() // Check the client can get the new member with the new endpoints. - listResp3, err := ListEtcdMembers(client1) + testutil.Eventually(re, func() bool { + // wait for etcd client sync endpoints + return len(client1.Endpoints()) == 1 + }) + + re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/utils/etcdutil/fastTick")) +} + +func TestEtcdScaleInAndOut(t *testing.T) { + re := require.New(t) + // Start a etcd server. + cfg1 := NewTestSingleConfig(t) + etcd1, err := embed.StartEtcd(cfg1) + defer func() { + etcd1.Close() + }() re.NoError(err) - re.Len(listResp3.Members, 1) - re.Equal(uint64(etcd2.Server.ID()), listResp3.Members[0].ID) + <-etcd1.Server.ReadyNotify() + + // Create two etcd clients with etcd1 as endpoint. + client1, err := CreateEtcdClient(nil, cfg1.LCUrls) // execute member change operation with this client + defer func() { + client1.Close() + }() + re.NoError(err) + client2, err := CreateEtcdClient(nil, cfg1.LCUrls) // check member change with this client + defer func() { + client2.Close() + }() + re.NoError(err) + + // Add a new member and check members + etcd2 := checkAddEtcdMember(t, cfg1, client1) + defer func() { + etcd2.Close() + }() + checkMembers(re, client2, []*embed.Etcd{etcd1, etcd2}) + + // scale in etcd1 + _, err = RemoveEtcdMember(client1, uint64(etcd1.Server.ID())) + re.NoError(err) + checkMembers(re, client2, []*embed.Etcd{etcd2}) +} + +func TestRandomKillEtcd(t *testing.T) { + re := require.New(t) + re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/utils/etcdutil/fastTick", "return(true)")) + // Start a etcd server. + cfg1 := NewTestSingleConfig(t) + etcd1, err := embed.StartEtcd(cfg1) + re.NoError(err) + <-etcd1.Server.ReadyNotify() + client1, err := CreateEtcdClient(nil, cfg1.LCUrls) + re.NoError(err) + defer func() { + client1.Close() + }() + + etcd2 := checkAddEtcdMember(t, cfg1, client1) + cfg2 := etcd2.Config() + <-etcd2.Server.ReadyNotify() - require.NoError(t, failpoint.Disable("github.com/tikv/pd/pkg/utils/etcdutil/autoSyncInterval")) + etcd3 := checkAddEtcdMember(t, &cfg2, client1) + <-etcd3.Server.ReadyNotify() + + time.Sleep(1 * time.Second) + re.Len(client1.Endpoints(), 3) + + // Randomly kill an etcd server and restart it + etcds := []*embed.Etcd{etcd1, etcd2, etcd3} + cfgs := []embed.Config{etcd1.Config(), etcd2.Config(), etcd3.Config()} + for i := 0; i < 10; i++ { + killIndex := rand.Intn(len(etcds)) + etcds[killIndex].Close() + testutil.Eventually(re, func() bool { + return IsHealthy(context.Background(), client1) + }) + etcd, err := embed.StartEtcd(&cfgs[killIndex]) + re.NoError(err) + <-etcd.Server.ReadyNotify() + etcds[killIndex] = etcd + testutil.Eventually(re, func() bool { + return IsHealthy(context.Background(), client1) + }) + } + for _, etcd := range etcds { + if etcd != nil { + etcd.Close() + } + } + re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/utils/etcdutil/fastTick")) } func TestEtcdWithHangLeaderEnableCheck(t *testing.T) { re := require.New(t) var err error // Test with enable check. - re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/utils/etcdutil/autoSyncInterval", "return(true)")) + re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/utils/etcdutil/fastTick", "return(true)")) err = checkEtcdWithHangLeader(t) re.NoError(err) - require.NoError(t, failpoint.Disable("github.com/tikv/pd/pkg/utils/etcdutil/autoSyncInterval")) + re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/utils/etcdutil/fastTick")) // Test with disable check. - re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/utils/etcdutil/closeKeepAliveCheck", "return(true)")) + re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/utils/etcdutil/closeTick", "return(true)")) err = checkEtcdWithHangLeader(t) re.Error(err) - require.NoError(t, failpoint.Disable("github.com/tikv/pd/pkg/utils/etcdutil/closeKeepAliveCheck")) + re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/utils/etcdutil/closeTick")) } func checkEtcdWithHangLeader(t *testing.T) error { @@ -265,6 +381,9 @@ func checkEtcdWithHangLeader(t *testing.T) error { // Start a etcd server. cfg1 := NewTestSingleConfig(t) etcd1, err := embed.StartEtcd(cfg1) + defer func() { + etcd1.Close() + }() re.NoError(err) ep1 := cfg1.LCUrls[0].String() <-etcd1.Server.ReadyNotify() @@ -277,10 +396,13 @@ func checkEtcdWithHangLeader(t *testing.T) error { // Create a etcd client with etcd1 as endpoint. urls, err := types.NewURLs([]string{proxyAddr}) re.NoError(err) - client1, err := createEtcdClient(nil, urls) + client1, err := CreateEtcdClient(nil, urls) + defer func() { + client1.Close() + }() re.NoError(err) - // Add a new member and set the client endpoints to etcd1 and etcd2. + // Add a new member etcd2 := checkAddEtcdMember(t, cfg1, client1) defer etcd2.Close() checkMembers(re, client1, []*embed.Etcd{etcd1, etcd2}) @@ -288,7 +410,7 @@ func checkEtcdWithHangLeader(t *testing.T) error { // Hang the etcd1 and wait for the client to connect to etcd2. enableDiscard.Store(true) - time.Sleep(defaultDialKeepAliveTime + defaultDialKeepAliveTimeout*2) + time.Sleep(time.Second) _, err = EtcdKVGet(client1, "test/key1") return err } @@ -381,3 +503,279 @@ func ioCopy(dst io.Writer, src io.Reader, enableDiscard *atomic.Bool) (err error } return err } + +type loopWatcherTestSuite struct { + suite.Suite + ctx context.Context + cancel context.CancelFunc + wg sync.WaitGroup + cleans []func() + etcd *embed.Etcd + client *clientv3.Client + config *embed.Config +} + +func TestLoopWatcherTestSuite(t *testing.T) { + suite.Run(t, new(loopWatcherTestSuite)) +} + +func (suite *loopWatcherTestSuite) SetupSuite() { + var err error + t := suite.T() + suite.ctx, suite.cancel = context.WithCancel(context.Background()) + suite.cleans = make([]func(), 0) + // Start a etcd server and create a client with etcd1 as endpoint. + suite.config = NewTestSingleConfig(t) + suite.startEtcd() + suite.client, err = CreateEtcdClient(nil, suite.config.LCUrls) + suite.NoError(err) + suite.cleans = append(suite.cleans, func() { + suite.client.Close() + }) +} + +func (suite *loopWatcherTestSuite) TearDownSuite() { + suite.cancel() + suite.wg.Wait() + for _, clean := range suite.cleans { + clean() + } +} + +func (suite *loopWatcherTestSuite) TestLoadWithoutKey() { + cache := struct { + sync.RWMutex + data map[string]struct{} + }{ + data: make(map[string]struct{}), + } + watcher := NewLoopWatcher( + suite.ctx, + &suite.wg, + suite.client, + "test", + "TestLoadWithoutKey", + func(kv *mvccpb.KeyValue) error { + cache.Lock() + defer cache.Unlock() + cache.data[string(kv.Key)] = struct{}{} + return nil + }, + func(kv *mvccpb.KeyValue) error { return nil }, + func() error { return nil }, + ) + watcher.StartWatchLoop() + err := watcher.WaitLoad() + suite.NoError(err) // although no key, watcher returns no error + cache.RLock() + defer cache.RUnlock() + suite.Len(cache.data, 0) +} + +func (suite *loopWatcherTestSuite) TestCallBack() { + cache := struct { + sync.RWMutex + data map[string]struct{} + }{ + data: make(map[string]struct{}), + } + result := make([]string, 0) + watcher := NewLoopWatcher( + suite.ctx, + &suite.wg, + suite.client, + "test", + "TestCallBack", + func(kv *mvccpb.KeyValue) error { + result = append(result, string(kv.Key)) + return nil + }, + func(kv *mvccpb.KeyValue) error { + cache.Lock() + defer cache.Unlock() + delete(cache.data, string(kv.Key)) + return nil + }, + func() error { + cache.Lock() + defer cache.Unlock() + for _, r := range result { + cache.data[r] = struct{}{} + } + result = result[:0] + return nil + }, + clientv3.WithPrefix(), + ) + watcher.StartWatchLoop() + err := watcher.WaitLoad() + suite.NoError(err) + + // put 10 keys + for i := 0; i < 10; i++ { + suite.put(fmt.Sprintf("TestCallBack%d", i), "") + } + time.Sleep(time.Second) + cache.RLock() + suite.Len(cache.data, 10) + cache.RUnlock() + + // delete 10 keys + for i := 0; i < 10; i++ { + key := fmt.Sprintf("TestCallBack%d", i) + _, err = suite.client.Delete(suite.ctx, key) + suite.NoError(err) + } + time.Sleep(time.Second) + cache.RLock() + suite.Empty(cache.data) + cache.RUnlock() +} + +func (suite *loopWatcherTestSuite) TestWatcherLoadLimit() { + for count := 1; count < 10; count++ { + for limit := 0; limit < 10; limit++ { + ctx, cancel := context.WithCancel(suite.ctx) + for i := 0; i < count; i++ { + suite.put(fmt.Sprintf("TestWatcherLoadLimit%d", i), "") + } + cache := struct { + sync.RWMutex + data []string + }{ + data: make([]string, 0), + } + watcher := NewLoopWatcher( + ctx, + &suite.wg, + suite.client, + "test", + "TestWatcherLoadLimit", + func(kv *mvccpb.KeyValue) error { + cache.Lock() + defer cache.Unlock() + cache.data = append(cache.data, string(kv.Key)) + return nil + }, + func(kv *mvccpb.KeyValue) error { + return nil + }, + func() error { + return nil + }, + clientv3.WithPrefix(), + ) + watcher.StartWatchLoop() + err := watcher.WaitLoad() + suite.NoError(err) + cache.RLock() + suite.Len(cache.data, count) + cache.RUnlock() + cancel() + } + } +} + +func (suite *loopWatcherTestSuite) TestWatcherBreak() { + cache := struct { + sync.RWMutex + data string + }{} + checkCache := func(expect string) { + testutil.Eventually(suite.Require(), func() bool { + cache.RLock() + defer cache.RUnlock() + return cache.data == expect + }, testutil.WithWaitFor(time.Second)) + } + + watcher := NewLoopWatcher( + suite.ctx, + &suite.wg, + suite.client, + "test", + "TestWatcherBreak", + func(kv *mvccpb.KeyValue) error { + if string(kv.Key) == "TestWatcherBreak" { + cache.Lock() + defer cache.Unlock() + cache.data = string(kv.Value) + } + return nil + }, + func(kv *mvccpb.KeyValue) error { return nil }, + func() error { return nil }, + ) + watcher.watchChangeRetryInterval = 100 * time.Millisecond + watcher.StartWatchLoop() + err := watcher.WaitLoad() + suite.NoError(err) + checkCache("") + + // we use close client and update client in failpoint to simulate the network error and recover + failpoint.Enable("github.com/tikv/pd/pkg/utils/etcdutil/updateClient", "return(true)") + + // Case1: restart the etcd server + suite.etcd.Close() + suite.startEtcd() + suite.put("TestWatcherBreak", "1") + checkCache("1") + + // Case2: close the etcd client and put a new value after watcher restarts + suite.client.Close() + suite.client, err = CreateEtcdClient(nil, suite.config.LCUrls) + suite.NoError(err) + watcher.updateClientCh <- suite.client + suite.put("TestWatcherBreak", "2") + checkCache("2") + + // Case3: close the etcd client and put a new value before watcher restarts + suite.client.Close() + suite.client, err = CreateEtcdClient(nil, suite.config.LCUrls) + suite.NoError(err) + suite.put("TestWatcherBreak", "3") + watcher.updateClientCh <- suite.client + checkCache("3") + + // Case4: close the etcd client and put a new value with compact + suite.client.Close() + suite.client, err = CreateEtcdClient(nil, suite.config.LCUrls) + suite.NoError(err) + suite.put("TestWatcherBreak", "4") + resp, err := EtcdKVGet(suite.client, "TestWatcherBreak") + suite.NoError(err) + revision := resp.Header.Revision + resp2, err := suite.etcd.Server.Compact(suite.ctx, &etcdserverpb.CompactionRequest{Revision: revision}) + suite.NoError(err) + suite.Equal(revision, resp2.Header.Revision) + watcher.updateClientCh <- suite.client + checkCache("4") + + // Case5: there is an error data in cache + cache.Lock() + cache.data = "error" + cache.Unlock() + watcher.ForceLoad() + checkCache("4") + + failpoint.Disable("github.com/tikv/pd/pkg/utils/etcdutil/updateClient") +} + +func (suite *loopWatcherTestSuite) startEtcd() { + etcd1, err := embed.StartEtcd(suite.config) + suite.NoError(err) + suite.etcd = etcd1 + <-etcd1.Server.ReadyNotify() + suite.cleans = append(suite.cleans, func() { + suite.etcd.Close() + }) +} + +func (suite *loopWatcherTestSuite) put(key, value string) { + kv := clientv3.NewKV(suite.client) + _, err := kv.Put(suite.ctx, key, value) + suite.NoError(err) + resp, err := kv.Get(suite.ctx, key) + suite.NoError(err) + suite.Equal(value, string(resp.Kvs[0].Value)) +} diff --git a/pkg/swaggerserver/swagger_handler.go b/pkg/utils/etcdutil/metrics.go similarity index 60% rename from pkg/swaggerserver/swagger_handler.go rename to pkg/utils/etcdutil/metrics.go index 69cff3d2751..f78e0864ba2 100644 --- a/pkg/swaggerserver/swagger_handler.go +++ b/pkg/utils/etcdutil/metrics.go @@ -1,4 +1,4 @@ -// Copyright 2020 TiKV Project Authors. +// Copyright 2023 TiKV Project Authors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,18 +12,18 @@ // See the License for the specific language governing permissions and // limitations under the License. -//go:build swagger_server -// +build swagger_server +package etcdutil -package swaggerserver +import "github.com/prometheus/client_golang/prometheus" -import ( - "net/http" +var etcdStateGauge = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: "pd", + Subsystem: "server", + Name: "etcd_client", + Help: "Etcd client states.", + }, []string{"type"}) - httpSwagger "github.com/swaggo/http-swagger" - _ "github.com/tikv/pd/docs/swagger" -) - -func handler() http.Handler { - return httpSwagger.Handler() +func init() { + prometheus.MustRegister(etcdStateGauge) } diff --git a/pkg/utils/grpcutil/grpcutil.go b/pkg/utils/grpcutil/grpcutil.go index 5aae6d97e34..78895a07a08 100644 --- a/pkg/utils/grpcutil/grpcutil.go +++ b/pkg/utils/grpcutil/grpcutil.go @@ -21,15 +21,19 @@ import ( "net/url" "github.com/pingcap/log" + "github.com/pkg/errors" "github.com/tikv/pd/pkg/errs" "go.etcd.io/etcd/pkg/transport" + "go.uber.org/zap" "google.golang.org/grpc" "google.golang.org/grpc/credentials" "google.golang.org/grpc/metadata" ) -// ForwardMetadataKey is used to record the forwarded host of PD. -const ForwardMetadataKey = "pd-forwarded-host" +const ( + // ForwardMetadataKey is used to record the forwarded host of PD. + ForwardMetadataKey = "pd-forwarded-host" +) // TLSConfig is the configuration for supporting tls. type TLSConfig struct { @@ -160,3 +164,42 @@ func GetForwardedHost(ctx context.Context) string { } return "" } + +func establish(ctx context.Context, addr string, tlsConfig *TLSConfig, do ...grpc.DialOption) (*grpc.ClientConn, error) { + tlsCfg, err := tlsConfig.ToTLSConfig() + if err != nil { + return nil, err + } + cc, err := GetClientConn( + ctx, + addr, + tlsCfg, + do..., + ) + if err != nil { + return nil, errors.WithStack(err) + } + return cc, nil +} + +// CreateClientConn creates a client connection to the given target. +func CreateClientConn(ctx context.Context, addr string, tlsConfig *TLSConfig, do ...grpc.DialOption) *grpc.ClientConn { + var ( + conn *grpc.ClientConn + err error + ) + for { + select { + case <-ctx.Done(): + return nil + default: + } + conn, err = establish(ctx, addr, tlsConfig, do...) + if err != nil { + log.Error("cannot establish connection", zap.String("addr", addr), errs.ZapError(err)) + continue + } + break + } + return conn +} diff --git a/pkg/utils/grpcutil/grpcutil_test.go b/pkg/utils/grpcutil/grpcutil_test.go index ac3ba3da3fc..ea1de4fb681 100644 --- a/pkg/utils/grpcutil/grpcutil_test.go +++ b/pkg/utils/grpcutil/grpcutil_test.go @@ -24,9 +24,9 @@ func TestToTLSConfig(t *testing.T) { t.Parallel() re := require.New(t) tlsConfig := TLSConfig{ - KeyPath: "../../../tests/client/cert/pd-server-key.pem", - CertPath: "../../../tests/client/cert/pd-server.pem", - CAPath: "../../../tests/client/cert/ca.pem", + KeyPath: "../../../tests/integrations/client/cert/pd-server-key.pem", + CertPath: "../../../tests/integrations/client/cert/pd-server.pem", + CAPath: "../../../tests/integrations/client/cert/ca.pem", } // test without bytes _, err := tlsConfig.ToTLSConfig() diff --git a/pkg/utils/keyutil/util.go b/pkg/utils/keyutil/util.go index 196369bdef2..a5513d4108d 100644 --- a/pkg/utils/keyutil/util.go +++ b/pkg/utils/keyutil/util.go @@ -40,3 +40,29 @@ func MinKey(a, b []byte) []byte { } return a } + +type boundary int + +const ( + left boundary = iota + right +) + +// less returns true if a < b. +// If the key is empty and the boundary is right, the keys is infinite. +func less(a, b []byte, boundary boundary) bool { + ret := bytes.Compare(a, b) + if ret < 0 { + return true + } + if boundary == right && len(b) == 0 && len(a) > 0 { + return true + } + return false +} + +// Between returns true if startKey < key < endKey. +// If the key is empty and the boundary is right, the keys is infinite. +func Between(startKey, endKey, key []byte) bool { + return less(startKey, key, left) && less(key, endKey, right) +} diff --git a/pkg/utils/keyutil/util_test.go b/pkg/utils/keyutil/util_test.go index dc149d9c81e..374faa1f797 100644 --- a/pkg/utils/keyutil/util_test.go +++ b/pkg/utils/keyutil/util_test.go @@ -28,3 +28,104 @@ func TestKeyUtil(t *testing.T) { key := BuildKeyRangeKey(startKey, endKey) re.Equal("61-62", key) } + +func TestLess(t *testing.T) { + re := require.New(t) + TestData := []struct { + a []byte + b []byte + boundary boundary + expect bool + }{ + { + []byte("a"), + []byte("b"), + left, + true, + }, + { + []byte("a"), + []byte("b"), + right, + true, + }, + { + []byte("a"), + []byte(""), + left, + false, + }, + { + []byte("a"), + []byte(""), + right, + true, + }, + { + []byte("a"), + []byte("a"), + right, + false, + }, + { + []byte(""), + []byte(""), + right, + false, + }, + { + []byte(""), + []byte(""), + left, + false, + }, + } + for _, data := range TestData { + re.Equal(data.expect, less(data.a, data.b, data.boundary)) + } +} + +func TestBetween(t *testing.T) { + re := require.New(t) + TestData := []struct { + startKey []byte + endKey []byte + key []byte + + expect bool + }{ + { + []byte("a"), + []byte("c"), + []byte("b"), + true, + }, + { + []byte("a"), + []byte("c"), + []byte("c"), + false, + }, + { + []byte("a"), + []byte(""), + []byte("b"), + true, + }, + { + []byte("a"), + []byte(""), + []byte(""), + false, + }, + { + []byte("a"), + []byte(""), + []byte("a"), + false, + }, + } + for _, data := range TestData { + re.Equal(data.expect, Between(data.startKey, data.endKey, data.key)) + } +} diff --git a/pkg/utils/logutil/log.go b/pkg/utils/logutil/log.go index a46f47da891..abb6a2783a0 100644 --- a/pkg/utils/logutil/log.go +++ b/pkg/utils/logutil/log.go @@ -152,3 +152,13 @@ type stringer struct { func (s stringer) String() string { return "?" } + +// CondUint32 constructs a field with the given key and value conditionally. +// If the condition is true, it constructs a field with uint32 type; otherwise, +// skip the field. +func CondUint32(key string, val uint32, condition bool) zap.Field { + if condition { + return zap.Uint32(key, val) + } + return zap.Skip() +} diff --git a/pkg/utils/metricutil/metricutil.go b/pkg/utils/metricutil/metricutil.go index 3e802fa8952..f0f0220c311 100644 --- a/pkg/utils/metricutil/metricutil.go +++ b/pkg/utils/metricutil/metricutil.go @@ -23,6 +23,7 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/push" "github.com/tikv/pd/pkg/errs" + "github.com/tikv/pd/pkg/utils/logutil" "github.com/tikv/pd/pkg/utils/typeutil" ) @@ -63,6 +64,8 @@ func camelCaseToSnakeCase(str string) string { // prometheusPushClient pushes metrics to Prometheus Pushgateway. func prometheusPushClient(job, addr string, interval time.Duration) { + defer logutil.LogPanic() + pusher := push.New(addr, job). Gatherer(prometheus.DefaultGatherer). Grouping("instance", instanceName()) diff --git a/pkg/utils/testutil/operator_check.go b/pkg/utils/operatorutil/operator_check.go similarity index 99% rename from pkg/utils/testutil/operator_check.go rename to pkg/utils/operatorutil/operator_check.go index ee7aa2a9852..f6517be29d7 100644 --- a/pkg/utils/testutil/operator_check.go +++ b/pkg/utils/operatorutil/operator_check.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package testutil +package operatorutil import ( "github.com/stretchr/testify/require" diff --git a/pkg/utils/syncutil/lock_group.go b/pkg/utils/syncutil/lock_group.go index e6abe21c9e8..a38f490368c 100644 --- a/pkg/utils/syncutil/lock_group.go +++ b/pkg/utils/syncutil/lock_group.go @@ -16,11 +16,17 @@ package syncutil import "fmt" +type lockEntry struct { + mu *Mutex + refCount int +} + // LockGroup is a map of mutex that locks entries with different id separately. // It's used levitate lock contentions of using a global lock. type LockGroup struct { - groupLock Mutex // protects group. - entries map[uint32]*Mutex // map of locks with id as key. + groupLock Mutex // protects group. + removeEntryOnUnlock bool // if remove entry from entries on Unlock(). + entries map[uint32]*lockEntry // map of locks with id as key. // hashFn hashes id to map key, it's main purpose is to limit the total // number of mutexes in the group, as using a mutex for every id is too memory heavy. hashFn func(id uint32) uint32 @@ -36,10 +42,17 @@ func WithHash(hashFn func(id uint32) uint32) LockGroupOption { } } +// WithRemoveEntryOnUnlock sets the lockGroup's removeEntryOnUnlock to provided value. +func WithRemoveEntryOnUnlock(removeEntryOnUnlock bool) LockGroupOption { + return func(lg *LockGroup) { + lg.removeEntryOnUnlock = removeEntryOnUnlock + } +} + // NewLockGroup create and return an empty lockGroup. func NewLockGroup(options ...LockGroupOption) *LockGroup { lockGroup := &LockGroup{ - entries: make(map[uint32]*Mutex), + entries: make(map[uint32]*lockEntry), // If no custom hash function provided, use identity hash. hashFn: func(id uint32) uint32 { return id }, } @@ -56,11 +69,15 @@ func (g *LockGroup) Lock(id uint32) { e, ok := g.entries[hashedID] // If target id's lock has not been initialized, create a new lock. if !ok { - e = &Mutex{} + e = &lockEntry{ + mu: &Mutex{}, + refCount: 0, + } g.entries[hashedID] = e } + e.refCount++ g.groupLock.Unlock() - e.Lock() + e.mu.Lock() } // Unlock unlocks the target mutex based on the hash of the id. @@ -73,6 +90,15 @@ func (g *LockGroup) Unlock(id uint32) { g.groupLock.Unlock() panic(fmt.Errorf("unlock requested for key %v, but no entry found", id)) } + e.refCount-- + if e.refCount == -1 { + // Ref count should never be negative, otherwise there should be a run-time error and panic. + g.groupLock.Unlock() + panic(fmt.Errorf("unlock requested for key %v, but ref count is negative", id)) + } + if g.removeEntryOnUnlock && e.refCount == 0 { + delete(g.entries, hashedID) + } g.groupLock.Unlock() - e.Unlock() + e.mu.Unlock() } diff --git a/pkg/utils/syncutil/lock_group_test.go b/pkg/utils/syncutil/lock_group_test.go index 4e7dd123700..ff306983e05 100644 --- a/pkg/utils/syncutil/lock_group_test.go +++ b/pkg/utils/syncutil/lock_group_test.go @@ -31,7 +31,7 @@ func TestLockGroup(t *testing.T) { for i := 0; i < concurrency; i++ { go func(spaceID uint32) { defer wg.Done() - mustSequentialUpdateSingle(re, spaceID, group) + mustSequentialUpdateSingle(re, spaceID, group, concurrency) }(rand.Uint32()) } wg.Wait() @@ -39,9 +39,39 @@ func TestLockGroup(t *testing.T) { re.LessOrEqual(len(group.entries), 16) } +func TestLockGroupWithRemoveEntryOnUnlock(t *testing.T) { + re := require.New(t) + group := NewLockGroup(WithRemoveEntryOnUnlock(true)) + maxID := 1024 + + // Test Concurrent lock/unlock. + var wg sync.WaitGroup + wg.Add(maxID) + for i := 0; i < maxID; i++ { + go func(spaceID uint32) { + defer wg.Done() + mustSequentialUpdateSingle(re, spaceID, group, 10) + }(uint32(i)) + } + + // Test range lock in a scenario with non-consecutive large key space. One of example is + // keyspace group split loads non-consecutive keyspace meta in batches and lock all loaded + // keyspace meta within a batch at the same time. + for i := 0; i < maxID; i++ { + group.Lock(uint32(i)) + } + re.Equal(len(group.entries), maxID) + for i := 0; i < maxID; i++ { + group.Unlock(uint32(i)) + } + + wg.Wait() + // Check that size of the lock group is limited. + re.Equal(len(group.entries), 0) +} + // mustSequentialUpdateSingle checks that for any given update, update is sequential. -func mustSequentialUpdateSingle(re *require.Assertions, spaceID uint32, group *LockGroup) { - concurrency := 50 +func mustSequentialUpdateSingle(re *require.Assertions, spaceID uint32, group *LockGroup, concurrency int) { total := 0 var wg sync.WaitGroup wg.Add(concurrency) diff --git a/pkg/utils/testutil/testutil.go b/pkg/utils/testutil/testutil.go index abe714384a5..7d31f2263c6 100644 --- a/pkg/utils/testutil/testutil.go +++ b/pkg/utils/testutil/testutil.go @@ -29,6 +29,9 @@ const ( defaultTickInterval = time.Millisecond * 100 ) +// CleanupFunc closes test pd server(s) and deletes any files left behind. +type CleanupFunc func() + // WaitOp represents available options when execute Eventually. type WaitOp struct { waitFor time.Duration @@ -71,10 +74,9 @@ func NewRequestHeader(clusterID uint64) *pdpb.RequestHeader { } } -// MustNewGrpcClient must create a new grpc client. +// MustNewGrpcClient must create a new PD grpc client. func MustNewGrpcClient(re *require.Assertions, addr string) pdpb.PDClient { conn, err := grpc.Dial(strings.TrimPrefix(addr, "http://"), grpc.WithInsecure()) - re.NoError(err) return pdpb.NewPDClient(conn) } diff --git a/pkg/utils/tsoutil/tso_dispatcher.go b/pkg/utils/tsoutil/tso_dispatcher.go new file mode 100644 index 00000000000..f9585ba5cdd --- /dev/null +++ b/pkg/utils/tsoutil/tso_dispatcher.go @@ -0,0 +1,258 @@ +// Copyright 2023 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tsoutil + +import ( + "context" + "strings" + "sync" + "time" + + "github.com/pingcap/kvproto/pkg/pdpb" + "github.com/pingcap/log" + "github.com/prometheus/client_golang/prometheus" + "github.com/tikv/pd/pkg/errs" + "github.com/tikv/pd/pkg/timerpool" + "github.com/tikv/pd/pkg/utils/etcdutil" + "github.com/tikv/pd/pkg/utils/logutil" + "go.uber.org/zap" + "google.golang.org/grpc" +) + +const ( + maxMergeRequests = 10000 + // DefaultTSOProxyTimeout defines the default timeout value of TSP Proxying + DefaultTSOProxyTimeout = 3 * time.Second +) + +type tsoResp interface { + GetTimestamp() *pdpb.Timestamp +} + +// TSODispatcher dispatches the TSO requests to the corresponding forwarding TSO channels. +type TSODispatcher struct { + tsoProxyHandleDuration prometheus.Histogram + tsoProxyBatchSize prometheus.Histogram + + // dispatchChs is used to dispatch different TSO requests to the corresponding forwarding TSO channels. + dispatchChs sync.Map // Store as map[string]chan Request +} + +// NewTSODispatcher creates and returns a TSODispatcher +func NewTSODispatcher(tsoProxyHandleDuration, tsoProxyBatchSize prometheus.Histogram) *TSODispatcher { + tsoDispatcher := &TSODispatcher{ + tsoProxyHandleDuration: tsoProxyHandleDuration, + tsoProxyBatchSize: tsoProxyBatchSize, + } + return tsoDispatcher +} + +// DispatchRequest is the entry point for dispatching/forwarding a tso request to the destination host +func (s *TSODispatcher) DispatchRequest( + ctx context.Context, + req Request, + tsoProtoFactory ProtoFactory, + doneCh <-chan struct{}, + errCh chan<- error, + tsoPrimaryWatchers ...*etcdutil.LoopWatcher) { + val, loaded := s.dispatchChs.LoadOrStore(req.getForwardedHost(), make(chan Request, maxMergeRequests)) + reqCh := val.(chan Request) + if !loaded { + tsDeadlineCh := make(chan *TSDeadline, 1) + go s.dispatch(ctx, tsoProtoFactory, req.getForwardedHost(), req.getClientConn(), reqCh, tsDeadlineCh, doneCh, errCh, tsoPrimaryWatchers...) + go WatchTSDeadline(ctx, tsDeadlineCh) + } + reqCh <- req +} + +func (s *TSODispatcher) dispatch( + ctx context.Context, + tsoProtoFactory ProtoFactory, + forwardedHost string, + clientConn *grpc.ClientConn, + tsoRequestCh <-chan Request, + tsDeadlineCh chan<- *TSDeadline, + doneCh <-chan struct{}, + errCh chan<- error, + tsoPrimaryWatchers ...*etcdutil.LoopWatcher) { + defer logutil.LogPanic() + dispatcherCtx, ctxCancel := context.WithCancel(ctx) + defer ctxCancel() + defer s.dispatchChs.Delete(forwardedHost) + + forwardStream, cancel, err := tsoProtoFactory.createForwardStream(ctx, clientConn) + if err != nil || forwardStream == nil { + log.Error("create tso forwarding stream error", + zap.String("forwarded-host", forwardedHost), + errs.ZapError(errs.ErrGRPCCreateStream, err)) + select { + case <-dispatcherCtx.Done(): + return + case _, ok := <-doneCh: + if !ok { + return + } + case errCh <- err: + close(errCh) + return + } + } + defer cancel() + + requests := make([]Request, maxMergeRequests+1) + needUpdateServicePrimaryAddr := len(tsoPrimaryWatchers) > 0 && tsoPrimaryWatchers[0] != nil + for { + select { + case first := <-tsoRequestCh: + pendingTSOReqCount := len(tsoRequestCh) + 1 + requests[0] = first + for i := 1; i < pendingTSOReqCount; i++ { + requests[i] = <-tsoRequestCh + } + done := make(chan struct{}) + dl := NewTSDeadline(DefaultTSOProxyTimeout, done, cancel) + select { + case tsDeadlineCh <- dl: + case <-dispatcherCtx.Done(): + return + } + err = s.processRequests(forwardStream, requests[:pendingTSOReqCount], tsoProtoFactory) + close(done) + if err != nil { + log.Error("proxy forward tso error", + zap.String("forwarded-host", forwardedHost), + errs.ZapError(errs.ErrGRPCSend, err)) + if needUpdateServicePrimaryAddr && strings.Contains(err.Error(), errs.NotLeaderErr) { + tsoPrimaryWatchers[0].ForceLoad() + } + select { + case <-dispatcherCtx.Done(): + return + case _, ok := <-doneCh: + if !ok { + return + } + case errCh <- err: + close(errCh) + return + } + } + case <-dispatcherCtx.Done(): + return + } + } +} + +func (s *TSODispatcher) processRequests(forwardStream stream, requests []Request, tsoProtoFactory ProtoFactory) error { + // Merge the requests + count := uint32(0) + for _, request := range requests { + count += request.getCount() + } + + start := time.Now() + resp, err := requests[0].process(forwardStream, count, tsoProtoFactory) + if err != nil { + return err + } + s.tsoProxyHandleDuration.Observe(time.Since(start).Seconds()) + s.tsoProxyBatchSize.Observe(float64(count)) + // Split the response + ts := resp.GetTimestamp() + physical, logical, suffixBits := ts.GetPhysical(), ts.GetLogical(), ts.GetSuffixBits() + // `logical` is the largest ts's logical part here, we need to do the subtracting before we finish each TSO request. + // This is different from the logic of client batch, for example, if we have a largest ts whose logical part is 10, + // count is 5, then the splitting results should be 5 and 10. + firstLogical := addLogical(logical, -int64(count), suffixBits) + return s.finishRequest(requests, physical, firstLogical, suffixBits) +} + +// Because of the suffix, we need to shift the count before we add it to the logical part. +func addLogical(logical, count int64, suffixBits uint32) int64 { + return logical + count< -packages=(`go list ./...`) -dirs=(`find . -iname "*_test.go" -exec dirname {} \; | sort -u | sed -e "s/^\./github.com\/tikv\/pd/"`) -tasks=($(comm -12 <(printf "%s\n" "${packages[@]}") <(printf "%s\n" "${dirs[@]}"))) +if [[ $2 -gt 10 ]]; then + # Get integration test list. + makefile_dirs=($(find . -iname "Makefile" -exec dirname {} \; | sort -u)) + submod_dirs=($(find . -iname "go.mod" -exec dirname {} \; | sort -u)) + integration_tasks=$(comm -12 <(printf "%s\n" "${makefile_dirs[@]}") <(printf "%s\n" "${submod_dirs[@]}") | grep "./tests/integrations/*") + # Currently, we only have 3 integration tests, so we can hardcode the task index. + for t in ${integration_tasks[@]}; do + if [[ "$t" = "./tests/integrations/client" && "$2" = 11 ]]; then + printf "%s " "$t" + break + elif [[ "$t" = "./tests/integrations/tso" && "$2" = 12 ]]; then + printf "%s " "$t" + break + elif [[ "$t" = "./tests/integrations/mcs" && "$2" = 13 ]]; then + printf "%s " "$t" + break + fi + done +else + # Get package test list. + packages=($(go list ./...)) + dirs=($(find . -iname "*_test.go" -exec dirname {} \; | sort -u | sed -e "s/^\./github.com\/tikv\/pd/")) + tasks=($(comm -12 <(printf "%s\n" "${packages[@]}") <(printf "%s\n" "${dirs[@]}"))) -weight () { - [[ $1 == "github.com/tikv/pd/server/api" ]] && return 30 - [[ $1 == "github.com/tikv/pd/pkg/schedule" ]] && return 30 - [[ $1 =~ "pd/tests" ]] && return 5 - return 1 -} + weight() { + [[ $1 == "github.com/tikv/pd/server/api" ]] && return 30 + [[ $1 == "github.com/tikv/pd/pkg/schedule" ]] && return 30 + [[ $1 =~ "pd/tests" ]] && return 5 + return 1 + } -scores=(`seq "$1" | xargs -I{} echo 0`) + scores=($(seq "$1" | xargs -I{} echo 0)) -res=() -for t in ${tasks[@]}; do - min_i=0 - for i in ${!scores[@]}; do - [[ ${scores[i]} -lt ${scores[$min_i]} ]] && min_i=$i + res=() + for t in ${tasks[@]}; do + min_i=0 + for i in ${!scores[@]}; do + [[ ${scores[i]} -lt ${scores[$min_i]} ]] && min_i=$i + done + weight $t + scores[$min_i]=$((${scores[$min_i]} + $?)) + [[ $(($min_i + 1)) -eq $2 ]] && res+=($t) done - weight $t - scores[$min_i]=$((${scores[$min_i]} + $?)) - [[ $(($min_i+1)) -eq $2 ]] && res+=($t) -done - -printf "%s " "${res[@]}" + printf "%s " "${res[@]}" +fi diff --git a/server/api/admin.go b/server/api/admin.go index 334d1882a66..c81193f1468 100644 --- a/server/api/admin.go +++ b/server/api/admin.go @@ -21,10 +21,13 @@ import ( "strconv" "github.com/gorilla/mux" + "github.com/pingcap/log" + "github.com/tikv/pd/pkg/core" "github.com/tikv/pd/pkg/errs" "github.com/tikv/pd/pkg/utils/apiutil" "github.com/tikv/pd/server" "github.com/unrolled/render" + "go.uber.org/zap" ) type adminHandler struct { @@ -59,6 +62,43 @@ func (h *adminHandler) DeleteRegionCache(w http.ResponseWriter, r *http.Request) h.rd.JSON(w, http.StatusOK, "The region is removed from server cache.") } +// @Tags admin +// @Summary Remove target region from region cache and storage. +// @Param id path integer true "Region Id" +// @Produce json +// @Success 200 {string} string "The region is removed from server storage." +// @Failure 400 {string} string "The input is invalid." +// @Router /admin/storage/region/{id} [delete] +func (h *adminHandler) DeleteRegionStorage(w http.ResponseWriter, r *http.Request) { + rc := getCluster(r) + vars := mux.Vars(r) + regionIDStr := vars["id"] + regionID, err := strconv.ParseUint(regionIDStr, 10, 64) + if err != nil { + h.rd.JSON(w, http.StatusBadRequest, err.Error()) + return + } + targetRegion := rc.GetRegion(regionID) + if targetRegion == nil { + h.rd.JSON(w, http.StatusBadRequest, "failed to get target region from cache") + return + } + + // Remove region from storage + if err = rc.GetStorage().DeleteRegion(targetRegion.GetMeta()); err != nil { + log.Error("failed to delete region from storage", + zap.Uint64("region-id", targetRegion.GetID()), + zap.Stringer("region-meta", core.RegionToHexMeta(targetRegion.GetMeta())), + errs.ZapError(err)) + h.rd.JSON(w, http.StatusOK, "failed to delete region from storage.") + return + } + // Remove region from cache. + rc.DropCacheRegion(regionID) + + h.rd.JSON(w, http.StatusOK, "The region is removed from server cache and region meta storage.") +} + // @Tags admin // @Summary Drop all regions from cache. // @Produce json diff --git a/server/api/admin_test.go b/server/api/admin_test.go index 9e89de968dc..1f2b386eb98 100644 --- a/server/api/admin_test.go +++ b/server/api/admin_test.go @@ -34,7 +34,7 @@ import ( type adminTestSuite struct { suite.Suite svr *server.Server - cleanup cleanUpFunc + cleanup tu.CleanupFunc urlPrefix string } diff --git a/server/api/checker_test.go b/server/api/checker_test.go index 461d3ac2cb0..d6098b776cf 100644 --- a/server/api/checker_test.go +++ b/server/api/checker_test.go @@ -29,7 +29,7 @@ import ( type checkerTestSuite struct { suite.Suite svr *server.Server - cleanup cleanUpFunc + cleanup tu.CleanupFunc urlPrefix string } diff --git a/server/api/cluster_test.go b/server/api/cluster_test.go index c8ecc12ca8d..01aa6ba5f24 100644 --- a/server/api/cluster_test.go +++ b/server/api/cluster_test.go @@ -21,16 +21,16 @@ import ( "github.com/pingcap/kvproto/pkg/metapb" "github.com/stretchr/testify/suite" + sc "github.com/tikv/pd/pkg/schedule/config" tu "github.com/tikv/pd/pkg/utils/testutil" "github.com/tikv/pd/server" "github.com/tikv/pd/server/cluster" - "github.com/tikv/pd/server/config" ) type clusterTestSuite struct { suite.Suite svr *server.Server - cleanup cleanUpFunc + cleanup tu.CleanupFunc urlPrefix string } @@ -70,7 +70,7 @@ func (suite *clusterTestSuite) TestCluster() { suite.NoError(err) c2 := &metapb.Cluster{} - r := config.ReplicationConfig{ + r := sc.ReplicationConfig{ MaxReplicas: 6, EnablePlacementRules: true, } @@ -98,7 +98,7 @@ func (suite *clusterTestSuite) testGetClusterStatus() { suite.NoError(err) suite.True(status.RaftBootstrapTime.After(now)) suite.False(status.IsInitialized) - suite.svr.SetReplicationConfig(config.ReplicationConfig{MaxReplicas: 1}) + suite.svr.SetReplicationConfig(sc.ReplicationConfig{MaxReplicas: 1}) err = tu.ReadGetJSON(re, testDialClient, url, &status) suite.NoError(err) suite.True(status.RaftBootstrapTime.After(now)) diff --git a/server/api/config.go b/server/api/config.go index 4b01cbfffcd..c63bd953c37 100644 --- a/server/api/config.go +++ b/server/api/config.go @@ -27,6 +27,7 @@ import ( "github.com/pingcap/errcode" "github.com/pingcap/errors" "github.com/pingcap/log" + sc "github.com/tikv/pd/pkg/schedule/config" "github.com/tikv/pd/pkg/utils/apiutil" "github.com/tikv/pd/pkg/utils/jsonutil" "github.com/tikv/pd/pkg/utils/logutil" @@ -36,6 +37,10 @@ import ( "github.com/unrolled/render" ) +// This line is to ensure the package `sc` could always be imported so that +// the swagger could generate the right definitions for the config structs. +var _ *sc.ScheduleConfig = nil + type confHandler struct { svr *server.Server rd *render.Render @@ -161,10 +166,28 @@ func (h *confHandler) updateConfig(cfg *config.Config, key string, value interfa case "cluster-version": return h.updateClusterVersion(value) case "label-property": // TODO: support changing label-property + case "keyspace": + return h.updateKeyspaceConfig(cfg, kp[len(kp)-1], value) } return errors.Errorf("config prefix %s not found", kp[0]) } +func (h *confHandler) updateKeyspaceConfig(config *config.Config, key string, value interface{}) error { + updated, found, err := jsonutil.AddKeyValue(&config.Keyspace, key, value) + if err != nil { + return err + } + + if !found { + return errors.Errorf("config item %s not found", key) + } + + if updated { + err = h.svr.SetKeyspaceConfig(config.Keyspace) + } + return err +} + func (h *confHandler) updateSchedule(config *config.Config, key string, value interface{}) error { updated, found, err := jsonutil.AddKeyValue(&config.Schedule, key, value) if err != nil { @@ -275,7 +298,7 @@ func getConfigMap(cfg map[string]interface{}, key []string, value interface{}) m // @Tags config // @Summary Get schedule config. // @Produce json -// @Success 200 {object} config.ScheduleConfig +// @Success 200 {object} sc.ScheduleConfig // @Router /config/schedule [get] func (h *confHandler) GetScheduleConfig(w http.ResponseWriter, r *http.Request) { cfg := h.svr.GetScheduleConfig() @@ -338,7 +361,7 @@ func (h *confHandler) SetScheduleConfig(w http.ResponseWriter, r *http.Request) // @Tags config // @Summary Get replication config. // @Produce json -// @Success 200 {object} config.ReplicationConfig +// @Success 200 {object} sc.ReplicationConfig // @Router /config/replicate [get] func (h *confHandler) GetReplicationConfig(w http.ResponseWriter, r *http.Request) { h.rd.JSON(w, http.StatusOK, h.svr.GetReplicationConfig()) diff --git a/server/api/config_test.go b/server/api/config_test.go index b7a89553043..fbfb3f94518 100644 --- a/server/api/config_test.go +++ b/server/api/config_test.go @@ -21,6 +21,7 @@ import ( "time" "github.com/stretchr/testify/suite" + sc "github.com/tikv/pd/pkg/schedule/config" tu "github.com/tikv/pd/pkg/utils/testutil" "github.com/tikv/pd/pkg/utils/typeutil" "github.com/tikv/pd/pkg/versioninfo" @@ -31,7 +32,7 @@ import ( type configTestSuite struct { suite.Suite svr *server.Server - cleanup cleanUpFunc + cleanup tu.CleanupFunc urlPrefix string } @@ -165,23 +166,23 @@ func (suite *configTestSuite) TestConfigAll() { func (suite *configTestSuite) TestConfigSchedule() { re := suite.Require() addr := fmt.Sprintf("%s/config/schedule", suite.urlPrefix) - sc := &config.ScheduleConfig{} - suite.NoError(tu.ReadGetJSON(re, testDialClient, addr, sc)) - sc.MaxStoreDownTime.Duration = time.Second - postData, err := json.Marshal(sc) + scheduleConfig := &sc.ScheduleConfig{} + suite.NoError(tu.ReadGetJSON(re, testDialClient, addr, scheduleConfig)) + scheduleConfig.MaxStoreDownTime.Duration = time.Second + postData, err := json.Marshal(scheduleConfig) suite.NoError(err) err = tu.CheckPostJSON(testDialClient, addr, postData, tu.StatusOK(re)) suite.NoError(err) - sc1 := &config.ScheduleConfig{} - suite.NoError(tu.ReadGetJSON(re, testDialClient, addr, sc1)) - suite.Equal(*sc1, *sc) + scheduleConfig1 := &sc.ScheduleConfig{} + suite.NoError(tu.ReadGetJSON(re, testDialClient, addr, scheduleConfig1)) + suite.Equal(*scheduleConfig1, *scheduleConfig) } func (suite *configTestSuite) TestConfigReplication() { re := suite.Require() addr := fmt.Sprintf("%s/config/replicate", suite.urlPrefix) - rc := &config.ReplicationConfig{} + rc := &sc.ReplicationConfig{} err := tu.ReadGetJSON(re, testDialClient, addr, rc) suite.NoError(err) @@ -206,7 +207,7 @@ func (suite *configTestSuite) TestConfigReplication() { err = tu.CheckPostJSON(testDialClient, addr, postData, tu.StatusOK(re)) suite.NoError(err) - rc4 := &config.ReplicationConfig{} + rc4 := &sc.ReplicationConfig{} err = tu.ReadGetJSON(re, testDialClient, addr, rc4) suite.NoError(err) diff --git a/server/api/diagnostic.go b/server/api/diagnostic.go index 6a90baebd94..f83f9c83efb 100644 --- a/server/api/diagnostic.go +++ b/server/api/diagnostic.go @@ -19,8 +19,8 @@ import ( "github.com/gorilla/mux" "github.com/tikv/pd/pkg/errs" + "github.com/tikv/pd/pkg/schedule/schedulers" "github.com/tikv/pd/server" - "github.com/tikv/pd/server/cluster" "github.com/unrolled/render" ) @@ -38,7 +38,7 @@ func newDiagnosticHandler(svr *server.Server, rd *render.Render) *diagnosticHand func (h *diagnosticHandler) GetDiagnosticResult(w http.ResponseWriter, r *http.Request) { name := mux.Vars(r)["name"] - if _, ok := cluster.DiagnosableSummaryFunc[name]; !ok { + if _, ok := schedulers.DiagnosableSummaryFunc[name]; !ok { h.rd.JSON(w, http.StatusBadRequest, errs.ErrSchedulerUndiagnosable.FastGenByArgs(name).Error()) return } diff --git a/server/api/diagnostic_test.go b/server/api/diagnostic_test.go index 3fece41248b..8a39b2e0007 100644 --- a/server/api/diagnostic_test.go +++ b/server/api/diagnostic_test.go @@ -27,14 +27,13 @@ import ( "github.com/tikv/pd/pkg/utils/apiutil" tu "github.com/tikv/pd/pkg/utils/testutil" "github.com/tikv/pd/server" - "github.com/tikv/pd/server/cluster" "github.com/tikv/pd/server/config" ) type diagnosticTestSuite struct { suite.Suite svr *server.Server - cleanup cleanUpFunc + cleanup tu.CleanupFunc urlPrefix string configPrefix string schedulerPrifex string @@ -66,7 +65,7 @@ func (suite *diagnosticTestSuite) TearDownSuite() { func (suite *diagnosticTestSuite) checkStatus(status string, url string) { re := suite.Require() suite.Eventually(func() bool { - result := &cluster.DiagnosticResult{} + result := &schedulers.DiagnosticResult{} err := tu.ReadGetJSON(re, testDialClient, url, result) suite.NoError(err) return result.Status == status @@ -81,7 +80,7 @@ func (suite *diagnosticTestSuite) TestSchedulerDiagnosticAPI() { suite.NoError(err) suite.NoError(tu.ReadGetJSON(re, testDialClient, addr, cfg)) - suite.False(cfg.Schedule.EnableDiagnostic) + suite.True(cfg.Schedule.EnableDiagnostic) ms := map[string]interface{}{ "enable-diagnostic": "true", @@ -95,7 +94,7 @@ func (suite *diagnosticTestSuite) TestSchedulerDiagnosticAPI() { suite.True(cfg.Schedule.EnableDiagnostic) balanceRegionURL := suite.urlPrefix + "/" + schedulers.BalanceRegionName - result := &cluster.DiagnosticResult{} + result := &schedulers.DiagnosticResult{} err = tu.ReadGetJSON(re, testDialClient, balanceRegionURL, result) suite.NoError(err) suite.Equal("disabled", result.Status) diff --git a/server/api/hot_status.go b/server/api/hot_status.go index 23540f9499f..749a371300a 100644 --- a/server/api/hot_status.go +++ b/server/api/hot_status.go @@ -21,7 +21,9 @@ import ( "net/http" "strconv" + "github.com/tikv/pd/pkg/core" "github.com/tikv/pd/pkg/statistics" + "github.com/tikv/pd/pkg/statistics/buckets" "github.com/tikv/pd/pkg/storage" "github.com/tikv/pd/server" "github.com/unrolled/render" @@ -32,6 +34,32 @@ type hotStatusHandler struct { rd *render.Render } +// HotBucketsResponse is the response for hot buckets. +type HotBucketsResponse map[uint64][]*HotBucketsItem + +// HotBucketsItem is the item of hot buckets. +type HotBucketsItem struct { + StartKey string `json:"start_key"` + EndKey string `json:"end_key"` + HotDegree int `json:"hot_degree"` + ReadBytes uint64 `json:"read_bytes"` + ReadKeys uint64 `json:"read_keys"` + WriteBytes uint64 `json:"write_bytes"` + WriteKeys uint64 `json:"write_keys"` +} + +func convert(buckets *buckets.BucketStat) *HotBucketsItem { + return &HotBucketsItem{ + StartKey: core.HexRegionKeyStr(buckets.StartKey), + EndKey: core.HexRegionKeyStr(buckets.EndKey), + HotDegree: buckets.HotDegree, + ReadBytes: buckets.Loads[statistics.RegionReadBytes], + ReadKeys: buckets.Loads[statistics.RegionReadKeys], + WriteBytes: buckets.Loads[statistics.RegionWriteBytes], + WriteKeys: buckets.Loads[statistics.RegionWriteKeys], + } +} + // HotStoreStats is used to record the status of hot stores. type HotStoreStats struct { BytesWriteStats map[uint64]float64 `json:"bytes-write-rate,omitempty"` @@ -169,6 +197,30 @@ func (h *hotStatusHandler) GetHotStores(w http.ResponseWriter, r *http.Request) h.rd.JSON(w, http.StatusOK, stats) } +// @Tags hotspot +// @Summary List the hot buckets. +// @Produce json +// @Success 200 {object} HotBucketsResponse +// @Router /hotspot/buckets [get] +func (h *hotStatusHandler) GetHotBuckets(w http.ResponseWriter, r *http.Request) { + regionIDs := r.URL.Query()["region_id"] + ids := make([]uint64, len(regionIDs)) + for i, regionID := range regionIDs { + if id, err := strconv.ParseUint(regionID, 10, 64); err == nil { + ids[i] = id + } + } + stats := h.Handler.GetHotBuckets(ids...) + ret := HotBucketsResponse{} + for regionID, stats := range stats { + ret[regionID] = make([]*HotBucketsItem, len(stats)) + for i, stat := range stats { + ret[regionID][i] = convert(stat) + } + } + h.rd.JSON(w, http.StatusOK, ret) +} + // @Tags hotspot // @Summary List the history hot regions. // @Accept json @@ -190,7 +242,7 @@ func (h *hotStatusHandler) GetHistoryHotRegions(w http.ResponseWriter, r *http.R h.rd.JSON(w, http.StatusBadRequest, err.Error()) return } - results, err := getAllRequestHistroyHotRegion(h.Handler, historyHotRegionsRequest) + results, err := getAllRequestHistoryHotRegion(h.Handler, historyHotRegionsRequest) if err != nil { h.rd.JSON(w, http.StatusInternalServerError, err.Error()) return @@ -198,7 +250,7 @@ func (h *hotStatusHandler) GetHistoryHotRegions(w http.ResponseWriter, r *http.R h.rd.JSON(w, http.StatusOK, results) } -func getAllRequestHistroyHotRegion(handler *server.Handler, request *HistoryHotRegionsRequest) (*storage.HistoryHotRegions, error) { +func getAllRequestHistoryHotRegion(handler *server.Handler, request *HistoryHotRegionsRequest) (*storage.HistoryHotRegions, error) { var hotRegionTypes = storage.HotRegionTypes if len(request.HotRegionTypes) != 0 { hotRegionTypes = request.HotRegionTypes diff --git a/server/api/hot_status_test.go b/server/api/hot_status_test.go index 1582d62d703..a1d1bbc2617 100644 --- a/server/api/hot_status_test.go +++ b/server/api/hot_status_test.go @@ -31,7 +31,7 @@ import ( type hotStatusTestSuite struct { suite.Suite svr *server.Server - cleanup cleanUpFunc + cleanup tu.CleanupFunc urlPrefix string } diff --git a/server/api/label_test.go b/server/api/label_test.go index 7d9155832ae..9bcd40aac01 100644 --- a/server/api/label_test.go +++ b/server/api/label_test.go @@ -30,7 +30,7 @@ import ( type labelsStoreTestSuite struct { suite.Suite svr *server.Server - cleanup cleanUpFunc + cleanup tu.CleanupFunc urlPrefix string stores []*metapb.Store } @@ -190,7 +190,7 @@ type strictlyLabelsStoreTestSuite struct { suite.Suite svr *server.Server grpcSvr *server.GrpcServer - cleanup cleanUpFunc + cleanup tu.CleanupFunc urlPrefix string } diff --git a/server/api/log_test.go b/server/api/log_test.go index ce70b3b4dc2..27dceb52eca 100644 --- a/server/api/log_test.go +++ b/server/api/log_test.go @@ -28,7 +28,7 @@ import ( type logTestSuite struct { suite.Suite svr *server.Server - cleanup cleanUpFunc + cleanup tu.CleanupFunc urlPrefix string } diff --git a/server/api/member.go b/server/api/member.go index 173ecc545a5..3016b76088b 100644 --- a/server/api/member.go +++ b/server/api/member.go @@ -70,9 +70,12 @@ func getMembers(svr *server.Server) (*pdpb.GetMembersResponse, error) { if members.GetHeader().GetError() != nil { return nil, errors.WithStack(errors.New(members.GetHeader().GetError().String())) } - dclocationDistribution, err := svr.GetTSOAllocatorManager().GetClusterDCLocationsFromEtcd() - if err != nil { - return nil, errors.WithStack(err) + dclocationDistribution := make(map[string][]uint64) + if !svr.IsAPIServiceMode() { + dclocationDistribution, err = svr.GetTSOAllocatorManager().GetClusterDCLocationsFromEtcd() + if err != nil { + return nil, errors.WithStack(err) + } } for _, m := range members.GetMembers() { var e error diff --git a/server/api/member_test.go b/server/api/member_test.go index 1132010319d..eba2763704c 100644 --- a/server/api/member_test.go +++ b/server/api/member_test.go @@ -28,6 +28,7 @@ import ( "github.com/pingcap/kvproto/pkg/pdpb" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" + "github.com/tikv/pd/pkg/utils/testutil" "github.com/tikv/pd/server" "github.com/tikv/pd/server/config" ) @@ -36,7 +37,7 @@ type memberTestSuite struct { suite.Suite cfgs []*config.Config servers []*server.Server - clean func() + clean testutil.CleanupFunc } func TestMemberTestSuite(t *testing.T) { @@ -158,7 +159,7 @@ type resignTestSuite struct { suite.Suite cfgs []*config.Config servers []*server.Server - clean func() + clean testutil.CleanupFunc } func TestResignTestSuite(t *testing.T) { diff --git a/server/api/min_resolved_ts.go b/server/api/min_resolved_ts.go index 2e03646c809..0d30ea3395e 100644 --- a/server/api/min_resolved_ts.go +++ b/server/api/min_resolved_ts.go @@ -16,7 +16,9 @@ package api import ( "net/http" + "strconv" + "github.com/gorilla/mux" "github.com/tikv/pd/pkg/utils/typeutil" "github.com/tikv/pd/server" "github.com/unrolled/render" @@ -41,6 +43,30 @@ type minResolvedTS struct { PersistInterval typeutil.Duration `json:"persist_interval,omitempty"` } +// @Tags min_store_resolved_ts +// @Summary Get store-level min resolved ts. +// @Produce json +// @Success 200 {array} minResolvedTS +// @Failure 400 {string} string "The input is invalid." +// @Failure 500 {string} string "PD server failed to proceed the request." +// @Router /min-resolved-ts/{store_id} [get] +func (h *minResolvedTSHandler) GetStoreMinResolvedTS(w http.ResponseWriter, r *http.Request) { + c := h.svr.GetRaftCluster() + idStr := mux.Vars(r)["store_id"] + storeID, err := strconv.ParseUint(idStr, 10, 64) + if err != nil { + h.rd.JSON(w, http.StatusBadRequest, err.Error()) + return + } + value := c.GetStoreMinResolvedTS(storeID) + persistInterval := c.GetPDServerConfig().MinResolvedTSPersistenceInterval + h.rd.JSON(w, http.StatusOK, minResolvedTS{ + MinResolvedTS: value, + PersistInterval: persistInterval, + IsRealTime: persistInterval.Duration != 0, + }) +} + // @Tags min_resolved_ts // @Summary Get cluster-level min resolved ts. // @Produce json diff --git a/server/api/min_resolved_ts_test.go b/server/api/min_resolved_ts_test.go index e0d18ba4989..79ab71e2be1 100644 --- a/server/api/min_resolved_ts_test.go +++ b/server/api/min_resolved_ts_test.go @@ -24,6 +24,7 @@ import ( "github.com/stretchr/testify/suite" "github.com/tikv/pd/pkg/core" "github.com/tikv/pd/pkg/utils/apiutil" + "github.com/tikv/pd/pkg/utils/testutil" "github.com/tikv/pd/pkg/utils/typeutil" "github.com/tikv/pd/server" "github.com/tikv/pd/server/cluster" @@ -32,7 +33,7 @@ import ( type minResolvedTSTestSuite struct { suite.Suite svr *server.Server - cleanup cleanUpFunc + cleanup testutil.CleanupFunc url string defaultInterval time.Duration } diff --git a/server/api/operator.go b/server/api/operator.go index c166fbdff9d..6645a601fb0 100644 --- a/server/api/operator.go +++ b/server/api/operator.go @@ -44,7 +44,7 @@ func newOperatorHandler(handler *server.Handler, r *render.Render) *operatorHand // @Summary Get a Region's pending operator. // @Param region_id path int true "A Region's Id" // @Produce json -// @Success 200 {object} schedule.OperatorWithStatus +// @Success 200 {object} operator.OpWithStatus // @Failure 400 {string} string "The input is invalid." // @Failure 500 {string} string "PD server failed to proceed the request." // @Router /operators/{region_id} [get] diff --git a/server/api/operator_test.go b/server/api/operator_test.go index 4cc0ffb5237..ddb605c7d87 100644 --- a/server/api/operator_test.go +++ b/server/api/operator_test.go @@ -43,7 +43,7 @@ import ( type operatorTestSuite struct { suite.Suite svr *server.Server - cleanup cleanUpFunc + cleanup tu.CleanupFunc urlPrefix string } @@ -166,7 +166,7 @@ func (suite *operatorTestSuite) TestMergeRegionOperator() { type transferRegionOperatorTestSuite struct { suite.Suite svr *server.Server - cleanup cleanUpFunc + cleanup tu.CleanupFunc urlPrefix string } @@ -217,7 +217,13 @@ func (suite *transferRegionOperatorTestSuite) TestTransferRegionWithPlacementRul regionURL := fmt.Sprintf("%s/operators/%d", suite.urlPrefix, region.GetId()) operator := mustReadURL(re, regionURL) suite.Contains(operator, "operator not found") - + convertStepsToStr := func(steps []string) string { + stepStrs := make([]string, len(steps)) + for i := range steps { + stepStrs[i] = fmt.Sprintf("%d:{%s}", i, steps[i]) + } + return strings.Join(stepStrs, ", ") + } testCases := []struct { name string placementRuleEnable bool @@ -231,25 +237,25 @@ func (suite *transferRegionOperatorTestSuite) TestTransferRegionWithPlacementRul placementRuleEnable: false, input: []byte(`{"name":"transfer-region", "region_id": 1, "to_store_ids": [2, 3]}`), expectedError: nil, - expectSteps: strings.Join([]string{ + expectSteps: convertStepsToStr([]string{ pdoperator.AddLearner{ToStore: 3, PeerID: 1}.String(), pdoperator.PromoteLearner{ToStore: 3, PeerID: 1}.String(), pdoperator.TransferLeader{FromStore: 1, ToStore: 2}.String(), pdoperator.RemovePeer{FromStore: 1, PeerID: 1}.String(), - }, ", "), + }), }, { name: "placement rule disable with peer role", placementRuleEnable: false, input: []byte(`{"name":"transfer-region", "region_id": 1, "to_store_ids": [2, 3], "peer_roles":["follower", "leader"]}`), expectedError: nil, - expectSteps: strings.Join([]string{ + expectSteps: convertStepsToStr([]string{ pdoperator.AddLearner{ToStore: 3, PeerID: 2}.String(), pdoperator.PromoteLearner{ToStore: 3, PeerID: 2}.String(), pdoperator.TransferLeader{FromStore: 1, ToStore: 2}.String(), pdoperator.RemovePeer{FromStore: 1, PeerID: 2}.String(), pdoperator.TransferLeader{FromStore: 2, ToStore: 3}.String(), - }, ", "), + }), }, { name: "default placement rule without peer role", @@ -262,13 +268,13 @@ func (suite *transferRegionOperatorTestSuite) TestTransferRegionWithPlacementRul name: "default placement rule with peer role", placementRuleEnable: true, input: []byte(`{"name":"transfer-region", "region_id": 1, "to_store_ids": [2, 3], "peer_roles":["follower", "leader"]}`), - expectSteps: strings.Join([]string{ + expectSteps: convertStepsToStr([]string{ pdoperator.AddLearner{ToStore: 3, PeerID: 3}.String(), pdoperator.PromoteLearner{ToStore: 3, PeerID: 3}.String(), pdoperator.TransferLeader{FromStore: 1, ToStore: 2}.String(), pdoperator.RemovePeer{FromStore: 1, PeerID: 1}.String(), pdoperator.TransferLeader{FromStore: 2, ToStore: 3}.String(), - }, ", "), + }), }, { name: "default placement rule with invalid input", @@ -323,12 +329,12 @@ func (suite *transferRegionOperatorTestSuite) TestTransferRegionWithPlacementRul }, input: []byte(`{"name":"transfer-region", "region_id": 1, "to_store_ids": [2, 3], "peer_roles":["follower", "leader"]}`), expectedError: nil, - expectSteps: strings.Join([]string{ + expectSteps: convertStepsToStr([]string{ pdoperator.AddLearner{ToStore: 3, PeerID: 5}.String(), pdoperator.PromoteLearner{ToStore: 3, PeerID: 5}.String(), pdoperator.TransferLeader{FromStore: 1, ToStore: 3}.String(), pdoperator.RemovePeer{FromStore: 1, PeerID: 1}.String(), - }, ", "), + }), }, { name: "customized placement rule with valid peer role2", @@ -363,12 +369,12 @@ func (suite *transferRegionOperatorTestSuite) TestTransferRegionWithPlacementRul }, input: []byte(`{"name":"transfer-region", "region_id": 1, "to_store_ids": [2, 3], "peer_roles":["leader", "follower"]}`), expectedError: nil, - expectSteps: strings.Join([]string{ + expectSteps: convertStepsToStr([]string{ pdoperator.AddLearner{ToStore: 3, PeerID: 6}.String(), pdoperator.PromoteLearner{ToStore: 3, PeerID: 6}.String(), pdoperator.TransferLeader{FromStore: 1, ToStore: 2}.String(), pdoperator.RemovePeer{FromStore: 1, PeerID: 1}.String(), - }, ", "), + }), }, } for _, testCase := range testCases { diff --git a/server/api/plugin.go b/server/api/plugin.go index 192310cca7e..fd75cc6bb2b 100644 --- a/server/api/plugin.go +++ b/server/api/plugin.go @@ -19,9 +19,9 @@ import ( "net/http" "os" + "github.com/tikv/pd/pkg/schedule" "github.com/tikv/pd/pkg/utils/apiutil" "github.com/tikv/pd/server" - "github.com/tikv/pd/server/cluster" "github.com/unrolled/render" ) @@ -48,7 +48,7 @@ func newPluginHandler(handler *server.Handler, rd *render.Render) *pluginHandler // @Failure 500 {string} string "PD server failed to proceed the request." // @Router /plugin [post] func (h *pluginHandler) LoadPlugin(w http.ResponseWriter, r *http.Request) { - h.processPluginCommand(w, r, cluster.PluginLoad) + h.processPluginCommand(w, r, schedule.PluginLoad) } // FIXME: details of input json body params @@ -62,7 +62,7 @@ func (h *pluginHandler) LoadPlugin(w http.ResponseWriter, r *http.Request) { // @Failure 500 {string} string "PD server failed to proceed the request." // @Router /plugin [delete] func (h *pluginHandler) UnloadPlugin(w http.ResponseWriter, r *http.Request) { - h.processPluginCommand(w, r, cluster.PluginUnload) + h.processPluginCommand(w, r, schedule.PluginUnload) } func (h *pluginHandler) processPluginCommand(w http.ResponseWriter, r *http.Request, action string) { @@ -77,14 +77,14 @@ func (h *pluginHandler) processPluginCommand(w http.ResponseWriter, r *http.Requ } var err error switch action { - case cluster.PluginLoad: + case schedule.PluginLoad: err = h.PluginLoad(path) if err != nil { h.rd.JSON(w, http.StatusInternalServerError, err.Error()) return } h.rd.JSON(w, http.StatusOK, "Load plugin successfully.") - case cluster.PluginUnload: + case schedule.PluginUnload: err = h.PluginUnload(path) if err != nil { h.rd.JSON(w, http.StatusInternalServerError, err.Error()) diff --git a/server/api/pprof.go b/server/api/pprof.go index b64278a21b7..900c48f8368 100644 --- a/server/api/pprof.go +++ b/server/api/pprof.go @@ -209,8 +209,10 @@ func (h *pprofHandler) PProfThreadcreate(w http.ResponseWriter, r *http.Request) } func sleepWithCtx(ctx context.Context, d time.Duration) { + timer := time.NewTimer(d) + defer timer.Stop() select { - case <-time.After(d): + case <-timer.C: case <-ctx.Done(): } } diff --git a/server/api/pprof_test.go b/server/api/pprof_test.go index 9965dbed73b..43ad7ea5978 100644 --- a/server/api/pprof_test.go +++ b/server/api/pprof_test.go @@ -21,13 +21,14 @@ import ( "testing" "github.com/stretchr/testify/suite" + "github.com/tikv/pd/pkg/utils/testutil" "github.com/tikv/pd/server" ) type profTestSuite struct { suite.Suite svr *server.Server - cleanup cleanUpFunc + cleanup testutil.CleanupFunc urlPrefix string } diff --git a/server/api/region.go b/server/api/region.go index 69618b1a82c..1c21af53296 100644 --- a/server/api/region.go +++ b/server/api/region.go @@ -16,20 +16,24 @@ package api import ( "container/heap" + "context" "encoding/hex" "fmt" "net/http" "net/url" "sort" "strconv" + "strings" "github.com/gorilla/mux" + jwriter "github.com/mailru/easyjson/jwriter" "github.com/pingcap/failpoint" "github.com/pingcap/kvproto/pkg/metapb" "github.com/pingcap/kvproto/pkg/pdpb" "github.com/pingcap/kvproto/pkg/replication_modepb" "github.com/pingcap/log" "github.com/tikv/pd/pkg/core" + "github.com/tikv/pd/pkg/keyspace" "github.com/tikv/pd/pkg/schedule/filter" "github.com/tikv/pd/pkg/statistics" "github.com/tikv/pd/pkg/utils/apiutil" @@ -52,6 +56,17 @@ type MetaPeer struct { IsLearner bool `json:"is_learner,omitempty"` } +func (m *MetaPeer) setDefaultIfNil() { + if m.Peer == nil { + m.Peer = &metapb.Peer{ + Id: m.GetId(), + StoreId: m.GetStoreId(), + Role: m.GetRole(), + IsWitness: m.GetIsWitness(), + } + } +} + // PDPeerStats is api compatible with *pdpb.PeerStats. // NOTE: This type is exported by HTTP API. Please pay more attention when modifying it. type PDPeerStats struct { @@ -59,6 +74,16 @@ type PDPeerStats struct { Peer MetaPeer `json:"peer"` } +func (s *PDPeerStats) setDefaultIfNil() { + if s.PeerStats == nil { + s.PeerStats = &pdpb.PeerStats{ + Peer: s.GetPeer(), + DownSeconds: s.GetDownSeconds(), + } + } + s.Peer.setDefaultIfNil() +} + func fromPeer(peer *metapb.Peer) MetaPeer { if peer == nil { return MetaPeer{} @@ -101,6 +126,7 @@ func fromPeerStatsSlice(peers []*pdpb.PeerStats) []PDPeerStats { // RegionInfo records detail region info for api usage. // NOTE: This type is exported by HTTP API. Please pay more attention when modifying it. +// easyjson:json type RegionInfo struct { ID uint64 `json:"id"` StartKey string `json:"start_key"` @@ -167,9 +193,9 @@ func InitRegion(r *core.RegionInfo, s *RegionInfo) *RegionInfo { s.ApproximateSize = r.GetApproximateSize() s.ApproximateKeys = r.GetApproximateKeys() s.ReplicationStatus = fromPBReplicationStatus(r.GetReplicationStatus()) + s.Buckets = nil keys := r.GetBuckets().GetKeys() - if len(keys) > 0 { s.Buckets = make([]string, len(keys)) for i, key := range keys { @@ -311,15 +337,48 @@ func newRegionsHandler(svr *server.Server, rd *render.Render) *regionsHandler { } } -func convertToAPIRegions(regions []*core.RegionInfo) *RegionsInfo { - regionInfos := make([]RegionInfo, len(regions)) +// marshalRegionsInfoJSON marshals regions to bytes in `RegionsInfo`'s JSON format. +// It is used to reduce the cost of JSON serialization. +func marshalRegionsInfoJSON(ctx context.Context, regions []*core.RegionInfo) ([]byte, error) { + out := &jwriter.Writer{} + out.RawByte('{') + + out.RawString("\"count\":") + out.Int(len(regions)) + + out.RawString(",\"regions\":") + out.RawByte('[') + region := &RegionInfo{} for i, r := range regions { - InitRegion(r, ®ionInfos[i]) - } - return &RegionsInfo{ - Count: len(regions), - Regions: regionInfos, + select { + case <-ctx.Done(): + // Return early, avoid the unnecessary computation. + // See more details in https://github.com/tikv/pd/issues/6835 + return nil, ctx.Err() + default: + } + if i > 0 { + out.RawByte(',') + } + InitRegion(r, region) + // EasyJSON will not check anonymous struct pointer field and will panic if the field is nil. + // So we need to set the field to default value explicitly when the anonymous struct pointer is nil. + region.Leader.setDefaultIfNil() + for i := range region.Peers { + region.Peers[i].setDefaultIfNil() + } + for i := range region.PendingPeers { + region.PendingPeers[i].setDefaultIfNil() + } + for i := range region.DownPeers { + region.DownPeers[i].setDefaultIfNil() + } + region.MarshalEasyJSON(out) } + out.RawByte(']') + + out.RawByte('}') + return out.Buffer.BuildBytes(), out.Error } // @Tags region @@ -330,8 +389,12 @@ func convertToAPIRegions(regions []*core.RegionInfo) *RegionsInfo { func (h *regionsHandler) GetRegions(w http.ResponseWriter, r *http.Request) { rc := getCluster(r) regions := rc.GetRegions() - regionsInfo := convertToAPIRegions(regions) - h.rd.JSON(w, http.StatusOK, regionsInfo) + b, err := marshalRegionsInfoJSON(r.Context(), regions) + if err != nil { + h.rd.JSON(w, http.StatusInternalServerError, err.Error()) + return + } + h.rd.Data(w, http.StatusOK, b) } // @Tags region @@ -361,8 +424,12 @@ func (h *regionsHandler) ScanRegions(w http.ResponseWriter, r *http.Request) { limit = maxRegionLimit } regions := rc.ScanRegions([]byte(startKey), []byte(endKey), limit) - regionsInfo := convertToAPIRegions(regions) - h.rd.JSON(w, http.StatusOK, regionsInfo) + b, err := marshalRegionsInfoJSON(r.Context(), regions) + if err != nil { + h.rd.JSON(w, http.StatusInternalServerError, err.Error()) + return + } + h.rd.Data(w, http.StatusOK, b) } // @Tags region @@ -372,7 +439,7 @@ func (h *regionsHandler) ScanRegions(w http.ResponseWriter, r *http.Request) { // @Router /regions/count [get] func (h *regionsHandler) GetRegionCount(w http.ResponseWriter, r *http.Request) { rc := getCluster(r) - count := rc.GetRegionCount() + count := rc.GetTotalRegionCount() h.rd.JSON(w, http.StatusOK, &RegionsInfo{Count: count}) } @@ -393,8 +460,66 @@ func (h *regionsHandler) GetStoreRegions(w http.ResponseWriter, r *http.Request) return } regions := rc.GetStoreRegions(uint64(id)) - regionsInfo := convertToAPIRegions(regions) - h.rd.JSON(w, http.StatusOK, regionsInfo) + b, err := marshalRegionsInfoJSON(r.Context(), regions) + if err != nil { + h.rd.JSON(w, http.StatusInternalServerError, err.Error()) + return + } + h.rd.Data(w, http.StatusOK, b) +} + +// @Tags region +// @Summary List regions belongs to the given keyspace ID. +// @Param keyspace_id query string true "Keyspace ID" +// @Param limit query integer false "Limit count" default(16) +// @Produce json +// @Success 200 {object} RegionsInfo +// @Failure 400 {string} string "The input is invalid." +// @Router /regions/keyspace/id/{id} [get] +func (h *regionsHandler) GetKeyspaceRegions(w http.ResponseWriter, r *http.Request) { + rc := getCluster(r) + vars := mux.Vars(r) + keyspaceIDStr := vars["id"] + if keyspaceIDStr == "" { + h.rd.JSON(w, http.StatusBadRequest, "keyspace id is empty") + return + } + + keyspaceID64, err := strconv.ParseUint(keyspaceIDStr, 10, 32) + if err != nil { + h.rd.JSON(w, http.StatusBadRequest, err.Error()) + return + } + keyspaceID := uint32(keyspaceID64) + keyspaceManager := h.svr.GetKeyspaceManager() + if _, err := keyspaceManager.LoadKeyspaceByID(keyspaceID); err != nil { + h.rd.JSON(w, http.StatusBadRequest, err.Error()) + return + } + + limit := defaultRegionLimit + if limitStr := r.URL.Query().Get("limit"); limitStr != "" { + limit, err = strconv.Atoi(limitStr) + if err != nil { + h.rd.JSON(w, http.StatusBadRequest, err.Error()) + return + } + } + if limit > maxRegionLimit { + limit = maxRegionLimit + } + regionBound := keyspace.MakeRegionBound(keyspaceID) + regions := rc.ScanRegions(regionBound.RawLeftBound, regionBound.RawRightBound, limit) + if limit <= 0 || limit > len(regions) { + txnRegion := rc.ScanRegions(regionBound.TxnLeftBound, regionBound.TxnRightBound, limit-len(regions)) + regions = append(regions, txnRegion...) + } + b, err := marshalRegionsInfoJSON(r.Context(), regions) + if err != nil { + h.rd.JSON(w, http.StatusInternalServerError, err.Error()) + return + } + h.rd.Data(w, http.StatusOK, b) } // @Tags region @@ -404,14 +529,26 @@ func (h *regionsHandler) GetStoreRegions(w http.ResponseWriter, r *http.Request) // @Failure 500 {string} string "PD server failed to proceed the request." // @Router /regions/check/miss-peer [get] func (h *regionsHandler) GetMissPeerRegions(w http.ResponseWriter, r *http.Request) { + h.getRegionsByType(w, statistics.MissPeer, r) +} + +func (h *regionsHandler) getRegionsByType( + w http.ResponseWriter, + typ statistics.RegionStatisticType, + r *http.Request, +) { handler := h.svr.GetHandler() - regions, err := handler.GetRegionsByType(statistics.MissPeer) + regions, err := handler.GetRegionsByType(typ) + if err != nil { + h.rd.JSON(w, http.StatusInternalServerError, err.Error()) + return + } + b, err := marshalRegionsInfoJSON(r.Context(), regions) if err != nil { h.rd.JSON(w, http.StatusInternalServerError, err.Error()) return } - regionsInfo := convertToAPIRegions(regions) - h.rd.JSON(w, http.StatusOK, regionsInfo) + h.rd.Data(w, http.StatusOK, b) } // @Tags region @@ -421,14 +558,7 @@ func (h *regionsHandler) GetMissPeerRegions(w http.ResponseWriter, r *http.Reque // @Failure 500 {string} string "PD server failed to proceed the request." // @Router /regions/check/extra-peer [get] func (h *regionsHandler) GetExtraPeerRegions(w http.ResponseWriter, r *http.Request) { - handler := h.svr.GetHandler() - regions, err := handler.GetRegionsByType(statistics.ExtraPeer) - if err != nil { - h.rd.JSON(w, http.StatusInternalServerError, err.Error()) - return - } - regionsInfo := convertToAPIRegions(regions) - h.rd.JSON(w, http.StatusOK, regionsInfo) + h.getRegionsByType(w, statistics.ExtraPeer, r) } // @Tags region @@ -438,14 +568,7 @@ func (h *regionsHandler) GetExtraPeerRegions(w http.ResponseWriter, r *http.Requ // @Failure 500 {string} string "PD server failed to proceed the request." // @Router /regions/check/pending-peer [get] func (h *regionsHandler) GetPendingPeerRegions(w http.ResponseWriter, r *http.Request) { - handler := h.svr.GetHandler() - regions, err := handler.GetRegionsByType(statistics.PendingPeer) - if err != nil { - h.rd.JSON(w, http.StatusInternalServerError, err.Error()) - return - } - regionsInfo := convertToAPIRegions(regions) - h.rd.JSON(w, http.StatusOK, regionsInfo) + h.getRegionsByType(w, statistics.PendingPeer, r) } // @Tags region @@ -455,14 +578,7 @@ func (h *regionsHandler) GetPendingPeerRegions(w http.ResponseWriter, r *http.Re // @Failure 500 {string} string "PD server failed to proceed the request." // @Router /regions/check/down-peer [get] func (h *regionsHandler) GetDownPeerRegions(w http.ResponseWriter, r *http.Request) { - handler := h.svr.GetHandler() - regions, err := handler.GetRegionsByType(statistics.DownPeer) - if err != nil { - h.rd.JSON(w, http.StatusInternalServerError, err.Error()) - return - } - regionsInfo := convertToAPIRegions(regions) - h.rd.JSON(w, http.StatusOK, regionsInfo) + h.getRegionsByType(w, statistics.DownPeer, r) } // @Tags region @@ -472,14 +588,7 @@ func (h *regionsHandler) GetDownPeerRegions(w http.ResponseWriter, r *http.Reque // @Failure 500 {string} string "PD server failed to proceed the request." // @Router /regions/check/learner-peer [get] func (h *regionsHandler) GetLearnerPeerRegions(w http.ResponseWriter, r *http.Request) { - handler := h.svr.GetHandler() - regions, err := handler.GetRegionsByType(statistics.LearnerPeer) - if err != nil { - h.rd.JSON(w, http.StatusInternalServerError, err.Error()) - return - } - regionsInfo := convertToAPIRegions(regions) - h.rd.JSON(w, http.StatusOK, regionsInfo) + h.getRegionsByType(w, statistics.LearnerPeer, r) } // @Tags region @@ -489,14 +598,7 @@ func (h *regionsHandler) GetLearnerPeerRegions(w http.ResponseWriter, r *http.Re // @Failure 500 {string} string "PD server failed to proceed the request." // @Router /regions/check/offline-peer [get] func (h *regionsHandler) GetOfflinePeerRegions(w http.ResponseWriter, r *http.Request) { - handler := h.svr.GetHandler() - regions, err := handler.GetOfflinePeer(statistics.OfflinePeer) - if err != nil { - h.rd.JSON(w, http.StatusInternalServerError, err.Error()) - return - } - regionsInfo := convertToAPIRegions(regions) - h.rd.JSON(w, http.StatusOK, regionsInfo) + h.getRegionsByType(w, statistics.OfflinePeer, r) } // @Tags region @@ -506,14 +608,7 @@ func (h *regionsHandler) GetOfflinePeerRegions(w http.ResponseWriter, r *http.Re // @Failure 500 {string} string "PD server failed to proceed the request." // @Router /regions/check/oversized-region [get] func (h *regionsHandler) GetOverSizedRegions(w http.ResponseWriter, r *http.Request) { - handler := h.svr.GetHandler() - regions, err := handler.GetRegionsByType(statistics.OversizedRegion) - if err != nil { - h.rd.JSON(w, http.StatusInternalServerError, err.Error()) - return - } - regionsInfo := convertToAPIRegions(regions) - h.rd.JSON(w, http.StatusOK, regionsInfo) + h.getRegionsByType(w, statistics.OversizedRegion, r) } // @Tags region @@ -523,14 +618,7 @@ func (h *regionsHandler) GetOverSizedRegions(w http.ResponseWriter, r *http.Requ // @Failure 500 {string} string "PD server failed to proceed the request." // @Router /regions/check/undersized-region [get] func (h *regionsHandler) GetUndersizedRegions(w http.ResponseWriter, r *http.Request) { - handler := h.svr.GetHandler() - regions, err := handler.GetRegionsByType(statistics.UndersizedRegion) - if err != nil { - h.rd.JSON(w, http.StatusInternalServerError, err.Error()) - return - } - regionsInfo := convertToAPIRegions(regions) - h.rd.JSON(w, http.StatusOK, regionsInfo) + h.getRegionsByType(w, statistics.UndersizedRegion, r) } // @Tags region @@ -540,14 +628,7 @@ func (h *regionsHandler) GetUndersizedRegions(w http.ResponseWriter, r *http.Req // @Failure 500 {string} string "PD server failed to proceed the request." // @Router /regions/check/empty-region [get] func (h *regionsHandler) GetEmptyRegions(w http.ResponseWriter, r *http.Request) { - handler := h.svr.GetHandler() - regions, err := handler.GetRegionsByType(statistics.EmptyRegion) - if err != nil { - h.rd.JSON(w, http.StatusInternalServerError, err.Error()) - return - } - regionsInfo := convertToAPIRegions(regions) - h.rd.JSON(w, http.StatusOK, regionsInfo) + h.getRegionsByType(w, statistics.EmptyRegion, r) } type histItem struct { @@ -687,8 +768,12 @@ func (h *regionsHandler) GetRegionSiblings(w http.ResponseWriter, r *http.Reques } left, right := rc.GetAdjacentRegions(region) - regionsInfo := convertToAPIRegions([]*core.RegionInfo{left, right}) - h.rd.JSON(w, http.StatusOK, regionsInfo) + b, err := marshalRegionsInfoJSON(r.Context(), []*core.RegionInfo{left, right}) + if err != nil { + h.rd.JSON(w, http.StatusInternalServerError, err.Error()) + return + } + h.rd.Data(w, http.StatusOK, b) } const ( @@ -836,6 +921,61 @@ func (h *regionsHandler) AccelerateRegionsScheduleInRange(w http.ResponseWriter, h.rd.Text(w, http.StatusOK, fmt.Sprintf("Accelerate regions scheduling in a given range [%s,%s)", rawStartKey, rawEndKey)) } +// @Tags region +// @Summary Accelerate regions scheduling in given ranges, only receive hex format for keys +// @Accept json +// @Param body body object true "json params" +// @Param limit query integer false "Limit count" default(256) +// @Produce json +// @Success 200 {string} string "Accelerate regions scheduling in given ranges [startKey1, endKey1), [startKey2, endKey2), ..." +// @Failure 400 {string} string "The input is invalid." +// @Router /regions/accelerate-schedule/batch [post] +func (h *regionsHandler) AccelerateRegionsScheduleInRanges(w http.ResponseWriter, r *http.Request) { + rc := getCluster(r) + var input []map[string]interface{} + if err := apiutil.ReadJSONRespondError(h.rd, w, r.Body, &input); err != nil { + return + } + limit := 256 + if limitStr := r.URL.Query().Get("limit"); limitStr != "" { + var err error + limit, err = strconv.Atoi(limitStr) + if err != nil { + h.rd.JSON(w, http.StatusBadRequest, err.Error()) + return + } + } + if limit > maxRegionLimit { + limit = maxRegionLimit + } + var msgBuilder strings.Builder + msgBuilder.Grow(128) + msgBuilder.WriteString("Accelerate regions scheduling in given ranges: ") + var regions []*core.RegionInfo + for _, rg := range input { + startKey, rawStartKey, err := apiutil.ParseKey("start_key", rg) + if err != nil { + h.rd.JSON(w, http.StatusBadRequest, err.Error()) + return + } + endKey, rawEndKey, err := apiutil.ParseKey("end_key", rg) + if err != nil { + h.rd.JSON(w, http.StatusBadRequest, err.Error()) + return + } + regions = append(regions, rc.ScanRegions(startKey, endKey, limit)...) + msgBuilder.WriteString(fmt.Sprintf("[%s,%s), ", rawStartKey, rawEndKey)) + } + if len(regions) > 0 { + regionsIDList := make([]uint64, 0, len(regions)) + for _, region := range regions { + regionsIDList = append(regionsIDList, region.GetID()) + } + rc.AddSuspectRegions(regionsIDList...) + } + h.rd.Text(w, http.StatusOK, msgBuilder.String()) +} + func (h *regionsHandler) GetTopNRegions(w http.ResponseWriter, r *http.Request, less func(a, b *core.RegionInfo) bool) { rc := getCluster(r) limit := defaultRegionLimit @@ -851,8 +991,12 @@ func (h *regionsHandler) GetTopNRegions(w http.ResponseWriter, r *http.Request, limit = maxRegionLimit } regions := TopNRegions(rc.GetRegions(), less, limit) - regionsInfo := convertToAPIRegions(regions) - h.rd.JSON(w, http.StatusOK, regionsInfo) + b, err := marshalRegionsInfoJSON(r.Context(), regions) + if err != nil { + h.rd.JSON(w, http.StatusInternalServerError, err.Error()) + return + } + h.rd.Data(w, http.StatusOK, b) } // @Tags region @@ -904,7 +1048,7 @@ func (h *regionsHandler) ScatterRegions(w http.ResponseWriter, r *http.Request) h.rd.JSON(w, http.StatusBadRequest, "regions_id is invalid") return } - opsCount, failures, err = rc.GetRegionScatter().ScatterRegionsByID(ids, group, retryLimit) + opsCount, failures, err = rc.GetRegionScatter().ScatterRegionsByID(ids, group, retryLimit, false) if err != nil { h.rd.JSON(w, http.StatusInternalServerError, err.Error()) return diff --git a/server/api/region_easyjson.go b/server/api/region_easyjson.go new file mode 100644 index 00000000000..4bd9fe69e42 --- /dev/null +++ b/server/api/region_easyjson.go @@ -0,0 +1,567 @@ +// Code generated by easyjson for marshaling/unmarshaling. DO NOT EDIT. + +package api + +import ( + json "encoding/json" + easyjson "github.com/mailru/easyjson" + jlexer "github.com/mailru/easyjson/jlexer" + jwriter "github.com/mailru/easyjson/jwriter" + metapb "github.com/pingcap/kvproto/pkg/metapb" + pdpb "github.com/pingcap/kvproto/pkg/pdpb" +) + +// suppress unused package warning +var ( + _ *json.RawMessage + _ *jlexer.Lexer + _ *jwriter.Writer + _ easyjson.Marshaler +) + +func easyjson75d7afa0DecodeGithubComTikvPdServerApi(in *jlexer.Lexer, out *RegionInfo) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "id": + out.ID = uint64(in.Uint64()) + case "start_key": + out.StartKey = string(in.String()) + case "end_key": + out.EndKey = string(in.String()) + case "epoch": + if in.IsNull() { + in.Skip() + out.RegionEpoch = nil + } else { + if out.RegionEpoch == nil { + out.RegionEpoch = new(metapb.RegionEpoch) + } + easyjson75d7afa0DecodeGithubComPingcapKvprotoPkgMetapb(in, out.RegionEpoch) + } + case "peers": + if in.IsNull() { + in.Skip() + out.Peers = nil + } else { + in.Delim('[') + if out.Peers == nil { + if !in.IsDelim(']') { + out.Peers = make([]MetaPeer, 0, 2) + } else { + out.Peers = []MetaPeer{} + } + } else { + out.Peers = (out.Peers)[:0] + } + for !in.IsDelim(']') { + var v1 MetaPeer + easyjson75d7afa0DecodeGithubComTikvPdServerApi1(in, &v1) + out.Peers = append(out.Peers, v1) + in.WantComma() + } + in.Delim(']') + } + case "leader": + easyjson75d7afa0DecodeGithubComTikvPdServerApi1(in, &out.Leader) + case "down_peers": + if in.IsNull() { + in.Skip() + out.DownPeers = nil + } else { + in.Delim('[') + if out.DownPeers == nil { + if !in.IsDelim(']') { + out.DownPeers = make([]PDPeerStats, 0, 1) + } else { + out.DownPeers = []PDPeerStats{} + } + } else { + out.DownPeers = (out.DownPeers)[:0] + } + for !in.IsDelim(']') { + var v2 PDPeerStats + easyjson75d7afa0DecodeGithubComTikvPdServerApi2(in, &v2) + out.DownPeers = append(out.DownPeers, v2) + in.WantComma() + } + in.Delim(']') + } + case "pending_peers": + if in.IsNull() { + in.Skip() + out.PendingPeers = nil + } else { + in.Delim('[') + if out.PendingPeers == nil { + if !in.IsDelim(']') { + out.PendingPeers = make([]MetaPeer, 0, 2) + } else { + out.PendingPeers = []MetaPeer{} + } + } else { + out.PendingPeers = (out.PendingPeers)[:0] + } + for !in.IsDelim(']') { + var v3 MetaPeer + easyjson75d7afa0DecodeGithubComTikvPdServerApi1(in, &v3) + out.PendingPeers = append(out.PendingPeers, v3) + in.WantComma() + } + in.Delim(']') + } + case "cpu_usage": + out.CPUUsage = uint64(in.Uint64()) + case "written_bytes": + out.WrittenBytes = uint64(in.Uint64()) + case "read_bytes": + out.ReadBytes = uint64(in.Uint64()) + case "written_keys": + out.WrittenKeys = uint64(in.Uint64()) + case "read_keys": + out.ReadKeys = uint64(in.Uint64()) + case "approximate_size": + out.ApproximateSize = int64(in.Int64()) + case "approximate_keys": + out.ApproximateKeys = int64(in.Int64()) + case "buckets": + if in.IsNull() { + in.Skip() + out.Buckets = nil + } else { + in.Delim('[') + if out.Buckets == nil { + if !in.IsDelim(']') { + out.Buckets = make([]string, 0, 4) + } else { + out.Buckets = []string{} + } + } else { + out.Buckets = (out.Buckets)[:0] + } + for !in.IsDelim(']') { + var v4 string + v4 = string(in.String()) + out.Buckets = append(out.Buckets, v4) + in.WantComma() + } + in.Delim(']') + } + case "replication_status": + if in.IsNull() { + in.Skip() + out.ReplicationStatus = nil + } else { + if out.ReplicationStatus == nil { + out.ReplicationStatus = new(ReplicationStatus) + } + easyjson75d7afa0DecodeGithubComTikvPdServerApi3(in, out.ReplicationStatus) + } + default: + in.SkipRecursive() + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} +func easyjson75d7afa0EncodeGithubComTikvPdServerApi(out *jwriter.Writer, in RegionInfo) { + out.RawByte('{') + first := true + _ = first + { + const prefix string = ",\"id\":" + out.RawString(prefix[1:]) + out.Uint64(uint64(in.ID)) + } + { + const prefix string = ",\"start_key\":" + out.RawString(prefix) + out.String(string(in.StartKey)) + } + { + const prefix string = ",\"end_key\":" + out.RawString(prefix) + out.String(string(in.EndKey)) + } + if in.RegionEpoch != nil { + const prefix string = ",\"epoch\":" + out.RawString(prefix) + easyjson75d7afa0EncodeGithubComPingcapKvprotoPkgMetapb(out, *in.RegionEpoch) + } + if len(in.Peers) != 0 { + const prefix string = ",\"peers\":" + out.RawString(prefix) + { + out.RawByte('[') + for v5, v6 := range in.Peers { + if v5 > 0 { + out.RawByte(',') + } + easyjson75d7afa0EncodeGithubComTikvPdServerApi1(out, v6) + } + out.RawByte(']') + } + } + if true { + const prefix string = ",\"leader\":" + out.RawString(prefix) + easyjson75d7afa0EncodeGithubComTikvPdServerApi1(out, in.Leader) + } + if len(in.DownPeers) != 0 { + const prefix string = ",\"down_peers\":" + out.RawString(prefix) + { + out.RawByte('[') + for v7, v8 := range in.DownPeers { + if v7 > 0 { + out.RawByte(',') + } + easyjson75d7afa0EncodeGithubComTikvPdServerApi2(out, v8) + } + out.RawByte(']') + } + } + if len(in.PendingPeers) != 0 { + const prefix string = ",\"pending_peers\":" + out.RawString(prefix) + { + out.RawByte('[') + for v9, v10 := range in.PendingPeers { + if v9 > 0 { + out.RawByte(',') + } + easyjson75d7afa0EncodeGithubComTikvPdServerApi1(out, v10) + } + out.RawByte(']') + } + } + { + const prefix string = ",\"cpu_usage\":" + out.RawString(prefix) + out.Uint64(uint64(in.CPUUsage)) + } + { + const prefix string = ",\"written_bytes\":" + out.RawString(prefix) + out.Uint64(uint64(in.WrittenBytes)) + } + { + const prefix string = ",\"read_bytes\":" + out.RawString(prefix) + out.Uint64(uint64(in.ReadBytes)) + } + { + const prefix string = ",\"written_keys\":" + out.RawString(prefix) + out.Uint64(uint64(in.WrittenKeys)) + } + { + const prefix string = ",\"read_keys\":" + out.RawString(prefix) + out.Uint64(uint64(in.ReadKeys)) + } + { + const prefix string = ",\"approximate_size\":" + out.RawString(prefix) + out.Int64(int64(in.ApproximateSize)) + } + { + const prefix string = ",\"approximate_keys\":" + out.RawString(prefix) + out.Int64(int64(in.ApproximateKeys)) + } + if len(in.Buckets) != 0 { + const prefix string = ",\"buckets\":" + out.RawString(prefix) + { + out.RawByte('[') + for v11, v12 := range in.Buckets { + if v11 > 0 { + out.RawByte(',') + } + out.String(string(v12)) + } + out.RawByte(']') + } + } + if in.ReplicationStatus != nil { + const prefix string = ",\"replication_status\":" + out.RawString(prefix) + easyjson75d7afa0EncodeGithubComTikvPdServerApi3(out, *in.ReplicationStatus) + } + out.RawByte('}') +} + +// MarshalJSON supports json.Marshaler interface +func (v RegionInfo) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + easyjson75d7afa0EncodeGithubComTikvPdServerApi(&w, v) + return w.Buffer.BuildBytes(), w.Error +} + +// MarshalEasyJSON supports easyjson.Marshaler interface +func (v RegionInfo) MarshalEasyJSON(w *jwriter.Writer) { + easyjson75d7afa0EncodeGithubComTikvPdServerApi(w, v) +} + +// UnmarshalJSON supports json.Unmarshaler interface +func (v *RegionInfo) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + easyjson75d7afa0DecodeGithubComTikvPdServerApi(&r, v) + return r.Error() +} + +// UnmarshalEasyJSON supports easyjson.Unmarshaler interface +func (v *RegionInfo) UnmarshalEasyJSON(l *jlexer.Lexer) { + easyjson75d7afa0DecodeGithubComTikvPdServerApi(l, v) +} +func easyjson75d7afa0DecodeGithubComTikvPdServerApi3(in *jlexer.Lexer, out *ReplicationStatus) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "state": + out.State = string(in.String()) + case "state_id": + out.StateID = uint64(in.Uint64()) + default: + in.SkipRecursive() + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} +func easyjson75d7afa0EncodeGithubComTikvPdServerApi3(out *jwriter.Writer, in ReplicationStatus) { + out.RawByte('{') + first := true + _ = first + { + const prefix string = ",\"state\":" + out.RawString(prefix[1:]) + out.String(string(in.State)) + } + { + const prefix string = ",\"state_id\":" + out.RawString(prefix) + out.Uint64(uint64(in.StateID)) + } + out.RawByte('}') +} +func easyjson75d7afa0DecodeGithubComTikvPdServerApi2(in *jlexer.Lexer, out *PDPeerStats) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + out.PeerStats = new(pdpb.PeerStats) + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "peer": + easyjson75d7afa0DecodeGithubComTikvPdServerApi1(in, &out.Peer) + case "down_seconds": + out.DownSeconds = uint64(in.Uint64()) + default: + in.SkipRecursive() + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} +func easyjson75d7afa0EncodeGithubComTikvPdServerApi2(out *jwriter.Writer, in PDPeerStats) { + out.RawByte('{') + first := true + _ = first + { + const prefix string = ",\"peer\":" + out.RawString(prefix[1:]) + easyjson75d7afa0EncodeGithubComTikvPdServerApi1(out, in.Peer) + } + if in.DownSeconds != 0 { + const prefix string = ",\"down_seconds\":" + out.RawString(prefix) + out.Uint64(uint64(in.DownSeconds)) + } + out.RawByte('}') +} +func easyjson75d7afa0DecodeGithubComTikvPdServerApi1(in *jlexer.Lexer, out *MetaPeer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + out.Peer = new(metapb.Peer) + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "role_name": + out.RoleName = string(in.String()) + case "is_learner": + out.IsLearner = bool(in.Bool()) + case "id": + out.Id = uint64(in.Uint64()) + case "store_id": + out.StoreId = uint64(in.Uint64()) + case "role": + out.Role = metapb.PeerRole(in.Int32()) + case "is_witness": + out.IsWitness = bool(in.Bool()) + default: + in.SkipRecursive() + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} +func easyjson75d7afa0EncodeGithubComTikvPdServerApi1(out *jwriter.Writer, in MetaPeer) { + out.RawByte('{') + first := true + _ = first + { + const prefix string = ",\"role_name\":" + out.RawString(prefix[1:]) + out.String(string(in.RoleName)) + } + if in.IsLearner { + const prefix string = ",\"is_learner\":" + out.RawString(prefix) + out.Bool(bool(in.IsLearner)) + } + if in.Id != 0 { + const prefix string = ",\"id\":" + out.RawString(prefix) + out.Uint64(uint64(in.Id)) + } + if in.StoreId != 0 { + const prefix string = ",\"store_id\":" + out.RawString(prefix) + out.Uint64(uint64(in.StoreId)) + } + if in.Role != 0 { + const prefix string = ",\"role\":" + out.RawString(prefix) + out.Int32(int32(in.Role)) + } + if in.IsWitness { + const prefix string = ",\"is_witness\":" + out.RawString(prefix) + out.Bool(bool(in.IsWitness)) + } + out.RawByte('}') +} +func easyjson75d7afa0DecodeGithubComPingcapKvprotoPkgMetapb(in *jlexer.Lexer, out *metapb.RegionEpoch) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "conf_ver": + out.ConfVer = uint64(in.Uint64()) + case "version": + out.Version = uint64(in.Uint64()) + default: + in.SkipRecursive() + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} +func easyjson75d7afa0EncodeGithubComPingcapKvprotoPkgMetapb(out *jwriter.Writer, in metapb.RegionEpoch) { + out.RawByte('{') + first := true + _ = first + if in.ConfVer != 0 { + const prefix string = ",\"conf_ver\":" + first = false + out.RawString(prefix[1:]) + out.Uint64(uint64(in.ConfVer)) + } + if in.Version != 0 { + const prefix string = ",\"version\":" + if first { + first = false + out.RawString(prefix[1:]) + } else { + out.RawString(prefix) + } + out.Uint64(uint64(in.Version)) + } + out.RawByte('}') +} diff --git a/server/api/region_label_test.go b/server/api/region_label_test.go index a425fad1b28..021ec7f1359 100644 --- a/server/api/region_label_test.go +++ b/server/api/region_label_test.go @@ -32,7 +32,7 @@ import ( type regionLabelTestSuite struct { suite.Suite svr *server.Server - cleanup cleanUpFunc + cleanup tu.CleanupFunc urlPrefix string } @@ -47,13 +47,13 @@ func (suite *regionLabelTestSuite) SetupSuite() { addr := suite.svr.GetAddr() suite.urlPrefix = fmt.Sprintf("%s%s/api/v1/config/region-label/", addr, apiPrefix) - suite.NoError(failpoint.Enable("github.com/tikv/pd/server/keyspace/skipSplitRegion", "return(true)")) + suite.NoError(failpoint.Enable("github.com/tikv/pd/pkg/keyspace/skipSplitRegion", "return(true)")) mustBootstrapCluster(re, suite.svr) } func (suite *regionLabelTestSuite) TearDownSuite() { suite.cleanup() - suite.NoError(failpoint.Disable("github.com/tikv/pd/server/keyspace/skipSplitRegion")) + suite.NoError(failpoint.Disable("github.com/tikv/pd/pkg/keyspace/skipSplitRegion")) } func (suite *regionLabelTestSuite) TestGetSet() { diff --git a/server/api/region_test.go b/server/api/region_test.go index 3478a387dda..63da19ab082 100644 --- a/server/api/region_test.go +++ b/server/api/region_test.go @@ -16,6 +16,7 @@ package api import ( "bytes" + "context" "encoding/hex" "encoding/json" "fmt" @@ -24,6 +25,7 @@ import ( "net/url" "sort" "testing" + "time" "github.com/docker/go-units" "github.com/pingcap/failpoint" @@ -33,6 +35,7 @@ import ( "github.com/stretchr/testify/suite" "github.com/tikv/pd/pkg/core" "github.com/tikv/pd/pkg/schedule/placement" + "github.com/tikv/pd/pkg/utils/apiutil" tu "github.com/tikv/pd/pkg/utils/testutil" "github.com/tikv/pd/server" ) @@ -86,7 +89,7 @@ func TestPeerStats(t *testing.T) { type regionTestSuite struct { suite.Suite svr *server.Server - cleanup cleanUpFunc + cleanup tu.CleanupFunc urlPrefix string } @@ -151,7 +154,10 @@ func (suite *regionTestSuite) TestRegionCheck() { core.SetApproximateKeys(10), core.SetApproximateSize(10)) downPeer := &metapb.Peer{Id: 13, StoreId: 2} - r = r.Clone(core.WithAddPeer(downPeer), core.WithDownPeers([]*pdpb.PeerStats{{Peer: downPeer, DownSeconds: 3600}}), core.WithPendingPeers([]*metapb.Peer{downPeer})) + r = r.Clone( + core.WithAddPeer(downPeer), + core.WithDownPeers([]*pdpb.PeerStats{{Peer: downPeer, DownSeconds: 3600}}), + core.WithPendingPeers([]*metapb.Peer{downPeer})) re := suite.Require() mustRegionHeartbeat(re, suite.svr, r) url := fmt.Sprintf("%s/region/id/%d", suite.urlPrefix, r.GetID()) @@ -201,6 +207,15 @@ func (suite *regionTestSuite) TestRegionCheck() { suite.NoError(tu.ReadGetJSON(re, testDialClient, url, &r7)) histKeys := []*histItem{{Start: 1000, End: 1999, Count: 1}} suite.Equal(histKeys, r7) + + mustPutStore(re, suite.svr, 2, metapb.StoreState_Offline, metapb.NodeState_Removing, []*metapb.StoreLabel{}) + mustRegionHeartbeat(re, suite.svr, r) + url = fmt.Sprintf("%s/regions/check/%s", suite.urlPrefix, "offline-peer") + r8 := &RegionsInfo{} + suite.NoError(tu.ReadGetJSON(re, testDialClient, url, r8)) + r4.Adjust() + suite.Equal(1, r8.Count) + suite.Equal(r.GetID(), r8.Regions[0].ID) } func (suite *regionTestSuite) TestRegions() { @@ -332,6 +347,26 @@ func (suite *regionTestSuite) TestAccelerateRegionsScheduleInRange() { suite.Len(idList, 2) } +func (suite *regionTestSuite) TestAccelerateRegionsScheduleInRanges() { + re := suite.Require() + r1 := core.NewTestRegionInfo(557, 13, []byte("a1"), []byte("a2")) + r2 := core.NewTestRegionInfo(558, 14, []byte("a2"), []byte("a3")) + r3 := core.NewTestRegionInfo(559, 15, []byte("a3"), []byte("a4")) + r4 := core.NewTestRegionInfo(560, 16, []byte("a4"), []byte("a5")) + r5 := core.NewTestRegionInfo(561, 17, []byte("a5"), []byte("a6")) + mustRegionHeartbeat(re, suite.svr, r1) + mustRegionHeartbeat(re, suite.svr, r2) + mustRegionHeartbeat(re, suite.svr, r3) + mustRegionHeartbeat(re, suite.svr, r4) + mustRegionHeartbeat(re, suite.svr, r5) + body := fmt.Sprintf(`[{"start_key":"%s", "end_key": "%s"}, {"start_key":"%s", "end_key": "%s"}]`, hex.EncodeToString([]byte("a1")), hex.EncodeToString([]byte("a3")), hex.EncodeToString([]byte("a4")), hex.EncodeToString([]byte("a6"))) + + err := tu.CheckPostJSON(testDialClient, fmt.Sprintf("%s/regions/accelerate-schedule/batch", suite.urlPrefix), []byte(body), tu.StatusOK(re)) + suite.NoError(err) + idList := suite.svr.GetRaftCluster().GetSuspectRegions() + suite.Len(idList, 4) +} + func (suite *regionTestSuite) TestScatterRegions() { re := suite.Require() r1 := core.NewTestRegionInfo(601, 13, []byte("b1"), []byte("b2")) @@ -420,10 +455,44 @@ func (suite *regionTestSuite) TestTopN() { } } +func TestRegionsWithKillRequest(t *testing.T) { + re := require.New(t) + svr, cleanup := mustNewServer(re) + defer cleanup() + server.MustWaitLeader(re, []*server.Server{svr}) + + addr := svr.GetAddr() + url := fmt.Sprintf("%s%s/api/v1/regions", addr, apiPrefix) + mustBootstrapCluster(re, svr) + regionCount := 100000 + for i := 0; i < regionCount; i++ { + r := core.NewTestRegionInfo(uint64(i+2), 1, + []byte(fmt.Sprintf("%09d", i)), + []byte(fmt.Sprintf("%09d", i+1)), + core.SetApproximateKeys(10), core.SetApproximateSize(10)) + mustRegionHeartbeat(re, svr, r) + } + + ctx, cancel := context.WithCancel(context.Background()) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, bytes.NewBuffer(nil)) + re.NoError(err) + respCh := make(chan *http.Response) + go func() { + resp, err := testDialClient.Do(req) // nolint:bodyclose + re.Error(err) + re.Contains(err.Error(), "context canceled") + respCh <- resp + }() + time.Sleep(100 * time.Millisecond) // wait for the request to be sent + cancel() // close the request + resp := <-respCh + re.Nil(resp) +} + type getRegionTestSuite struct { suite.Suite svr *server.Server - cleanup cleanUpFunc + cleanup tu.CleanupFunc urlPrefix string } @@ -531,7 +600,7 @@ func (suite *getRegionTestSuite) TestScanRegionByKeys() { type getRegionRangeHolesTestSuite struct { suite.Suite svr *server.Server - cleanup cleanUpFunc + cleanup tu.CleanupFunc urlPrefix string } @@ -580,7 +649,7 @@ func (suite *getRegionRangeHolesTestSuite) TestRegionRangeHoles() { type regionsReplicatedTestSuite struct { suite.Suite svr *server.Server - cleanup cleanUpFunc + cleanup tu.CleanupFunc urlPrefix string } @@ -702,54 +771,60 @@ func (suite *regionsReplicatedTestSuite) TestCheckRegionsReplicated() { suite.Equal("REPLICATED", status) } -// Create n regions (0..n) of n stores (0..n). -// Each region contains np peers, the first peer is the leader. -// (copied from server/cluster_test.go) -func newTestRegions() []*core.RegionInfo { - n := uint64(10000) - np := uint64(3) - - regions := make([]*core.RegionInfo, 0, n) - for i := uint64(0); i < n; i++ { - peers := make([]*metapb.Peer, 0, np) - for j := uint64(0); j < np; j++ { - peer := &metapb.Peer{ - Id: i*np + j, - } - peer.StoreId = (i + j) % n - peers = append(peers, peer) - } - region := &metapb.Region{ - Id: i, - Peers: peers, - StartKey: []byte(fmt.Sprintf("%d", i)), - EndKey: []byte(fmt.Sprintf("%d", i+1)), - RegionEpoch: &metapb.RegionEpoch{ConfVer: 2, Version: 2}, - } - regions = append(regions, core.NewRegionInfo(region, peers[0])) - } - return regions -} - -func BenchmarkRenderJSON(b *testing.B) { - regionInfos := newTestRegions() - rd := createStreamingRender() - regions := convertToAPIRegions(regionInfos) - - b.ResetTimer() - for i := 0; i < b.N; i++ { - var buffer bytes.Buffer - rd.JSON(&buffer, 200, regions) +func TestRegionsInfoMarshal(t *testing.T) { + re := require.New(t) + regionWithNilPeer := core.NewRegionInfo(&metapb.Region{Id: 1}, &metapb.Peer{Id: 1}) + core.SetPeers([]*metapb.Peer{{Id: 2}, nil})(regionWithNilPeer) + cases := [][]*core.RegionInfo{ + {}, + { + // leader is nil + core.NewRegionInfo(&metapb.Region{Id: 1}, nil), + // Peers is empty + core.NewRegionInfo(&metapb.Region{Id: 1}, &metapb.Peer{Id: 1}, + core.SetPeers([]*metapb.Peer{})), + // There is nil peer in peers. + regionWithNilPeer, + }, + { + // PendingPeers is empty + core.NewRegionInfo(&metapb.Region{Id: 1}, &metapb.Peer{Id: 1}, + core.WithPendingPeers([]*metapb.Peer{})), + // There is nil peer in peers. + core.NewRegionInfo(&metapb.Region{Id: 1}, &metapb.Peer{Id: 1}, + core.WithPendingPeers([]*metapb.Peer{nil})), + }, + { + // DownPeers is empty + core.NewRegionInfo(&metapb.Region{Id: 1}, &metapb.Peer{Id: 1}, + core.WithDownPeers([]*pdpb.PeerStats{})), + // There is nil peer in peers. + core.NewRegionInfo(&metapb.Region{Id: 1}, &metapb.Peer{Id: 1}, + core.WithDownPeers([]*pdpb.PeerStats{{Peer: nil}})), + }, + { + // Buckets is nil + core.NewRegionInfo(&metapb.Region{Id: 1}, &metapb.Peer{Id: 1}, + core.SetBuckets(nil)), + // Buckets is empty + core.NewRegionInfo(&metapb.Region{Id: 1}, &metapb.Peer{Id: 1}, + core.SetBuckets(&metapb.Buckets{})), + }, + { + core.NewRegionInfo(&metapb.Region{Id: 1, StartKey: []byte{}, EndKey: []byte{}, + RegionEpoch: &metapb.RegionEpoch{Version: 1, ConfVer: 1}}, + &metapb.Peer{Id: 1}, core.SetCPUUsage(10), + core.SetApproximateKeys(10), core.SetApproximateSize(10), + core.SetWrittenBytes(10), core.SetReadBytes(10), + core.SetReadKeys(10), core.SetWrittenKeys(10)), + }, } -} - -func BenchmarkConvertToAPIRegions(b *testing.B) { - regionInfos := newTestRegions() - - b.ResetTimer() - for i := 0; i < b.N; i++ { - regions := convertToAPIRegions(regionInfos) - _ = regions.Count + regionsInfo := &RegionsInfo{} + for _, regions := range cases { + b, err := marshalRegionsInfoJSON(context.Background(), regions) + re.NoError(err) + err = json.Unmarshal(b, regionsInfo) + re.NoError(err) } } @@ -768,3 +843,34 @@ func BenchmarkHexRegionKeyStr(b *testing.B) { _ = core.HexRegionKeyStr(key) } } + +func BenchmarkGetRegions(b *testing.B) { + re := require.New(b) + svr, cleanup := mustNewServer(re) + defer cleanup() + server.MustWaitLeader(re, []*server.Server{svr}) + + addr := svr.GetAddr() + url := fmt.Sprintf("%s%s/api/v1/regions", addr, apiPrefix) + mustBootstrapCluster(re, svr) + regionCount := 1000000 + for i := 0; i < regionCount; i++ { + r := core.NewTestRegionInfo(uint64(i+2), 1, + []byte(fmt.Sprintf("%09d", i)), + []byte(fmt.Sprintf("%09d", i+1)), + core.SetApproximateKeys(10), core.SetApproximateSize(10)) + mustRegionHeartbeat(re, svr, r) + } + resp, _ := apiutil.GetJSON(testDialClient, url, nil) + regions := &RegionsInfo{} + err := json.NewDecoder(resp.Body).Decode(regions) + re.NoError(err) + re.Equal(regionCount, regions.Count) + resp.Body.Close() + + b.ResetTimer() + for i := 0; i < b.N; i++ { + resp, _ := apiutil.GetJSON(testDialClient, url, nil) + resp.Body.Close() + } +} diff --git a/server/api/router.go b/server/api/router.go index a35d4b391f4..1e0d12d53b6 100644 --- a/server/api/router.go +++ b/server/api/router.go @@ -233,6 +233,7 @@ func createRouter(prefix string, svr *server.Server) *mux.Router { registerFunc(apiRouter, "/hotspot/regions/read", hotStatusHandler.GetHotReadRegions, setMethods(http.MethodGet), setAuditBackend(prometheus)) registerFunc(apiRouter, "/hotspot/regions/history", hotStatusHandler.GetHistoryHotRegions, setMethods(http.MethodGet), setAuditBackend(prometheus)) registerFunc(apiRouter, "/hotspot/stores", hotStatusHandler.GetHotStores, setMethods(http.MethodGet), setAuditBackend(prometheus)) + registerFunc(apiRouter, "/hotspot/buckets", hotStatusHandler.GetHotBuckets, setMethods(http.MethodGet), setAuditBackend(prometheus)) regionHandler := newRegionHandler(svr, rd) registerFunc(clusterRouter, "/region/id/{id}", regionHandler.GetRegionByID, setMethods(http.MethodGet), setAuditBackend(prometheus)) @@ -240,12 +241,13 @@ func createRouter(prefix string, svr *server.Server) *mux.Router { srd := createStreamingRender() regionsAllHandler := newRegionsHandler(svr, srd) - registerFunc(clusterRouter, "/regions", regionsAllHandler.GetRegions, setMethods(http.MethodGet), setAuditBackend(prometheus)) + registerFunc(clusterRouter, "/regions", regionsAllHandler.GetRegions, setMethods(http.MethodGet), setAuditBackend(localLog, prometheus)) regionsHandler := newRegionsHandler(svr, rd) registerFunc(clusterRouter, "/regions/key", regionsHandler.ScanRegions, setMethods(http.MethodGet), setAuditBackend(prometheus)) registerFunc(clusterRouter, "/regions/count", regionsHandler.GetRegionCount, setMethods(http.MethodGet), setAuditBackend(prometheus)) registerFunc(clusterRouter, "/regions/store/{id}", regionsHandler.GetStoreRegions, setMethods(http.MethodGet), setAuditBackend(prometheus)) + registerFunc(clusterRouter, "/regions/keyspace/id/{id}", regionsHandler.GetKeyspaceRegions, setMethods(http.MethodGet), setAuditBackend(prometheus)) registerFunc(clusterRouter, "/regions/writeflow", regionsHandler.GetTopWriteFlowRegions, setMethods(http.MethodGet), setAuditBackend(prometheus)) registerFunc(clusterRouter, "/regions/readflow", regionsHandler.GetTopReadFlowRegions, setMethods(http.MethodGet), setAuditBackend(prometheus)) registerFunc(clusterRouter, "/regions/confver", regionsHandler.GetTopConfVerRegions, setMethods(http.MethodGet), setAuditBackend(prometheus)) @@ -267,6 +269,7 @@ func createRouter(prefix string, svr *server.Server) *mux.Router { registerFunc(clusterRouter, "/regions/check/hist-keys", regionsHandler.GetKeysHistogram, setMethods(http.MethodGet), setAuditBackend(prometheus)) registerFunc(clusterRouter, "/regions/sibling/{id}", regionsHandler.GetRegionSiblings, setMethods(http.MethodGet), setAuditBackend(prometheus)) registerFunc(clusterRouter, "/regions/accelerate-schedule", regionsHandler.AccelerateRegionsScheduleInRange, setMethods(http.MethodPost), setAuditBackend(localLog, prometheus)) + registerFunc(clusterRouter, "/regions/accelerate-schedule/batch", regionsHandler.AccelerateRegionsScheduleInRanges, setMethods(http.MethodPost), setAuditBackend(localLog, prometheus)) registerFunc(clusterRouter, "/regions/scatter", regionsHandler.ScatterRegions, setMethods(http.MethodPost), setAuditBackend(localLog, prometheus)) registerFunc(clusterRouter, "/regions/split", regionsHandler.SplitRegions, setMethods(http.MethodPost), setAuditBackend(localLog, prometheus)) registerFunc(clusterRouter, "/regions/range-holes", regionsHandler.GetRangeHoles, setMethods(http.MethodGet), setAuditBackend(prometheus)) @@ -287,16 +290,16 @@ func createRouter(prefix string, svr *server.Server) *mux.Router { registerFunc(apiRouter, "/leader/transfer/{next_leader}", leaderHandler.TransferLeader, setMethods(http.MethodPost), setAuditBackend(localLog, prometheus)) statsHandler := newStatsHandler(svr, rd) - registerFunc(clusterRouter, "/stats/region", statsHandler.GetRegionStatus, setMethods(http.MethodGet), setAuditBackend(prometheus)) + registerFunc(clusterRouter, "/stats/region", statsHandler.GetRegionStatus, setMethods(http.MethodGet), setAuditBackend(localLog, prometheus)) trendHandler := newTrendHandler(svr, rd) registerFunc(apiRouter, "/trend", trendHandler.GetTrend, setMethods(http.MethodGet), setAuditBackend(prometheus)) adminHandler := newAdminHandler(svr, rd) registerFunc(clusterRouter, "/admin/cache/region/{id}", adminHandler.DeleteRegionCache, setMethods(http.MethodDelete), setAuditBackend(localLog, prometheus)) + registerFunc(clusterRouter, "/admin/storage/region/{id}", adminHandler.DeleteRegionStorage, setMethods(http.MethodDelete), setAuditBackend(localLog, prometheus)) registerFunc(clusterRouter, "/admin/cache/regions", adminHandler.DeleteAllRegionCache, setMethods(http.MethodDelete), setAuditBackend(localLog, prometheus)) registerFunc(apiRouter, "/admin/persist-file/{file_name}", adminHandler.SavePersistFile, setMethods(http.MethodPost), setAuditBackend(localLog, prometheus)) - registerFunc(apiRouter, "/admin/persist-file/{file_name}", adminHandler.SavePersistFile, setMethods(http.MethodPost), setAuditBackend(localLog, prometheus)) registerFunc(apiRouter, "/admin/cluster/markers/snapshot-recovering", adminHandler.IsSnapshotRecovering, setMethods(http.MethodGet), setAuditBackend(localLog, prometheus)) registerFunc(apiRouter, "/admin/cluster/markers/snapshot-recovering", adminHandler.MarkSnapshotRecovering, setMethods(http.MethodPost), setAuditBackend(localLog, prometheus)) registerFunc(apiRouter, "/admin/cluster/markers/snapshot-recovering", adminHandler.UnmarkSnapshotRecovering, setMethods(http.MethodDelete), setAuditBackend(localLog, prometheus)) @@ -345,6 +348,7 @@ func createRouter(prefix string, svr *server.Server) *mux.Router { // min resolved ts API minResolvedTSHandler := newMinResolvedTSHandler(svr, rd) registerFunc(clusterRouter, "/min-resolved-ts", minResolvedTSHandler.GetMinResolvedTS, setMethods(http.MethodGet), setAuditBackend(prometheus)) + registerFunc(clusterRouter, "/min-resolved-ts/{store_id}", minResolvedTSHandler.GetStoreMinResolvedTS, setMethods(http.MethodGet), setAuditBackend(prometheus)) // unsafe admin operation API unsafeOperationHandler := newUnsafeOperationHandler(svr, rd) diff --git a/server/api/rule.go b/server/api/rule.go index 015b82dc097..33c63a8faa2 100644 --- a/server/api/rule.go +++ b/server/api/rule.go @@ -70,7 +70,7 @@ func (h *ruleHandler) GetAllRules(w http.ResponseWriter, r *http.Request) { // @Failure 400 {string} string "The input is invalid." // @Failure 412 {string} string "Placement rules feature is disabled." // @Failure 500 {string} string "PD server failed to proceed the request." -// @Router /config/rules [get] +// @Router /config/rules [post] func (h *ruleHandler) SetAllRules(w http.ResponseWriter, r *http.Request) { cluster := getCluster(r) if !cluster.GetOpts().IsPlacementRulesEnabled() { diff --git a/server/api/rule_test.go b/server/api/rule_test.go index a4f6450e80d..d2000eb9562 100644 --- a/server/api/rule_test.go +++ b/server/api/rule_test.go @@ -35,7 +35,7 @@ import ( type ruleTestSuite struct { suite.Suite svr *server.Server - cleanup cleanUpFunc + cleanup tu.CleanupFunc urlPrefix string } @@ -850,7 +850,7 @@ type regionRuleTestSuite struct { suite.Suite svr *server.Server grpcSvr *server.GrpcServer - cleanup cleanUpFunc + cleanup tu.CleanupFunc urlPrefix string stores []*metapb.Store regions []*core.RegionInfo diff --git a/server/api/scheduler_test.go b/server/api/scheduler_test.go index 802cd8b485d..026d7a3cd2f 100644 --- a/server/api/scheduler_test.go +++ b/server/api/scheduler_test.go @@ -24,16 +24,16 @@ import ( "github.com/pingcap/failpoint" "github.com/pingcap/kvproto/pkg/metapb" "github.com/stretchr/testify/suite" + sc "github.com/tikv/pd/pkg/schedule/config" "github.com/tikv/pd/pkg/utils/apiutil" tu "github.com/tikv/pd/pkg/utils/testutil" "github.com/tikv/pd/server" - "github.com/tikv/pd/server/config" ) type scheduleTestSuite struct { suite.Suite svr *server.Server - cleanup cleanUpFunc + cleanup tu.CleanupFunc urlPrefix string } @@ -470,12 +470,12 @@ func (suite *scheduleTestSuite) TestDisable() { re := suite.Require() u := fmt.Sprintf("%s%s/api/v1/config/schedule", suite.svr.GetAddr(), apiPrefix) - var scheduleConfig config.ScheduleConfig + var scheduleConfig sc.ScheduleConfig err = tu.ReadGetJSON(re, testDialClient, u, &scheduleConfig) suite.NoError(err) originSchedulers := scheduleConfig.Schedulers - scheduleConfig.Schedulers = config.SchedulerConfigs{config.SchedulerConfig{Type: "shuffle-leader", Disable: true}} + scheduleConfig.Schedulers = sc.SchedulerConfigs{sc.SchedulerConfig{Type: "shuffle-leader", Disable: true}} body, err = json.Marshal(scheduleConfig) suite.NoError(err) err = tu.CheckPostJSON(testDialClient, u, body, tu.StatusOK(re)) diff --git a/server/api/server.go b/server/api/server.go index 6d0f5a5aa13..272e76cc60b 100644 --- a/server/api/server.go +++ b/server/api/server.go @@ -19,6 +19,7 @@ import ( "net/http" "github.com/gorilla/mux" + tsoapi "github.com/tikv/pd/pkg/mcs/tso/server/apis/v1" "github.com/tikv/pd/pkg/utils/apiutil" "github.com/tikv/pd/pkg/utils/apiutil/serverapi" "github.com/tikv/pd/server" @@ -28,7 +29,7 @@ import ( const apiPrefix = "/pd" // NewHandler creates a HTTP handler for API. -func NewHandler(ctx context.Context, svr *server.Server) (http.Handler, apiutil.APIServiceGroup, error) { +func NewHandler(_ context.Context, svr *server.Server) (http.Handler, apiutil.APIServiceGroup, error) { group := apiutil.APIServiceGroup{ Name: "core", IsCore: true, @@ -37,7 +38,8 @@ func NewHandler(ctx context.Context, svr *server.Server) (http.Handler, apiutil. r := createRouter(apiPrefix, svr) router.PathPrefix(apiPrefix).Handler(negroni.New( serverapi.NewRuntimeServiceValidator(svr, group), - serverapi.NewRedirector(svr), + serverapi.NewRedirector(svr, serverapi.MicroserviceRedirectRule( + apiPrefix+"/api/v1"+"/admin/reset-ts", tsoapi.APIPathPrefix+"/admin/reset-ts", "tso")), negroni.Wrap(r)), ) diff --git a/server/api/server_test.go b/server/api/server_test.go index 61dc74a0724..88253b3a624 100644 --- a/server/api/server_test.go +++ b/server/api/server_test.go @@ -68,16 +68,14 @@ func TestMain(m *testing.M) { goleak.VerifyTestMain(m, testutil.LeakOptions...) } -type cleanUpFunc func() - -func mustNewServer(re *require.Assertions, opts ...func(cfg *config.Config)) (*server.Server, cleanUpFunc) { +func mustNewServer(re *require.Assertions, opts ...func(cfg *config.Config)) (*server.Server, testutil.CleanupFunc) { _, svrs, cleanup := mustNewCluster(re, 1, opts...) return svrs[0], cleanup } var zapLogOnce sync.Once -func mustNewCluster(re *require.Assertions, num int, opts ...func(cfg *config.Config)) ([]*config.Config, []*server.Server, cleanUpFunc) { +func mustNewCluster(re *require.Assertions, num int, opts ...func(cfg *config.Config)) ([]*config.Config, []*server.Server, testutil.CleanupFunc) { ctx, cancel := context.WithCancel(context.Background()) svrs := make([]*server.Server, 0, num) cfgs := server.NewTestMultiConfig(assertutil.CheckerWithNilAssert(re), num) @@ -138,7 +136,7 @@ func mustBootstrapCluster(re *require.Assertions, s *server.Server) { type serviceTestSuite struct { suite.Suite svr *server.Server - cleanup cleanUpFunc + cleanup testutil.CleanupFunc } func TestServiceTestSuite(t *testing.T) { diff --git a/server/api/service_gc_safepoint_test.go b/server/api/service_gc_safepoint_test.go index b6661f6b8fd..fe52204dfb2 100644 --- a/server/api/service_gc_safepoint_test.go +++ b/server/api/service_gc_safepoint_test.go @@ -24,13 +24,14 @@ import ( "github.com/stretchr/testify/suite" "github.com/tikv/pd/pkg/storage/endpoint" "github.com/tikv/pd/pkg/utils/apiutil" + "github.com/tikv/pd/pkg/utils/testutil" "github.com/tikv/pd/server" ) type serviceGCSafepointTestSuite struct { suite.Suite svr *server.Server - cleanup cleanUpFunc + cleanup testutil.CleanupFunc urlPrefix string } diff --git a/server/api/service_middleware_test.go b/server/api/service_middleware_test.go index dde0ba0272a..e1a5853db16 100644 --- a/server/api/service_middleware_test.go +++ b/server/api/service_middleware_test.go @@ -31,7 +31,7 @@ import ( type auditMiddlewareTestSuite struct { suite.Suite svr *server.Server - cleanup cleanUpFunc + cleanup tu.CleanupFunc urlPrefix string } @@ -114,7 +114,7 @@ func (suite *auditMiddlewareTestSuite) TestConfigAuditSwitch() { type rateLimitConfigTestSuite struct { suite.Suite svr *server.Server - cleanup cleanUpFunc + cleanup tu.CleanupFunc urlPrefix string } diff --git a/server/api/stats.go b/server/api/stats.go index 21d9606a56a..e8b04ba588e 100644 --- a/server/api/stats.go +++ b/server/api/stats.go @@ -46,7 +46,7 @@ func (h *statsHandler) GetRegionStatus(w http.ResponseWriter, r *http.Request) { startKey, endKey := r.URL.Query().Get("start_key"), r.URL.Query().Get("end_key") var stats *statistics.RegionStats if r.URL.Query().Has("count") { - stats = rc.GetRangeCount([]byte(startKey), []byte(endKey)) + stats = rc.GetRegionCount([]byte(startKey), []byte(endKey)) } else { stats = rc.GetRegionStats([]byte(startKey), []byte(endKey)) } diff --git a/server/api/stats_test.go b/server/api/stats_test.go index 5a63181cf1e..4003f53ed9e 100644 --- a/server/api/stats_test.go +++ b/server/api/stats_test.go @@ -24,13 +24,14 @@ import ( "github.com/tikv/pd/pkg/core" "github.com/tikv/pd/pkg/statistics" "github.com/tikv/pd/pkg/utils/apiutil" + "github.com/tikv/pd/pkg/utils/testutil" "github.com/tikv/pd/server" ) type statsTestSuite struct { suite.Suite svr *server.Server - cleanup cleanUpFunc + cleanup testutil.CleanupFunc urlPrefix string } @@ -73,6 +74,7 @@ func (suite *statsTestSuite) TestRegionStats() { }, &metapb.Peer{Id: 101, StoreId: 1}, core.SetApproximateSize(100), + core.SetApproximateKvSize(80), core.SetApproximateKeys(50), ), core.NewRegionInfo( @@ -89,6 +91,7 @@ func (suite *statsTestSuite) TestRegionStats() { }, &metapb.Peer{Id: 105, StoreId: 4}, core.SetApproximateSize(200), + core.SetApproximateKvSize(180), core.SetApproximateKeys(150), ), core.NewRegionInfo( @@ -104,6 +107,7 @@ func (suite *statsTestSuite) TestRegionStats() { }, &metapb.Peer{Id: 107, StoreId: 5}, core.SetApproximateSize(1), + core.SetApproximateKvSize(1), core.SetApproximateKeys(1), ), core.NewRegionInfo( @@ -118,6 +122,7 @@ func (suite *statsTestSuite) TestRegionStats() { }, &metapb.Peer{Id: 108, StoreId: 4}, core.SetApproximateSize(50), + core.SetApproximateKvSize(30), core.SetApproximateKeys(20), ), } @@ -138,6 +143,7 @@ func (suite *statsTestSuite) TestRegionStats() { Count: 4, EmptyCount: 1, StorageSize: 351, + UserStorageSize: 291, StorageKeys: 221, StoreLeaderCount: map[uint64]int{1: 1, 4: 2, 5: 1}, StorePeerCount: map[uint64]int{1: 3, 2: 1, 3: 1, 4: 2, 5: 2}, @@ -151,6 +157,7 @@ func (suite *statsTestSuite) TestRegionStats() { Count: 2, EmptyCount: 1, StorageSize: 201, + UserStorageSize: 181, StorageKeys: 151, StoreLeaderCount: map[uint64]int{4: 1, 5: 1}, StorePeerCount: map[uint64]int{1: 2, 4: 1, 5: 2}, diff --git a/server/api/store.go b/server/api/store.go index 088fe1cfa2c..49384439bc4 100644 --- a/server/api/store.go +++ b/server/api/store.go @@ -30,10 +30,10 @@ import ( "github.com/tikv/pd/pkg/core/constant" "github.com/tikv/pd/pkg/core/storelimit" "github.com/tikv/pd/pkg/errs" + sc "github.com/tikv/pd/pkg/schedule/config" "github.com/tikv/pd/pkg/utils/apiutil" "github.com/tikv/pd/pkg/utils/typeutil" "github.com/tikv/pd/server" - "github.com/tikv/pd/server/config" "github.com/unrolled/render" ) @@ -75,9 +75,10 @@ type StoreStatus struct { RegionWeight float64 `json:"region_weight"` RegionScore float64 `json:"region_score"` RegionSize int64 `json:"region_size"` - WitnessCount int `json:"witness_count"` - SlowScore uint64 `json:"slow_score"` - SlowTrend SlowTrend `json:"slow_trend"` + LearnerCount int `json:"learner_count,omitempty"` + WitnessCount int `json:"witness_count,omitempty"` + SlowScore uint64 `json:"slow_score,omitempty"` + SlowTrend *SlowTrend `json:"slow_trend,omitempty"` SendingSnapCount uint32 `json:"sending_snap_count,omitempty"` ReceivingSnapCount uint32 `json:"receiving_snap_count,omitempty"` IsBusy bool `json:"is_busy,omitempty"` @@ -97,11 +98,11 @@ const ( downStateName = "Down" ) -func newStoreInfo(opt *config.ScheduleConfig, store *core.StoreInfo) *StoreInfo { - var slowTrend SlowTrend +func newStoreInfo(opt *sc.ScheduleConfig, store *core.StoreInfo) *StoreInfo { + var slowTrend *SlowTrend coreSlowTrend := store.GetSlowTrend() if coreSlowTrend != nil { - slowTrend = SlowTrend{coreSlowTrend.CauseValue, coreSlowTrend.CauseRate, coreSlowTrend.ResultValue, coreSlowTrend.ResultRate} + slowTrend = &SlowTrend{coreSlowTrend.CauseValue, coreSlowTrend.CauseRate, coreSlowTrend.ResultValue, coreSlowTrend.ResultRate} } s := &StoreInfo{ Store: &MetaStore{ @@ -120,6 +121,7 @@ func newStoreInfo(opt *config.ScheduleConfig, store *core.StoreInfo) *StoreInfo RegionWeight: store.GetRegionWeight(), RegionScore: store.RegionScore(opt.RegionScoreFormulaVersion, opt.HighSpaceRatio, opt.LowSpaceRatio, 0), RegionSize: store.GetRegionSize(), + LearnerCount: store.GetLearnerCount(), WitnessCount: store.GetWitnessCount(), SlowScore: store.GetSlowScore(), SlowTrend: slowTrend, @@ -312,7 +314,7 @@ func (h *storeHandler) SetStoreLabel(w http.ResponseWriter, r *http.Request) { }) } - if err := config.ValidateLabels(labels); err != nil { + if err := sc.ValidateLabels(labels); err != nil { apiutil.ErrorResp(h.rd, w, errcode.NewInvalidInputErr(err)) return } @@ -348,7 +350,7 @@ func (h *storeHandler) DeleteStoreLabel(w http.ResponseWriter, r *http.Request) if err := apiutil.ReadJSONRespondError(h.rd, w, r.Body, &labelKey); err != nil { return } - if err := config.ValidateLabelKey(labelKey); err != nil { + if err := sc.ValidateLabelKey(labelKey); err != nil { apiutil.ErrorResp(h.rd, w, errcode.NewInvalidInputErr(err)) return } @@ -582,7 +584,7 @@ func (h *storesHandler) SetAllStoresLimit(w http.ResponseWriter, r *http.Request }) } - if err := config.ValidateLabels(labels); err != nil { + if err := sc.ValidateLabels(labels); err != nil { apiutil.ErrorResp(h.rd, w, errcode.NewInvalidInputErr(err)) return } @@ -617,7 +619,7 @@ func (h *storesHandler) GetAllStoresLimit(w http.ResponseWriter, r *http.Request } } if !includeTombstone { - returned := make(map[uint64]config.StoreLimitConfig, len(limits)) + returned := make(map[uint64]sc.StoreLimitConfig, len(limits)) rc := getCluster(r) for storeID, v := range limits { store := rc.GetStore(storeID) diff --git a/server/api/store_test.go b/server/api/store_test.go index fb48bac80f1..4bcdf1953a5 100644 --- a/server/api/store_test.go +++ b/server/api/store_test.go @@ -40,7 +40,7 @@ type storeTestSuite struct { suite.Suite svr *server.Server grpcSvr *server.GrpcServer - cleanup cleanUpFunc + cleanup tu.CleanupFunc urlPrefix string stores []*metapb.Store } diff --git a/server/api/tso.go b/server/api/tso.go index cbe30887b73..ddac5b65bc4 100644 --- a/server/api/tso.go +++ b/server/api/tso.go @@ -69,6 +69,7 @@ func (h *tsoHandler) TransferLocalTSOAllocator(w http.ResponseWriter, r *http.Re h.rd.JSON(w, http.StatusNotFound, fmt.Sprintf("not found, pd: %s", name)) return } + // TODO: support local tso forward in api service mode in the future. err := h.svr.GetTSOAllocatorManager().TransferAllocatorForDCLocation(dcLocation, memberID) if err != nil { h.rd.JSON(w, http.StatusInternalServerError, err.Error()) diff --git a/server/api/tso_test.go b/server/api/tso_test.go index 13552b5efd5..f61e460162e 100644 --- a/server/api/tso_test.go +++ b/server/api/tso_test.go @@ -28,7 +28,7 @@ import ( type tsoTestSuite struct { suite.Suite svr *server.Server - cleanup cleanUpFunc + cleanup tu.CleanupFunc urlPrefix string } diff --git a/server/api/unsafe_operation_test.go b/server/api/unsafe_operation_test.go index 6882b2b3166..9a5b3887175 100644 --- a/server/api/unsafe_operation_test.go +++ b/server/api/unsafe_operation_test.go @@ -21,15 +21,15 @@ import ( "github.com/pingcap/kvproto/pkg/metapb" "github.com/stretchr/testify/suite" + "github.com/tikv/pd/pkg/unsaferecovery" tu "github.com/tikv/pd/pkg/utils/testutil" "github.com/tikv/pd/server" - "github.com/tikv/pd/server/cluster" ) type unsafeOperationTestSuite struct { suite.Suite svr *server.Server - cleanup cleanUpFunc + cleanup tu.CleanupFunc urlPrefix string } @@ -80,7 +80,7 @@ func (suite *unsafeOperationTestSuite) TestRemoveFailedStores() { suite.NoError(err) // Test show - var output []cluster.StageOutput + var output []unsaferecovery.StageOutput err = tu.ReadGetJSON(re, testDialClient, suite.urlPrefix+"/remove-failed-stores/show", &output) suite.NoError(err) } diff --git a/server/apiv2/handlers/keyspace.go b/server/apiv2/handlers/keyspace.go index a5f930a5375..b93dc84faf8 100644 --- a/server/apiv2/handlers/keyspace.go +++ b/server/apiv2/handlers/keyspace.go @@ -25,11 +25,13 @@ import ( "github.com/pingcap/errors" "github.com/pingcap/kvproto/pkg/keyspacepb" "github.com/tikv/pd/pkg/errs" + "github.com/tikv/pd/pkg/keyspace" "github.com/tikv/pd/server" "github.com/tikv/pd/server/apiv2/middlewares" - "github.com/tikv/pd/server/keyspace" ) +const managerUninitializedErr = "keyspace manager is not initialized" + // RegisterKeyspace register keyspace related handlers to router paths. func RegisterKeyspace(r *gin.RouterGroup) { router := r.Group("keyspaces") @@ -51,17 +53,21 @@ type CreateKeyspaceParams struct { // CreateKeyspace creates keyspace according to given input. // -// @Tags keyspaces -// @Summary Create new keyspace. -// @Param body body CreateKeyspaceParams true "Create keyspace parameters" -// @Produce json -// @Success 200 {object} KeyspaceMeta -// @Failure 400 {string} string "The input is invalid." -// @Failure 500 {string} string "PD server failed to proceed the request." -// @Router /keyspaces [post] +// @Tags keyspaces +// @Summary Create new keyspace. +// @Param body body CreateKeyspaceParams true "Create keyspace parameters" +// @Produce json +// @Success 200 {object} KeyspaceMeta +// @Failure 400 {string} string "The input is invalid." +// @Failure 500 {string} string "PD server failed to proceed the request." +// @Router /keyspaces [post] func CreateKeyspace(c *gin.Context) { - svr := c.MustGet("server").(*server.Server) + svr := c.MustGet(middlewares.ServerContextKey).(*server.Server) manager := svr.GetKeyspaceManager() + if manager == nil { + c.AbortWithStatusJSON(http.StatusInternalServerError, managerUninitializedErr) + return + } createParams := &CreateKeyspaceParams{} err := c.BindJSON(createParams) if err != nil { @@ -69,9 +75,10 @@ func CreateKeyspace(c *gin.Context) { return } req := &keyspace.CreateKeyspaceRequest{ - Name: createParams.Name, - Config: createParams.Config, - Now: time.Now().Unix(), + Name: createParams.Name, + Config: createParams.Config, + CreateTime: time.Now().Unix(), + IsPreAlloc: false, } meta, err := manager.CreateKeyspace(req) if err != nil { @@ -83,42 +90,64 @@ func CreateKeyspace(c *gin.Context) { // LoadKeyspace returns target keyspace. // -// @Tags keyspaces -// @Summary Get keyspace info. -// @Param name path string true "Keyspace Name" -// @Produce json -// @Success 200 {object} KeyspaceMeta -// @Failure 500 {string} string "PD server failed to proceed the request." -// @Router /keyspaces/{name} [get] +// @Tags keyspaces +// @Summary Get keyspace info. +// @Param name path string true "Keyspace Name" +// @Produce json +// @Success 200 {object} KeyspaceMeta +// @Failure 500 {string} string "PD server failed to proceed the request." +// @Router /keyspaces/{name} [get] func LoadKeyspace(c *gin.Context) { - svr := c.MustGet("server").(*server.Server) + svr := c.MustGet(middlewares.ServerContextKey).(*server.Server) manager := svr.GetKeyspaceManager() + if manager == nil { + c.AbortWithStatusJSON(http.StatusInternalServerError, managerUninitializedErr) + return + } name := c.Param("name") meta, err := manager.LoadKeyspace(name) if err != nil { c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error()) return } + if value, ok := c.GetQuery("force_refresh_group_id"); ok && value == "true" { + groupManager := svr.GetKeyspaceGroupManager() + if groupManager == nil { + c.AbortWithStatusJSON(http.StatusInternalServerError, managerUninitializedErr) + return + } + // keyspace has been checked in LoadKeyspace, so no need to check again. + groupID, err := groupManager.GetGroupByKeyspaceID(meta.GetId()) + if err != nil { + c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error()) + return + } + meta.Config[keyspace.TSOKeyspaceGroupIDKey] = strconv.FormatUint(uint64(groupID), 10) + } c.IndentedJSON(http.StatusOK, &KeyspaceMeta{meta}) } // LoadKeyspaceByID returns target keyspace. // -// @Tags keyspaces -// @Summary Get keyspace info. -// @Param id path string true "Keyspace id" -// @Produce json -// @Success 200 {object} KeyspaceMeta -// @Failure 500 {string} string "PD server failed to proceed the request." -// @Router /keyspaces/id/{id} [get] +// @Tags keyspaces +// @Summary Get keyspace info. +// @Param id path string true "Keyspace id" +// @Produce json +// @Success 200 {object} KeyspaceMeta +// @Failure 500 {string} string "PD server failed to proceed the request." +// @Router /keyspaces/id/{id} [get] func LoadKeyspaceByID(c *gin.Context) { id, err := strconv.ParseUint(c.Param("id"), 10, 64) if err != nil || id == 0 { c.AbortWithStatusJSON(http.StatusInternalServerError, "invalid keyspace id") return } - svr := c.MustGet("server").(*server.Server) + svr := c.MustGet(middlewares.ServerContextKey).(*server.Server) manager := svr.GetKeyspaceManager() + if manager == nil { + c.AbortWithStatusJSON(http.StatusInternalServerError, managerUninitializedErr) + return + } meta, err := manager.LoadKeyspaceByID(uint32(id)) if err != nil { c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error()) @@ -127,18 +156,18 @@ func LoadKeyspaceByID(c *gin.Context) { c.IndentedJSON(http.StatusOK, &KeyspaceMeta{meta}) } -// parseLoadAllQuery parses LoadAllKeyspaces' query parameters. +// parseLoadAllQuery parses LoadAllKeyspaces'/GetKeyspaceGroups' query parameters. // page_token: -// The keyspace id of the scan start. If not set, scan from keyspace with id 1. -// It's string of spaceID of the previous scan result's last element (next_page_token). +// The keyspace/keyspace group id of the scan start. If not set, scan from keyspace/keyspace group with id 1. +// It's string of ID of the previous scan result's last element (next_page_token). // limit: -// The maximum number of keyspace metas to return. If not set, no limit is posed. -// Every scan scans limit + 1 keyspaces (if limit != 0), the extra scanned keyspace +// The maximum number of keyspace metas/keyspace groups to return. If not set, no limit is posed. +// Every scan scans limit + 1 keyspaces/keyspace groups (if limit != 0), the extra scanned keyspace/keyspace group // is to check if there's more, and used to set next_page_token in response. func parseLoadAllQuery(c *gin.Context) (scanStart uint32, scanLimit int, err error) { pageToken, set := c.GetQuery("page_token") if !set || pageToken == "" { - // If pageToken is empty or unset, then scan from spaceID of 1. + // If pageToken is empty or unset, then scan from ID of 1. scanStart = 0 } else { scanStart64, err := strconv.ParseUint(pageToken, 10, 32) @@ -175,18 +204,22 @@ type LoadAllKeyspacesResponse struct { // LoadAllKeyspaces loads range of keyspaces. // -// @Tags keyspaces -// @Summary list keyspaces. -// @Param page_token query string false "page token" -// @Param limit query string false "maximum number of results to return" -// @Produce json -// @Success 200 {object} LoadAllKeyspacesResponse -// @Failure 400 {string} string "The input is invalid." -// @Failure 500 {string} string "PD server failed to proceed the request." -// @Router /keyspaces [get] +// @Tags keyspaces +// @Summary list keyspaces. +// @Param page_token query string false "page token" +// @Param limit query string false "maximum number of results to return" +// @Produce json +// @Success 200 {object} LoadAllKeyspacesResponse +// @Failure 400 {string} string "The input is invalid." +// @Failure 500 {string} string "PD server failed to proceed the request." +// @Router /keyspaces [get] func LoadAllKeyspaces(c *gin.Context) { - svr := c.MustGet("server").(*server.Server) + svr := c.MustGet(middlewares.ServerContextKey).(*server.Server) manager := svr.GetKeyspaceManager() + if manager == nil { + c.AbortWithStatusJSON(http.StatusInternalServerError, managerUninitializedErr) + return + } scanStart, scanLimit, err := parseLoadAllQuery(c) if err != nil { c.AbortWithStatusJSON(http.StatusBadRequest, err.Error()) @@ -235,19 +268,23 @@ type UpdateConfigParams struct { // This api uses PATCH semantic and supports JSON Merge Patch. // format and processing rules. // -// @Tags keyspaces -// @Summary Update keyspace config. -// @Param name path string true "Keyspace Name" -// @Param body body UpdateConfigParams true "Update keyspace parameters" -// @Produce json -// @Success 200 {object} KeyspaceMeta -// @Failure 400 {string} string "The input is invalid." -// @Failure 500 {string} string "PD server failed to proceed the request." +// @Tags keyspaces +// @Summary Update keyspace config. +// @Param name path string true "Keyspace Name" +// @Param body body UpdateConfigParams true "Update keyspace parameters" +// @Produce json +// @Success 200 {object} KeyspaceMeta +// @Failure 400 {string} string "The input is invalid." +// @Failure 500 {string} string "PD server failed to proceed the request." // // Router /keyspaces/{name}/config [patch] func UpdateKeyspaceConfig(c *gin.Context) { - svr := c.MustGet("server").(*server.Server) + svr := c.MustGet(middlewares.ServerContextKey).(*server.Server) manager := svr.GetKeyspaceManager() + if manager == nil { + c.AbortWithStatusJSON(http.StatusInternalServerError, managerUninitializedErr) + return + } name := c.Param("name") configParams := &UpdateConfigParams{} err := c.BindJSON(configParams) @@ -292,19 +329,23 @@ type UpdateStateParam struct { // UpdateKeyspaceState update the target keyspace's state. // -// @Tags keyspaces -// @Summary Update keyspace state. -// @Param name path string true "Keyspace Name" -// @Param body body UpdateStateParam true "New state for the keyspace" -// @Produce json -// @Success 200 {object} KeyspaceMeta -// @Failure 400 {string} string "The input is invalid." -// @Failure 500 {string} string "PD server failed to proceed the request." +// @Tags keyspaces +// @Summary Update keyspace state. +// @Param name path string true "Keyspace Name" +// @Param body body UpdateStateParam true "New state for the keyspace" +// @Produce json +// @Success 200 {object} KeyspaceMeta +// @Failure 400 {string} string "The input is invalid." +// @Failure 500 {string} string "PD server failed to proceed the request." // // Router /keyspaces/{name}/state [put] func UpdateKeyspaceState(c *gin.Context) { - svr := c.MustGet("server").(*server.Server) + svr := c.MustGet(middlewares.ServerContextKey).(*server.Server) manager := svr.GetKeyspaceManager() + if manager == nil { + c.AbortWithStatusJSON(http.StatusInternalServerError, managerUninitializedErr) + return + } name := c.Param("name") param := &UpdateStateParam{} err := c.BindJSON(param) diff --git a/server/apiv2/handlers/tso_keyspace_group.go b/server/apiv2/handlers/tso_keyspace_group.go new file mode 100644 index 00000000000..fc7ef680104 --- /dev/null +++ b/server/apiv2/handlers/tso_keyspace_group.go @@ -0,0 +1,543 @@ +// Copyright 2023 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package handlers + +import ( + "net/http" + "strconv" + "strings" + "sync" + + "github.com/gin-gonic/gin" + "github.com/pkg/errors" + "github.com/tikv/pd/pkg/errs" + "github.com/tikv/pd/pkg/mcs/utils" + "github.com/tikv/pd/pkg/slice" + "github.com/tikv/pd/pkg/storage/endpoint" + "github.com/tikv/pd/server" + "github.com/tikv/pd/server/apiv2/middlewares" +) + +const groupManagerUninitializedErr = "keyspace group manager is not initialized" + +// RegisterTSOKeyspaceGroup registers keyspace group handlers to the server. +func RegisterTSOKeyspaceGroup(r *gin.RouterGroup) { + router := r.Group("tso/keyspace-groups") + router.Use(middlewares.BootstrapChecker()) + router.POST("", CreateKeyspaceGroups) + router.GET("", GetKeyspaceGroups) + router.GET("/:id", GetKeyspaceGroupByID) + router.DELETE("/:id", DeleteKeyspaceGroupByID) + router.PATCH("/:id", SetNodesForKeyspaceGroup) // only to support set nodes + router.PATCH("/:id/*node", SetPriorityForKeyspaceGroup) // only to support set priority + router.POST("/:id/alloc", AllocNodesForKeyspaceGroup) + router.POST("/:id/split", SplitKeyspaceGroupByID) + router.DELETE("/:id/split", FinishSplitKeyspaceByID) + router.POST("/:id/merge", MergeKeyspaceGroups) + router.DELETE("/:id/merge", FinishMergeKeyspaceByID) +} + +// CreateKeyspaceGroupParams defines the params for creating keyspace groups. +type CreateKeyspaceGroupParams struct { + KeyspaceGroups []*endpoint.KeyspaceGroup `json:"keyspace-groups"` +} + +// CreateKeyspaceGroups creates keyspace groups. +func CreateKeyspaceGroups(c *gin.Context) { + createParams := &CreateKeyspaceGroupParams{} + err := c.BindJSON(createParams) + if err != nil { + c.AbortWithStatusJSON(http.StatusBadRequest, errs.ErrBindJSON.Wrap(err).GenWithStackByCause()) + return + } + for _, keyspaceGroup := range createParams.KeyspaceGroups { + if !isValid(keyspaceGroup.ID) { + c.AbortWithStatusJSON(http.StatusBadRequest, "invalid keyspace group id") + return + } + if keyspaceGroup.UserKind == "" { + keyspaceGroup.UserKind = endpoint.Basic.String() + } else if !endpoint.IsUserKindValid(keyspaceGroup.UserKind) { + c.AbortWithStatusJSON(http.StatusBadRequest, "invalid user kind") + return + } + } + + svr := c.MustGet(middlewares.ServerContextKey).(*server.Server) + manager := svr.GetKeyspaceGroupManager() + if manager == nil { + c.AbortWithStatusJSON(http.StatusInternalServerError, groupManagerUninitializedErr) + return + } + err = manager.CreateKeyspaceGroups(createParams.KeyspaceGroups) + if err != nil { + c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error()) + return + } + c.JSON(http.StatusOK, nil) +} + +// GetKeyspaceGroups gets keyspace groups from the start ID with limit. +// If limit is 0, it will load all keyspace groups from the start ID. +func GetKeyspaceGroups(c *gin.Context) { + scanStart, scanLimit, err := parseLoadAllQuery(c) + if err != nil { + c.AbortWithStatusJSON(http.StatusBadRequest, err.Error()) + return + } + + svr := c.MustGet(middlewares.ServerContextKey).(*server.Server) + manager := svr.GetKeyspaceGroupManager() + if manager == nil { + c.AbortWithStatusJSON(http.StatusInternalServerError, groupManagerUninitializedErr) + return + } + keyspaceGroups, err := manager.GetKeyspaceGroups(scanStart, scanLimit) + if err != nil { + c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error()) + return + } + var kgs []*endpoint.KeyspaceGroup + state, set := c.GetQuery("state") + if set { + state := strings.ToLower(state) + switch state { + case "merge": + for _, keyspaceGroup := range keyspaceGroups { + if keyspaceGroup.MergeState != nil { + kgs = append(kgs, keyspaceGroup) + } + } + case "split": + for _, keyspaceGroup := range keyspaceGroups { + if keyspaceGroup.SplitState != nil { + kgs = append(kgs, keyspaceGroup) + } + } + default: + } + } else { + kgs = keyspaceGroups + } + + c.IndentedJSON(http.StatusOK, kgs) +} + +// GetKeyspaceGroupPrimaryResponse defines the response for getting primary node of keyspace group. +type GetKeyspaceGroupPrimaryResponse struct { + ID uint32 `json:"id"` + Primary string `json:"primary"` +} + +// GetKeyspaceGroupByID gets keyspace group by ID. +func GetKeyspaceGroupByID(c *gin.Context) { + id, err := validateKeyspaceGroupID(c) + if err != nil { + c.AbortWithStatusJSON(http.StatusBadRequest, "invalid keyspace group id") + return + } + + svr := c.MustGet(middlewares.ServerContextKey).(*server.Server) + manager := svr.GetKeyspaceGroupManager() + if manager == nil { + c.AbortWithStatusJSON(http.StatusInternalServerError, groupManagerUninitializedErr) + return + } + + fields := c.Query("fields") // split by comma if need to add more fields + if fields == "primary" { + primary, err := manager.GetKeyspaceGroupPrimaryByID(id) + if err != nil { + c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error()) + return + } + c.JSON(http.StatusOK, &GetKeyspaceGroupPrimaryResponse{ + ID: id, + Primary: primary, + }) + return + } + + kg, err := manager.GetKeyspaceGroupByID(id) + if err != nil { + c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error()) + return + } + c.IndentedJSON(http.StatusOK, kg) +} + +// DeleteKeyspaceGroupByID deletes keyspace group by ID. +func DeleteKeyspaceGroupByID(c *gin.Context) { + id, err := validateKeyspaceGroupID(c) + if err != nil { + c.AbortWithStatusJSON(http.StatusBadRequest, "invalid keyspace group id") + return + } + + svr := c.MustGet(middlewares.ServerContextKey).(*server.Server) + manager := svr.GetKeyspaceGroupManager() + if manager == nil { + c.AbortWithStatusJSON(http.StatusInternalServerError, groupManagerUninitializedErr) + return + } + kg, err := manager.DeleteKeyspaceGroupByID(id) + if err != nil { + c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error()) + return + } + c.IndentedJSON(http.StatusOK, kg) +} + +// SplitKeyspaceGroupByIDParams defines the params for splitting a keyspace group. +type SplitKeyspaceGroupByIDParams struct { + NewID uint32 `json:"new-id"` + Keyspaces []uint32 `json:"keyspaces"` + // StartKeyspaceID and EndKeyspaceID are used to indicate the range of keyspaces to be split. + StartKeyspaceID uint32 `json:"start-keyspace-id"` + EndKeyspaceID uint32 `json:"end-keyspace-id"` +} + +var patrolKeyspaceAssignmentState struct { + sync.RWMutex + patrolled bool +} + +// SplitKeyspaceGroupByID splits keyspace group by ID into a new keyspace group with the given new ID. +// And the keyspaces in the old keyspace group will be moved to the new keyspace group. +func SplitKeyspaceGroupByID(c *gin.Context) { + id, err := validateKeyspaceGroupID(c) + if err != nil { + c.AbortWithStatusJSON(http.StatusBadRequest, "invalid keyspace group id") + return + } + splitParams := &SplitKeyspaceGroupByIDParams{} + err = c.BindJSON(splitParams) + if err != nil { + c.AbortWithStatusJSON(http.StatusBadRequest, errs.ErrBindJSON.Wrap(err).GenWithStackByCause()) + return + } + if !isValid(splitParams.NewID) { + c.AbortWithStatusJSON(http.StatusBadRequest, "invalid keyspace group id") + return + } + if len(splitParams.Keyspaces) == 0 && splitParams.StartKeyspaceID == 0 && splitParams.EndKeyspaceID == 0 { + c.AbortWithStatusJSON(http.StatusBadRequest, "invalid empty keyspaces") + return + } + if splitParams.StartKeyspaceID < utils.DefaultKeyspaceID || + splitParams.StartKeyspaceID > splitParams.EndKeyspaceID { + c.AbortWithStatusJSON(http.StatusBadRequest, "invalid start/end keyspace id") + return + } + + svr := c.MustGet(middlewares.ServerContextKey).(*server.Server) + patrolKeyspaceAssignmentState.Lock() + if !patrolKeyspaceAssignmentState.patrolled { + // Patrol keyspace assignment before splitting keyspace group. + manager := svr.GetKeyspaceManager() + if manager == nil { + c.AbortWithStatusJSON(http.StatusInternalServerError, managerUninitializedErr) + return + } + err = manager.PatrolKeyspaceAssignment(splitParams.StartKeyspaceID, splitParams.EndKeyspaceID) + if err != nil { + c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error()) + patrolKeyspaceAssignmentState.Unlock() + return + } + patrolKeyspaceAssignmentState.patrolled = true + } + patrolKeyspaceAssignmentState.Unlock() + groupManager := svr.GetKeyspaceGroupManager() + if groupManager == nil { + c.AbortWithStatusJSON(http.StatusInternalServerError, groupManagerUninitializedErr) + return + } + // Split keyspace group. + err = groupManager.SplitKeyspaceGroupByID( + id, splitParams.NewID, + splitParams.Keyspaces, splitParams.StartKeyspaceID, splitParams.EndKeyspaceID) + if err != nil { + c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error()) + return + } + c.JSON(http.StatusOK, nil) +} + +// FinishSplitKeyspaceByID finishes split keyspace group by ID. +func FinishSplitKeyspaceByID(c *gin.Context) { + id, err := validateKeyspaceGroupID(c) + if err != nil { + c.AbortWithStatusJSON(http.StatusBadRequest, "invalid keyspace group id") + return + } + + svr := c.MustGet(middlewares.ServerContextKey).(*server.Server) + manager := svr.GetKeyspaceGroupManager() + err = manager.FinishSplitKeyspaceByID(id) + if err != nil { + c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error()) + return + } + c.JSON(http.StatusOK, nil) +} + +// MergeKeyspaceGroupsParams defines the params for merging the keyspace groups. +type MergeKeyspaceGroupsParams struct { + MergeList []uint32 `json:"merge-list"` + MergeAllIntoDefault bool `json:"merge-all-into-default"` +} + +// MergeKeyspaceGroups merges the keyspace groups in the merge list into the target keyspace group. +func MergeKeyspaceGroups(c *gin.Context) { + id, err := validateKeyspaceGroupID(c) + if err != nil { + c.AbortWithStatusJSON(http.StatusBadRequest, "invalid keyspace group id") + return + } + mergeParams := &MergeKeyspaceGroupsParams{} + err = c.BindJSON(mergeParams) + if err != nil { + c.AbortWithStatusJSON(http.StatusBadRequest, errs.ErrBindJSON.Wrap(err).GenWithStackByCause()) + return + } + if len(mergeParams.MergeList) == 0 && !mergeParams.MergeAllIntoDefault { + c.AbortWithStatusJSON(http.StatusBadRequest, "invalid empty merge list") + return + } + if len(mergeParams.MergeList) > 0 && mergeParams.MergeAllIntoDefault { + c.AbortWithStatusJSON(http.StatusBadRequest, "non-empty merge list when merge all into default") + return + } + for _, mergeID := range mergeParams.MergeList { + if !isValid(mergeID) { + c.AbortWithStatusJSON(http.StatusBadRequest, "invalid keyspace group id") + return + } + } + + svr := c.MustGet(middlewares.ServerContextKey).(*server.Server) + groupManager := svr.GetKeyspaceGroupManager() + if groupManager == nil { + c.AbortWithStatusJSON(http.StatusInternalServerError, groupManagerUninitializedErr) + return + } + // Merge keyspace group. + if mergeParams.MergeAllIntoDefault { + err = groupManager.MergeAllIntoDefaultKeyspaceGroup() + } else { + err = groupManager.MergeKeyspaceGroups(id, mergeParams.MergeList) + } + if err != nil { + c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error()) + return + } + c.JSON(http.StatusOK, nil) +} + +// FinishMergeKeyspaceByID finishes merging keyspace group by ID. +func FinishMergeKeyspaceByID(c *gin.Context) { + id, err := validateKeyspaceGroupID(c) + if err != nil { + c.AbortWithStatusJSON(http.StatusBadRequest, "invalid keyspace group id") + return + } + + svr := c.MustGet(middlewares.ServerContextKey).(*server.Server) + manager := svr.GetKeyspaceGroupManager() + err = manager.FinishMergeKeyspaceByID(id) + if err != nil { + c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error()) + return + } + c.JSON(http.StatusOK, nil) +} + +// AllocNodesForKeyspaceGroupParams defines the params for allocating nodes for keyspace groups. +type AllocNodesForKeyspaceGroupParams struct { + Replica int `json:"replica"` +} + +// AllocNodesForKeyspaceGroup allocates nodes for keyspace group. +func AllocNodesForKeyspaceGroup(c *gin.Context) { + id, err := validateKeyspaceGroupID(c) + if err != nil { + c.AbortWithStatusJSON(http.StatusBadRequest, "invalid keyspace group id") + return + } + svr := c.MustGet(middlewares.ServerContextKey).(*server.Server) + manager := svr.GetKeyspaceGroupManager() + if manager == nil { + c.AbortWithStatusJSON(http.StatusInternalServerError, groupManagerUninitializedErr) + return + } + allocParams := &AllocNodesForKeyspaceGroupParams{} + err = c.BindJSON(allocParams) + if err != nil { + c.AbortWithStatusJSON(http.StatusBadRequest, errs.ErrBindJSON.Wrap(err).GenWithStackByCause()) + return + } + if manager.GetNodesCount() < allocParams.Replica || allocParams.Replica < utils.DefaultKeyspaceGroupReplicaCount { + c.AbortWithStatusJSON(http.StatusBadRequest, "invalid replica, should be in [2, nodes_num]") + return + } + keyspaceGroup, err := manager.GetKeyspaceGroupByID(id) + if err != nil || keyspaceGroup == nil { + c.AbortWithStatusJSON(http.StatusBadRequest, "keyspace group does not exist") + return + } + if len(keyspaceGroup.Members) >= allocParams.Replica { + c.AbortWithStatusJSON(http.StatusBadRequest, "existed replica is larger than the new replica") + return + } + // get the nodes + nodes, err := manager.AllocNodesForKeyspaceGroup(id, allocParams.Replica) + if err != nil { + c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error()) + return + } + c.JSON(http.StatusOK, nodes) +} + +// SetNodesForKeyspaceGroupParams defines the params for setting nodes for keyspace group. +// Notes: it should be used carefully. +type SetNodesForKeyspaceGroupParams struct { + Nodes []string `json:"nodes"` +} + +// SetNodesForKeyspaceGroup sets nodes for keyspace group. +func SetNodesForKeyspaceGroup(c *gin.Context) { + id, err := validateKeyspaceGroupID(c) + if err != nil { + c.AbortWithStatusJSON(http.StatusBadRequest, "invalid keyspace group id") + return + } + svr := c.MustGet(middlewares.ServerContextKey).(*server.Server) + manager := svr.GetKeyspaceGroupManager() + if manager == nil { + c.AbortWithStatusJSON(http.StatusInternalServerError, groupManagerUninitializedErr) + return + } + setParams := &SetNodesForKeyspaceGroupParams{} + err = c.BindJSON(setParams) + if err != nil { + c.AbortWithStatusJSON(http.StatusBadRequest, errs.ErrBindJSON.Wrap(err).GenWithStackByCause()) + return + } + // check if keyspace group exists + keyspaceGroup, err := manager.GetKeyspaceGroupByID(id) + if err != nil || keyspaceGroup == nil { + c.AbortWithStatusJSON(http.StatusBadRequest, "keyspace group does not exist") + return + } + // check if nodes is less than default replica count + if len(setParams.Nodes) < utils.DefaultKeyspaceGroupReplicaCount { + c.AbortWithStatusJSON(http.StatusBadRequest, "invalid num of nodes") + return + } + // check if node exists + for _, node := range setParams.Nodes { + if !manager.IsExistNode(node) { + c.AbortWithStatusJSON(http.StatusBadRequest, "node does not exist") + return + } + } + // set nodes + err = manager.SetNodesForKeyspaceGroup(id, setParams.Nodes) + if err != nil { + c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error()) + return + } + c.JSON(http.StatusOK, nil) +} + +// SetPriorityForKeyspaceGroupParams defines the params for setting priority of tso node for the keyspace group. +type SetPriorityForKeyspaceGroupParams struct { + Priority int `json:"priority"` +} + +// SetPriorityForKeyspaceGroup sets priority of tso node for the keyspace group. +func SetPriorityForKeyspaceGroup(c *gin.Context) { + id, err := validateKeyspaceGroupID(c) + if err != nil { + c.AbortWithStatusJSON(http.StatusBadRequest, "invalid keyspace group id") + return + } + node, err := parseNodeAddress(c) + if err != nil { + c.AbortWithStatusJSON(http.StatusBadRequest, "invalid node address") + return + } + svr := c.MustGet(middlewares.ServerContextKey).(*server.Server) + manager := svr.GetKeyspaceGroupManager() + if manager == nil { + c.AbortWithStatusJSON(http.StatusInternalServerError, groupManagerUninitializedErr) + return + } + setParams := &SetPriorityForKeyspaceGroupParams{} + err = c.BindJSON(setParams) + if err != nil { + c.AbortWithStatusJSON(http.StatusBadRequest, errs.ErrBindJSON.Wrap(err).GenWithStackByCause()) + return + } + // check if keyspace group exists + kg, err := manager.GetKeyspaceGroupByID(id) + if err != nil || kg == nil { + c.AbortWithStatusJSON(http.StatusBadRequest, "keyspace group does not exist") + return + } + // check if node exists + members := kg.Members + if slice.NoneOf(members, func(i int) bool { + return members[i].Address == node + }) { + c.AbortWithStatusJSON(http.StatusBadRequest, "tso node does not exist in the keyspace group") + } + // set priority + err = manager.SetPriorityForKeyspaceGroup(id, node, setParams.Priority) + if err != nil { + c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error()) + return + } + c.JSON(http.StatusOK, nil) +} + +func validateKeyspaceGroupID(c *gin.Context) (uint32, error) { + id, err := strconv.ParseUint(c.Param("id"), 10, 64) + if err != nil { + return 0, err + } + if !isValid(uint32(id)) { + return 0, errors.Errorf("invalid keyspace group id: %d", id) + } + return uint32(id), nil +} + +func parseNodeAddress(c *gin.Context) (string, error) { + node := c.Param("node") + if node == "" { + return "", errors.New("invalid node address") + } + // In pd-ctl, we use url.PathEscape to escape the node address and replace the % to \%. + // But in the gin framework, it will unescape the node address automatically. + // So we need to replace the \/ to /. + node = strings.ReplaceAll(node, "\\/", "/") + node = strings.TrimPrefix(node, "/") + return node, nil +} + +func isValid(id uint32) bool { + return id >= utils.DefaultKeyspaceGroupID && id <= utils.MaxKeyspaceGroupCountInUse +} diff --git a/server/apiv2/middlewares/bootstrap_checker.go b/server/apiv2/middlewares/bootstrap_checker.go index 384847be931..794316d3d0f 100644 --- a/server/apiv2/middlewares/bootstrap_checker.go +++ b/server/apiv2/middlewares/bootstrap_checker.go @@ -22,10 +22,13 @@ import ( "github.com/tikv/pd/server" ) +// ServerContextKey is the key to get server from gin.Context. +const ServerContextKey = "server" + // BootstrapChecker is a middleware to check if raft cluster is started. func BootstrapChecker() gin.HandlerFunc { return func(c *gin.Context) { - svr := c.MustGet("server").(*server.Server) + svr := c.MustGet(ServerContextKey).(*server.Server) rc := svr.GetRaftCluster() if rc == nil { c.AbortWithStatusJSON(http.StatusInternalServerError, errs.ErrNotBootstrapped.FastGenByArgs().Error()) diff --git a/server/apiv2/middlewares/redirector.go b/server/apiv2/middlewares/redirector.go index 6f92c15754e..5539dd089dc 100644 --- a/server/apiv2/middlewares/redirector.go +++ b/server/apiv2/middlewares/redirector.go @@ -30,7 +30,7 @@ import ( // Redirector is a middleware to redirect the request to the right place. func Redirector() gin.HandlerFunc { return func(c *gin.Context) { - svr := c.MustGet("server").(*server.Server) + svr := c.MustGet(ServerContextKey).(*server.Server) allowFollowerHandle := len(c.Request.Header.Get(serverapi.PDAllowFollowerHandle)) > 0 isLeader := svr.GetMember().IsLeader() if !svr.IsClosed() && (allowFollowerHandle || isLeader) { diff --git a/server/apiv2/router.go b/server/apiv2/router.go index 5307b2b5c0f..383d336caae 100644 --- a/server/apiv2/router.go +++ b/server/apiv2/router.go @@ -57,11 +57,12 @@ func NewV2Handler(_ context.Context, svr *server.Server) (http.Handler, apiutil. }) router := gin.New() router.Use(func(c *gin.Context) { - c.Set("server", svr) + c.Set(middlewares.ServerContextKey, svr) c.Next() }) router.Use(middlewares.Redirector()) root := router.Group(apiV2Prefix) handlers.RegisterKeyspace(root) + handlers.RegisterTSOKeyspaceGroup(root) return router, group, nil } diff --git a/server/cluster/cluster.go b/server/cluster/cluster.go index f4b13513db6..b2ad25cf0ca 100644 --- a/server/cluster/cluster.go +++ b/server/cluster/cluster.go @@ -22,7 +22,6 @@ import ( "strconv" "strings" "sync" - "sync/atomic" "time" "github.com/coreos/go-semver/semver" @@ -34,22 +33,30 @@ import ( "github.com/tikv/pd/pkg/core" "github.com/tikv/pd/pkg/core/storelimit" "github.com/tikv/pd/pkg/errs" + "github.com/tikv/pd/pkg/gc" "github.com/tikv/pd/pkg/gctuner" "github.com/tikv/pd/pkg/id" + "github.com/tikv/pd/pkg/keyspace" "github.com/tikv/pd/pkg/memory" "github.com/tikv/pd/pkg/progress" + "github.com/tikv/pd/pkg/replication" "github.com/tikv/pd/pkg/schedule" "github.com/tikv/pd/pkg/schedule/checker" sc "github.com/tikv/pd/pkg/schedule/config" "github.com/tikv/pd/pkg/schedule/hbstream" "github.com/tikv/pd/pkg/schedule/labeler" + "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/placement" + "github.com/tikv/pd/pkg/schedule/scatter" "github.com/tikv/pd/pkg/schedule/schedulers" + "github.com/tikv/pd/pkg/schedule/splitter" "github.com/tikv/pd/pkg/slice" "github.com/tikv/pd/pkg/statistics" "github.com/tikv/pd/pkg/statistics/buckets" "github.com/tikv/pd/pkg/storage" "github.com/tikv/pd/pkg/storage/endpoint" + "github.com/tikv/pd/pkg/syncer" + "github.com/tikv/pd/pkg/unsaferecovery" "github.com/tikv/pd/pkg/utils/etcdutil" "github.com/tikv/pd/pkg/utils/logutil" "github.com/tikv/pd/pkg/utils/netutil" @@ -57,8 +64,6 @@ import ( "github.com/tikv/pd/pkg/utils/typeutil" "github.com/tikv/pd/pkg/versioninfo" "github.com/tikv/pd/server/config" - syncer "github.com/tikv/pd/server/region_syncer" - "github.com/tikv/pd/server/replication" "go.etcd.io/etcd/clientv3" "go.uber.org/zap" ) @@ -94,6 +99,10 @@ const ( removingAction = "removing" preparingAction = "preparing" gcTunerCheckCfgInterval = 10 * time.Second + + // minSnapshotDurationSec is the minimum duration that a store can tolerate. + // It should enlarge the limiter if the snapshot's duration is less than this value. + minSnapshotDurationSec = 5 ) // Server is the interface for cluster. @@ -107,6 +116,9 @@ type Server interface { GetBasicCluster() *core.BasicCluster GetMembers() ([]*pdpb.Member, error) ReplicateFileToMember(ctx context.Context, member *pdpb.Member, name string, data []byte) error + GetKeyspaceGroupManager() *keyspace.GroupManager + IsAPIServiceMode() bool + GetSafePointV2Manager() *gc.SafePointV2Manager } // RaftCluster is used for cluster config management. @@ -127,7 +139,7 @@ type RaftCluster struct { etcdClient *clientv3.Client httpClient *http.Client - running atomic.Bool + running bool meta *metapb.Cluster storeConfigManager *config.StoreConfigManager storage storage.Storage @@ -143,7 +155,7 @@ type RaftCluster struct { core *core.BasicCluster // cached cluster info opt *config.PersistOptions limiter *StoreLimiter - coordinator *coordinator + coordinator *schedule.Coordinator labelLevelStats *statistics.LabelStatistics regionStats *statistics.RegionStatistics hotStat *statistics.HotStat @@ -152,10 +164,11 @@ type RaftCluster struct { ruleManager *placement.RuleManager regionLabeler *labeler.RegionLabeler replicationMode *replication.ModeManager - unsafeRecoveryController *unsafeRecoveryController + unsafeRecoveryController *unsaferecovery.Controller progressManager *progress.Manager regionSyncer *syncer.RegionSyncer changedRegions chan *core.RegionInfo + keyspaceGroupManager *keyspace.GroupManager } // Status saves some state information. @@ -179,10 +192,25 @@ func NewRaftCluster(ctx context.Context, clusterID uint64, regionSyncer *syncer. } // GetStoreConfig returns the store config. -func (c *RaftCluster) GetStoreConfig() sc.StoreConfig { +func (c *RaftCluster) GetStoreConfig() sc.StoreConfigProvider { return c.storeConfigManager.GetStoreConfig() } +// GetCheckerConfig returns the checker config. +func (c *RaftCluster) GetCheckerConfig() sc.CheckerConfigProvider { + return c.GetOpts() +} + +// GetSchedulerConfig returns the scheduler config. +func (c *RaftCluster) GetSchedulerConfig() sc.SchedulerConfigProvider { + return c.GetOpts() +} + +// GetSharedConfig returns the shared config. +func (c *RaftCluster) GetSharedConfig() sc.SharedConfigProvider { + return c.GetOpts() +} + // LoadClusterStatus loads the cluster status. func (c *RaftCluster) LoadClusterStatus() (*Status, error) { bootstrapTime, err := c.loadBootstrapTime() @@ -205,7 +233,7 @@ func (c *RaftCluster) LoadClusterStatus() (*Status, error) { } func (c *RaftCluster) isInitialized() bool { - if c.core.GetRegionCount() > 1 { + if c.core.GetTotalRegionCount() > 1 { return true } region := c.core.GetRegionByKey(nil) @@ -233,7 +261,8 @@ func (c *RaftCluster) InitCluster( id id.Allocator, opt *config.PersistOptions, storage storage.Storage, - basicCluster *core.BasicCluster) { + basicCluster *core.BasicCluster, + keyspaceGroupManager *keyspace.GroupManager) { c.core, c.opt, c.storage, c.id = basicCluster, opt, storage, id c.ctx, c.cancel = context.WithCancel(c.serverCtx) c.labelLevelStats = statistics.NewLabelStatistics() @@ -243,20 +272,21 @@ func (c *RaftCluster) InitCluster( c.progressManager = progress.NewManager() c.changedRegions = make(chan *core.RegionInfo, defaultChangedRegionsLimit) c.prevStoreLimit = make(map[uint64]map[storelimit.Type]float64) - c.unsafeRecoveryController = newUnsafeRecoveryController(c) + c.unsafeRecoveryController = unsaferecovery.NewController(c) + c.keyspaceGroupManager = keyspaceGroupManager } // Start starts a cluster. func (c *RaftCluster) Start(s Server) error { - if c.IsRunning() { + c.Lock() + defer c.Unlock() + + if c.running { log.Warn("raft cluster has already been started") return nil } - c.Lock() - defer c.Unlock() - - c.InitCluster(s.GetAllocator(), s.GetPersistOptions(), s.GetStorage(), s.GetBasicCluster()) + c.InitCluster(s.GetAllocator(), s.GetPersistOptions(), s.GetStorage(), s.GetBasicCluster(), s.GetKeyspaceGroupManager()) cluster, err := c.LoadClusterInfo() if err != nil { return err @@ -265,6 +295,13 @@ func (c *RaftCluster) Start(s Server) error { return nil } + if s.IsAPIServiceMode() { + err = c.keyspaceGroupManager.Bootstrap(c.ctx) + if err != nil { + return err + } + } + c.ruleManager = placement.NewRuleManager(c.storage, c, c.GetOpts()) if c.opt.IsPlacementRulesEnabled() { err = c.ruleManager.Initialize(c.opt.GetMaxReplicas(), c.opt.GetLocationLabels()) @@ -272,7 +309,6 @@ func (c *RaftCluster) Start(s Server) error { return err } } - c.regionLabeler, err = labeler.NewRegionLabeler(c.ctx, c.storage, regionLabelGCInterval) if err != nil { return err @@ -283,8 +319,8 @@ func (c *RaftCluster) Start(s Server) error { return err } c.storeConfigManager = config.NewStoreConfigManager(c.httpClient) - c.coordinator = newCoordinator(c.ctx, cluster, s.GetHBStreams()) - c.regionStats = statistics.NewRegionStatistics(c.opt, c.ruleManager, c.storeConfigManager) + c.coordinator = schedule.NewCoordinator(c.ctx, cluster, s.GetHBStreams()) + c.regionStats = statistics.NewRegionStatistics(c.core, c.opt, c.ruleManager, c.storeConfigManager) c.limiter = NewStoreLimiter(s.GetPersistOptions()) c.externalTS, err = c.storage.LoadExternalTS() if err != nil { @@ -303,7 +339,7 @@ func (c *RaftCluster) Start(s Server) error { go c.runUpdateStoreStats() go c.startGCTuner() - c.running.Store(true) + c.running = true return nil } @@ -380,30 +416,43 @@ func (c *RaftCluster) startGCTuner() { func (c *RaftCluster) runSyncConfig() { defer logutil.LogPanic() defer c.wg.Done() - ticker := time.NewTicker(time.Minute) defer ticker.Stop() + stores := c.GetStores() + syncFunc := func() { + synced, switchRaftV2Config := syncConfig(c.storeConfigManager, stores) + if switchRaftV2Config { + c.GetOpts().UseRaftV2() + if err := c.opt.Persist(c.GetStorage()); err != nil { + log.Warn("store config persisted failed", zap.Error(err)) + } + } + if !synced { + stores = c.GetStores() + } + } - syncConfig(c.storeConfigManager, stores) + syncFunc() for { select { case <-c.ctx.Done(): log.Info("sync store config job is stopped") return case <-ticker.C: - if !syncConfig(c.storeConfigManager, stores) { - stores = c.GetStores() - } + syncFunc() } } } -func syncConfig(manager *config.StoreConfigManager, stores []*core.StoreInfo) bool { +// syncConfig syncs the config of the stores. +// synced is true if sync config from one tikv. +// switchRaftV2 is true if the config of tikv engine is changed and engine is raft-kv2. +func syncConfig(manager *config.StoreConfigManager, stores []*core.StoreInfo) (synced bool, switchRaftV2 bool) { for index := 0; index < len(stores); index++ { // filter out the stores that are tiflash store := stores[index] - if core.IsStoreContainLabel(store.GetMeta(), core.EngineKey, core.EngineTiFlash) { + if store.IsTiFlash() { continue } @@ -413,16 +462,19 @@ func syncConfig(manager *config.StoreConfigManager, stores []*core.StoreInfo) bo } // it will try next store if the current store is failed. address := netutil.ResolveLoopBackAddr(stores[index].GetStatusAddress(), stores[index].GetAddress()) - if err := manager.ObserveConfig(address); err != nil { + switchRaftV2, err := manager.ObserveConfig(address) + if err != nil { storeSyncConfigEvent.WithLabelValues(address, "fail").Inc() log.Debug("sync store config failed, it will try next store", zap.Error(err)) continue + } else if switchRaftV2 { + storeSyncConfigEvent.WithLabelValues(address, "raft-v2").Inc() } storeSyncConfigEvent.WithLabelValues(address, "succ").Inc() - // it will only try one store. - return true + + return true, switchRaftV2 } - return false + return false, false } // LoadClusterInfo loads cluster related info. @@ -453,7 +505,7 @@ func (c *RaftCluster) LoadClusterInfo() (*RaftCluster, error) { return nil, err } log.Info("load regions", - zap.Int("count", c.core.GetRegionCount()), + zap.Int("count", c.core.GetTotalRegionCount()), zap.Duration("cost", time.Since(start)), ) for _, store := range c.GetStores() { @@ -554,10 +606,11 @@ func (c *RaftCluster) runUpdateStoreStats() { } } +// runCoordinator runs the main scheduling loop. func (c *RaftCluster) runCoordinator() { defer logutil.LogPanic() defer c.wg.Done() - c.coordinator.runUntilStop() + c.coordinator.RunUntilStop() } func (c *RaftCluster) syncRegions() { @@ -575,131 +628,104 @@ func (c *RaftCluster) runReplicationMode() { // Stop stops the cluster. func (c *RaftCluster) Stop() { c.Lock() - if !c.running.CompareAndSwap(true, false) { + if !c.running { c.Unlock() return } - - c.coordinator.stop() + c.running = false + c.coordinator.Stop() c.cancel() c.Unlock() + c.wg.Wait() log.Info("raftcluster is stopped") } // IsRunning return if the cluster is running. func (c *RaftCluster) IsRunning() bool { - return c.running.Load() + c.RLock() + defer c.RUnlock() + return c.running } // Context returns the context of RaftCluster. func (c *RaftCluster) Context() context.Context { - if c.running.Load() { + c.RLock() + defer c.RUnlock() + if c.running { return c.ctx } return nil } // GetCoordinator returns the coordinator. -func (c *RaftCluster) GetCoordinator() *coordinator { +func (c *RaftCluster) GetCoordinator() *schedule.Coordinator { return c.coordinator } // GetOperatorController returns the operator controller. -func (c *RaftCluster) GetOperatorController() *schedule.OperatorController { - return c.coordinator.opController +func (c *RaftCluster) GetOperatorController() *operator.Controller { + return c.coordinator.GetOperatorController() } // SetPrepared set the prepare check to prepared. Only for test purpose. func (c *RaftCluster) SetPrepared() { - c.coordinator.prepareChecker.Lock() - defer c.coordinator.prepareChecker.Unlock() - c.coordinator.prepareChecker.prepared = true + c.coordinator.GetPrepareChecker().SetPrepared() } // GetRegionScatter returns the region scatter. -func (c *RaftCluster) GetRegionScatter() *schedule.RegionScatterer { - return c.coordinator.regionScatterer +func (c *RaftCluster) GetRegionScatter() *scatter.RegionScatterer { + return c.coordinator.GetRegionScatterer() } // GetRegionSplitter returns the region splitter -func (c *RaftCluster) GetRegionSplitter() *schedule.RegionSplitter { - return c.coordinator.regionSplitter +func (c *RaftCluster) GetRegionSplitter() *splitter.RegionSplitter { + return c.coordinator.GetRegionSplitter() } // GetMergeChecker returns merge checker. func (c *RaftCluster) GetMergeChecker() *checker.MergeChecker { - return c.coordinator.checkers.GetMergeChecker() + return c.coordinator.GetMergeChecker() } // GetRuleChecker returns rule checker. func (c *RaftCluster) GetRuleChecker() *checker.RuleChecker { - return c.coordinator.checkers.GetRuleChecker() -} - -// RecordOpStepWithTTL records OpStep with TTL -func (c *RaftCluster) RecordOpStepWithTTL(regionID uint64) { - c.GetRuleChecker().RecordRegionPromoteToNonWitness(regionID) + return c.coordinator.GetRuleChecker() } // GetSchedulers gets all schedulers. func (c *RaftCluster) GetSchedulers() []string { - return c.coordinator.getSchedulers() + return c.coordinator.GetSchedulersController().GetSchedulerNames() } // GetSchedulerHandlers gets all scheduler handlers. func (c *RaftCluster) GetSchedulerHandlers() map[string]http.Handler { - return c.coordinator.getSchedulerHandlers() + return c.coordinator.GetSchedulersController().GetSchedulerHandlers() } // AddScheduler adds a scheduler. -func (c *RaftCluster) AddScheduler(scheduler schedule.Scheduler, args ...string) error { - return c.coordinator.addScheduler(scheduler, args...) +func (c *RaftCluster) AddScheduler(scheduler schedulers.Scheduler, args ...string) error { + return c.coordinator.GetSchedulersController().AddScheduler(scheduler, args...) } // RemoveScheduler removes a scheduler. func (c *RaftCluster) RemoveScheduler(name string) error { - return c.coordinator.removeScheduler(name) + return c.coordinator.GetSchedulersController().RemoveScheduler(name) } // PauseOrResumeScheduler pauses or resumes a scheduler. func (c *RaftCluster) PauseOrResumeScheduler(name string, t int64) error { - return c.coordinator.pauseOrResumeScheduler(name, t) -} - -// IsSchedulerPaused checks if a scheduler is paused. -func (c *RaftCluster) IsSchedulerPaused(name string) (bool, error) { - return c.coordinator.isSchedulerPaused(name) -} - -// IsSchedulerDisabled checks if a scheduler is disabled. -func (c *RaftCluster) IsSchedulerDisabled(name string) (bool, error) { - return c.coordinator.isSchedulerDisabled(name) -} - -// IsSchedulerAllowed checks if a scheduler is allowed. -func (c *RaftCluster) IsSchedulerAllowed(name string) (bool, error) { - return c.coordinator.isSchedulerAllowed(name) -} - -// IsSchedulerExisted checks if a scheduler is existed. -func (c *RaftCluster) IsSchedulerExisted(name string) (bool, error) { - return c.coordinator.isSchedulerExisted(name) + return c.coordinator.GetSchedulersController().PauseOrResumeScheduler(name, t) } // PauseOrResumeChecker pauses or resumes checker. func (c *RaftCluster) PauseOrResumeChecker(name string, t int64) error { - return c.coordinator.pauseOrResumeChecker(name, t) -} - -// IsCheckerPaused returns if checker is paused -func (c *RaftCluster) IsCheckerPaused(name string) (bool, error) { - return c.coordinator.isCheckerPaused(name) + return c.coordinator.PauseOrResumeChecker(name, t) } -// GetAllocator returns cluster's id allocator. -func (c *RaftCluster) GetAllocator() id.Allocator { - return c.id +// AllocID returns a global unique ID. +func (c *RaftCluster) AllocID() (uint64, error) { + return c.id.Alloc() } // GetRegionSyncer returns the region syncer. @@ -724,36 +750,32 @@ func (c *RaftCluster) GetRegionLabeler() *labeler.RegionLabeler { // GetStorage returns the storage. func (c *RaftCluster) GetStorage() storage.Storage { - c.RLock() - defer c.RUnlock() return c.storage } -// SetStorage set the storage for test purpose. -func (c *RaftCluster) SetStorage(s storage.Storage) { - c.Lock() - defer c.Unlock() - c.storage = s -} - // GetOpts returns cluster's configuration. // There is no need a lock since it won't changed. -func (c *RaftCluster) GetOpts() sc.Config { +func (c *RaftCluster) GetOpts() sc.ConfProvider { + return c.opt +} + +// GetPersistOptions returns cluster's configuration. +func (c *RaftCluster) GetPersistOptions() *config.PersistOptions { return c.opt } // GetScheduleConfig returns scheduling configurations. -func (c *RaftCluster) GetScheduleConfig() *config.ScheduleConfig { +func (c *RaftCluster) GetScheduleConfig() *sc.ScheduleConfig { return c.opt.GetScheduleConfig() } // SetScheduleConfig sets the PD scheduling configuration. -func (c *RaftCluster) SetScheduleConfig(cfg *config.ScheduleConfig) { +func (c *RaftCluster) SetScheduleConfig(cfg *sc.ScheduleConfig) { c.opt.SetScheduleConfig(cfg) } // GetReplicationConfig returns replication configurations. -func (c *RaftCluster) GetReplicationConfig() *config.ReplicationConfig { +func (c *RaftCluster) GetReplicationConfig() *sc.ReplicationConfig { return c.opt.GetReplicationConfig() } @@ -769,12 +791,12 @@ func (c *RaftCluster) SetPDServerConfig(cfg *config.PDServerConfig) { // AddSuspectRegions adds regions to suspect list. func (c *RaftCluster) AddSuspectRegions(regionIDs ...uint64) { - c.coordinator.checkers.AddSuspectRegions(regionIDs...) + c.coordinator.GetCheckerController().AddSuspectRegions(regionIDs...) } // GetSuspectRegions gets all suspect regions. func (c *RaftCluster) GetSuspectRegions() []uint64 { - return c.coordinator.checkers.GetSuspectRegions() + return c.coordinator.GetCheckerController().GetSuspectRegions() } // GetHotStat gets hot stat for test. @@ -784,11 +806,11 @@ func (c *RaftCluster) GetHotStat() *statistics.HotStat { // RemoveSuspectRegion removes region from suspect list. func (c *RaftCluster) RemoveSuspectRegion(id uint64) { - c.coordinator.checkers.RemoveSuspectRegion(id) + c.coordinator.GetCheckerController().RemoveSuspectRegion(id) } // GetUnsafeRecoveryController returns the unsafe recovery controller. -func (c *RaftCluster) GetUnsafeRecoveryController() *unsafeRecoveryController { +func (c *RaftCluster) GetUnsafeRecoveryController() *unsaferecovery.Controller { return c.unsafeRecoveryController } @@ -796,19 +818,19 @@ func (c *RaftCluster) GetUnsafeRecoveryController() *unsafeRecoveryController { // The instance of each keyRange is like following format: // [2][]byte: start key/end key func (c *RaftCluster) AddSuspectKeyRange(start, end []byte) { - c.coordinator.checkers.AddSuspectKeyRange(start, end) + c.coordinator.GetCheckerController().AddSuspectKeyRange(start, end) } // PopOneSuspectKeyRange gets one suspect keyRange group. // it would return value and true if pop success, or return empty [][2][]byte and false // if suspectKeyRanges couldn't pop keyRange group. func (c *RaftCluster) PopOneSuspectKeyRange() ([2][]byte, bool) { - return c.coordinator.checkers.PopOneSuspectKeyRange() + return c.coordinator.GetCheckerController().PopOneSuspectKeyRange() } // ClearSuspectKeyRanges clears the suspect keyRanges, only for unit test func (c *RaftCluster) ClearSuspectKeyRanges() { - c.coordinator.checkers.ClearSuspectKeyRanges() + c.coordinator.GetCheckerController().ClearSuspectKeyRanges() } // HandleStoreHeartbeat updates the store status. @@ -822,17 +844,29 @@ func (c *RaftCluster) HandleStoreHeartbeat(heartbeat *pdpb.StoreHeartbeatRequest return errors.Errorf("store %v not found", storeID) } + limit := store.GetStoreLimit() + version := c.opt.GetStoreLimitVersion() + var opt core.StoreCreateOption + if limit == nil || limit.Version() != version { + if version == storelimit.VersionV2 { + limit = storelimit.NewSlidingWindows() + } else { + limit = storelimit.NewStoreRateLimit(0.0) + } + opt = core.SetStoreLimit(limit) + } + nowTime := time.Now() var newStore *core.StoreInfo // If this cluster has slow stores, we should awaken hibernated regions in other stores. if needAwaken, slowStoreIDs := c.NeedAwakenAllRegionsInStore(storeID); needAwaken { log.Info("forcely awaken hibernated regions", zap.Uint64("store-id", storeID), zap.Uint64s("slow-stores", slowStoreIDs)) - newStore = store.Clone(core.SetStoreStats(stats), core.SetLastHeartbeatTS(nowTime), core.SetLastAwakenTime(nowTime)) + newStore = store.Clone(core.SetStoreStats(stats), core.SetLastHeartbeatTS(nowTime), core.SetLastAwakenTime(nowTime), opt) resp.AwakenRegions = &pdpb.AwakenRegions{ AbnormalStores: slowStoreIDs, } } else { - newStore = store.Clone(core.SetStoreStats(stats), core.SetLastHeartbeatTS(nowTime)) + newStore = store.Clone(core.SetStoreStats(stats), core.SetLastHeartbeatTS(nowTime), opt) } if newStore.IsLowSpace(c.opt.GetLowSpaceRatio()) { @@ -842,7 +876,7 @@ func (c *RaftCluster) HandleStoreHeartbeat(heartbeat *pdpb.StoreHeartbeatRequest zap.Uint64("available", newStore.GetAvailable())) } if newStore.NeedPersist() && c.storage != nil { - if err := c.storage.SaveStore(newStore.GetMeta()); err != nil { + if err := c.storage.SaveStoreMeta(newStore.GetMeta()); err != nil { log.Error("failed to persist store", zap.Uint64("store-id", storeID), errs.ZapError(err)) } else { newStore = newStore.Clone(core.SetLastPersistTime(nowTime)) @@ -858,11 +892,6 @@ func (c *RaftCluster) HandleStoreHeartbeat(heartbeat *pdpb.StoreHeartbeatRequest reportInterval := stats.GetInterval() interval := reportInterval.GetEndTimestamp() - reportInterval.GetStartTimestamp() - // c.limiter is nil before "start" is called - if c.limiter != nil && c.opt.GetStoreLimitMode() == "auto" { - c.limiter.Collect(newStore.GetStoreStats()) - } - regions := make(map[uint64]*core.RegionInfo, len(stats.GetPeerStats())) for _, peerStat := range stats.GetPeerStats() { regionID := peerStat.GetRegionId() @@ -893,7 +922,24 @@ func (c *RaftCluster) HandleStoreHeartbeat(heartbeat *pdpb.StoreHeartbeatRequest peerInfo := core.NewPeerInfo(peer, loads, interval) c.hotStat.CheckReadAsync(statistics.NewCheckPeerTask(peerInfo, region)) } - + for _, stat := range stats.GetSnapshotStats() { + // the duration of snapshot is the sum between to send and generate snapshot. + // notice: to enlarge the limit in time, we reset the executing duration when it less than the minSnapshotDurationSec. + dur := stat.GetSendDurationSec() + stat.GetGenerateDurationSec() + if dur < minSnapshotDurationSec { + dur = minSnapshotDurationSec + } + // This error is the diff between the executing duration and the waiting duration. + // The waiting duration is the total duration minus the executing duration. + // so e=executing_duration-waiting_duration=executing_duration-(total_duration-executing_duration)=2*executing_duration-total_duration + // Eg: the total duration is 20s, the executing duration is 10s, the error is 0s. + // Eg: the total duration is 20s, the executing duration is 8s, the error is -4s. + // Eg: the total duration is 10s, the executing duration is 12s, the error is 4s. + // if error is positive, it means the most time cost in executing, pd should send more snapshot to this tikv. + // if error is negative, it means the most time cost in waiting, pd should send less snapshot to this tikv. + e := int64(dur)*2 - int64(stat.GetTotalDurationSec()) + store.Feedback(float64(e)) + } // Here we will compare the reported regions with the previous hot peers to decide if it is still hot. c.hotStat.CheckReadAsync(statistics.NewCollectUnReportedPeerTask(storeID, regions, interval)) return nil @@ -930,7 +976,7 @@ func (c *RaftCluster) processReportBuckets(buckets *metapb.Buckets) error { // IsPrepared return true if the prepare checker is ready. func (c *RaftCluster) IsPrepared() bool { - return c.coordinator.prepareChecker.isPrepared() + return c.coordinator.GetPrepareChecker().IsPrepared() } var regionGuide = core.GenerateRegionGuideFunc(true) @@ -951,7 +997,7 @@ func (c *RaftCluster) processRegionHeartbeat(region *core.RegionInfo) error { peerInfo := core.NewPeerInfo(peer, region.GetWriteLoads(), interval) c.hotStat.CheckWriteAsync(statistics.NewCheckPeerTask(peerInfo, region)) } - c.coordinator.CheckTransferWitnessLeader(region) + c.coordinator.GetSchedulersController().CheckTransferWitnessLeader(region) hasRegionStats := c.regionStats != nil // Save to storage if meta is updated. @@ -999,7 +1045,7 @@ func (c *RaftCluster) processRegionHeartbeat(region *core.RegionInfo) error { } if !c.IsPrepared() && isNew { - c.coordinator.prepareChecker.collect(region) + c.coordinator.GetPrepareChecker().Collect(region) } if c.storage != nil { @@ -1064,7 +1110,7 @@ func (c *RaftCluster) GetPrevRegionByKey(regionKey []byte) *core.RegionInfo { // ScanRegions scans region with start key, until the region contains endKey, or // total number greater than limit. func (c *RaftCluster) ScanRegions(startKey, endKey []byte, limit int) []*core.RegionInfo { - return c.core.ScanRange(startKey, endKey, limit) + return c.core.ScanRegions(startKey, endKey, limit) } // GetRegion searches for a region by ID. @@ -1082,9 +1128,9 @@ func (c *RaftCluster) GetRegions() []*core.RegionInfo { return c.core.GetRegions() } -// GetRegionCount returns total count of regions -func (c *RaftCluster) GetRegionCount() int { - return c.core.GetRegionCount() +// GetTotalRegionCount returns total count of regions +func (c *RaftCluster) GetTotalRegionCount() int { + return c.core.GetTotalRegionCount() } // GetStoreRegions returns all regions' information with a given storeID. @@ -1193,21 +1239,15 @@ func (c *RaftCluster) GetRangeHoles() [][]string { } // UpdateStoreLabels updates a store's location labels -// If 'force' is true, then update the store's labels forcibly. +// If 'force' is true, the origin labels will be overwritten with the new one forcibly. func (c *RaftCluster) UpdateStoreLabels(storeID uint64, labels []*metapb.StoreLabel, force bool) error { store := c.GetStore(storeID) if store == nil { return errs.ErrInvalidStoreID.FastGenByArgs(storeID) } newStore := typeutil.DeepClone(store.GetMeta(), core.StoreFactory) - if force { - newStore.Labels = labels - } else { - // If 'force' isn't set, the given labels will merge into those labels which already existed in the store. - newStore.Labels = core.MergeLabels(newStore.GetLabels(), labels) - } - // PutStore will perform label merge. - return c.putStoreImpl(newStore) + newStore.Labels = labels + return c.putStoreImpl(newStore, force) } // DeleteStoreLabel updates a store's location labels @@ -1228,13 +1268,12 @@ func (c *RaftCluster) DeleteStoreLabel(storeID uint64, labelKey string) error { return errors.Errorf("the label key %s does not exist", labelKey) } newStore.Labels = labels - // PutStore will perform label merge. - return c.putStoreImpl(newStore) + return c.putStoreImpl(newStore, true) } // PutStore puts a store. func (c *RaftCluster) PutStore(store *metapb.Store) error { - if err := c.putStoreImpl(store); err != nil { + if err := c.putStoreImpl(store, false); err != nil { return err } c.OnStoreVersionChange() @@ -1243,8 +1282,9 @@ func (c *RaftCluster) PutStore(store *metapb.Store) error { } // putStoreImpl puts a store. -// If 'force' is true, then overwrite the store's labels. -func (c *RaftCluster) putStoreImpl(store *metapb.Store) error { +// If 'force' is true, the store's labels will overwrite those labels which already existed in the store. +// If 'force' is false, the store's labels will merge into those labels which already existed in the store. +func (c *RaftCluster) putStoreImpl(store *metapb.Store, force bool) error { c.Lock() defer c.Unlock() @@ -1274,6 +1314,9 @@ func (c *RaftCluster) putStoreImpl(store *metapb.Store) error { } else { // Use the given labels to update the store. labels := store.GetLabels() + if !force { + labels = core.MergeLabels(s.GetLabels(), labels) + } // Update an existed store. s = s.Clone( core.SetStoreAddress(store.Address, store.StatusAddress, store.PeerAddress), @@ -1419,7 +1462,7 @@ func (c *RaftCluster) getEvictLeaderStores() (evictStores []uint64) { if c.coordinator == nil { return nil } - handler, ok := c.coordinator.getSchedulerHandlers()[schedulers.EvictLeaderName] + handler, ok := c.coordinator.GetSchedulersController().GetSchedulerHandlers()[schedulers.EvictLeaderName] if !ok { return } @@ -1650,7 +1693,7 @@ func (c *RaftCluster) SetStoreWeight(storeID uint64, leaderWeight, regionWeight func (c *RaftCluster) putStoreLocked(store *core.StoreInfo) error { if c.storage != nil { - if err := c.storage.SaveStore(store.GetMeta()); err != nil { + if err := c.storage.SaveStoreMeta(store.GetMeta()); err != nil { return err } } @@ -1684,7 +1727,7 @@ func (c *RaftCluster) checkStores() { storeID := store.GetID() if store.IsPreparing() { - if store.GetUptime() >= c.opt.GetMaxStorePreparingTime() || c.GetRegionCount() < core.InitClusterRegionThreshold { + if store.GetUptime() >= c.opt.GetMaxStorePreparingTime() || c.GetTotalRegionCount() < core.InitClusterRegionThreshold { if err := c.ReadyToServe(storeID); err != nil { log.Error("change store to serving failed", zap.Stringer("store", store.GetMeta()), @@ -1994,7 +2037,7 @@ func (c *RaftCluster) RemoveTombStoneRecords() error { // deleteStore deletes the store from the cluster. it's concurrent safe. func (c *RaftCluster) deleteStore(store *core.StoreInfo) error { if c.storage != nil { - if err := c.storage.DeleteStore(store.GetMeta()); err != nil { + if err := c.storage.DeleteStoreMeta(store.GetMeta()); err != nil { return err } } @@ -2002,11 +2045,6 @@ func (c *RaftCluster) deleteStore(store *core.StoreInfo) error { return nil } -// SetHotPendingInfluenceMetrics sets pending influence in hot scheduler. -func (c *RaftCluster) SetHotPendingInfluenceMetrics(storeLabel, rwTy, dim string, load float64) { - hotPendingSum.WithLabelValues(storeLabel, rwTy, dim).Set(load) -} - func (c *RaftCluster) collectMetrics() { statsMap := statistics.NewStoreStatisticsMap(c.opt, c.storeConfigManager.GetStoreConfig()) stores := c.GetStores() @@ -2015,8 +2053,8 @@ func (c *RaftCluster) collectMetrics() { } statsMap.Collect() - c.coordinator.collectSchedulerMetrics() - c.coordinator.collectHotSpotMetrics() + c.coordinator.GetSchedulersController().CollectSchedulerMetrics() + c.coordinator.CollectHotSpotMetrics() c.collectClusterMetrics() c.collectHealthStatus() } @@ -2025,8 +2063,8 @@ func (c *RaftCluster) resetMetrics() { statsMap := statistics.NewStoreStatisticsMap(c.opt, c.storeConfigManager.GetStoreConfig()) statsMap.Reset() - c.coordinator.resetSchedulerMetrics() - c.coordinator.resetHotSpotMetrics() + c.coordinator.GetSchedulersController().ResetSchedulerMetrics() + c.coordinator.ResetHotSpotMetrics() c.resetClusterMetrics() c.resetHealthStatus() c.resetProgressIndicator() @@ -2086,15 +2124,8 @@ func (c *RaftCluster) GetRegionStatsByType(typ statistics.RegionStatisticType) [ return c.regionStats.GetRegionStatsByType(typ) } -// GetOfflineRegionStatsByType gets the status of the offline region by types. -func (c *RaftCluster) GetOfflineRegionStatsByType(typ statistics.RegionStatisticType) []*core.RegionInfo { - if c.regionStats == nil { - return nil - } - return c.regionStats.GetOfflineRegionStatsByType(typ) -} - -func (c *RaftCluster) updateRegionsLabelLevelStats(regions []*core.RegionInfo) { +// UpdateRegionsLabelLevelStats updates the status of the region label level by types. +func (c *RaftCluster) UpdateRegionsLabelLevelStats(regions []*core.RegionInfo) { for _, region := range regions { c.labelLevelStats.Observe(region, c.getStoresWithoutLabelLocked(region, core.EngineKey, core.EngineTiFlash), c.opt.GetLocationLabels()) } @@ -2185,13 +2216,13 @@ func (c *RaftCluster) PutMetaCluster(meta *metapb.Cluster) error { // GetRegionStats returns region statistics from cluster. func (c *RaftCluster) GetRegionStats(startKey, endKey []byte) *statistics.RegionStats { - return statistics.GetRegionStats(c.core.ScanRange(startKey, endKey, -1)) + return statistics.GetRegionStats(c.core.ScanRegions(startKey, endKey, -1)) } -// GetRangeCount returns the number of regions in the range. -func (c *RaftCluster) GetRangeCount(startKey, endKey []byte) *statistics.RegionStats { +// GetRegionCount returns the number of regions in the range. +func (c *RaftCluster) GetRegionCount(startKey, endKey []byte) *statistics.RegionStats { stats := &statistics.RegionStats{} - stats.Count = c.core.GetRangeCount(startKey, endKey) + stats.Count = c.core.GetRegionCount(startKey, endKey) return stats } @@ -2227,8 +2258,8 @@ func (c *RaftCluster) RegionReadStats() map[uint64][]*statistics.HotPeerStat { } // BucketsStats returns hot region's buckets stats. -func (c *RaftCluster) BucketsStats(degree int) map[uint64][]*buckets.BucketStat { - task := buckets.NewCollectBucketStatsTask(degree) +func (c *RaftCluster) BucketsStats(degree int, regionIDs ...uint64) map[uint64][]*buckets.BucketStat { + task := buckets.NewCollectBucketStatsTask(degree, regionIDs...) if !c.hotBuckets.CheckAsync(task) { return nil } @@ -2256,7 +2287,7 @@ func (c *RaftCluster) putRegion(region *core.RegionInfo) error { // GetHotWriteRegions gets hot write regions' info. func (c *RaftCluster) GetHotWriteRegions(storeIDs ...uint64) *statistics.StoreHotPeersInfos { - hotWriteRegions := c.coordinator.getHotRegionsByType(statistics.Write) + hotWriteRegions := c.coordinator.GetHotRegionsByType(statistics.Write) if len(storeIDs) > 0 && hotWriteRegions != nil { hotWriteRegions = getHotRegionsByStoreIDs(hotWriteRegions, storeIDs...) } @@ -2265,7 +2296,7 @@ func (c *RaftCluster) GetHotWriteRegions(storeIDs ...uint64) *statistics.StoreHo // GetHotReadRegions gets hot read regions' info. func (c *RaftCluster) GetHotReadRegions(storeIDs ...uint64) *statistics.StoreHotPeersInfos { - hotReadRegions := c.coordinator.getHotRegionsByType(statistics.Read) + hotReadRegions := c.coordinator.GetHotRegionsByType(statistics.Read) if len(storeIDs) > 0 && hotReadRegions != nil { hotReadRegions = getHotRegionsByStoreIDs(hotReadRegions, storeIDs...) } @@ -2296,7 +2327,7 @@ func (c *RaftCluster) GetStoreLimitByType(storeID uint64, typ storelimit.Type) f } // GetAllStoresLimit returns all store limit -func (c *RaftCluster) GetAllStoresLimit() map[uint64]config.StoreLimitConfig { +func (c *RaftCluster) GetAllStoresLimit() map[uint64]sc.StoreLimitConfig { return c.opt.GetAllStoresLimit() } @@ -2308,18 +2339,18 @@ func (c *RaftCluster) AddStoreLimit(store *metapb.Store) { return } - sc := config.StoreLimitConfig{ - AddPeer: config.DefaultStoreLimit.GetDefaultStoreLimit(storelimit.AddPeer), - RemovePeer: config.DefaultStoreLimit.GetDefaultStoreLimit(storelimit.RemovePeer), + slc := sc.StoreLimitConfig{ + AddPeer: sc.DefaultStoreLimit.GetDefaultStoreLimit(storelimit.AddPeer), + RemovePeer: sc.DefaultStoreLimit.GetDefaultStoreLimit(storelimit.RemovePeer), } if core.IsStoreContainLabel(store, core.EngineKey, core.EngineTiFlash) { - sc = config.StoreLimitConfig{ - AddPeer: config.DefaultTiFlashStoreLimit.GetDefaultStoreLimit(storelimit.AddPeer), - RemovePeer: config.DefaultTiFlashStoreLimit.GetDefaultStoreLimit(storelimit.RemovePeer), + slc = sc.StoreLimitConfig{ + AddPeer: sc.DefaultTiFlashStoreLimit.GetDefaultStoreLimit(storelimit.AddPeer), + RemovePeer: sc.DefaultTiFlashStoreLimit.GetDefaultStoreLimit(storelimit.RemovePeer), } } - cfg.StoreLimit[storeID] = sc + cfg.StoreLimit[storeID] = slc c.opt.SetScheduleConfig(cfg) var err error for i := 0; i < persistLimitRetryTimes; i++ { @@ -2446,6 +2477,16 @@ func (c *RaftCluster) GetMinResolvedTS() uint64 { return c.minResolvedTS } +// GetStoreMinResolvedTS returns the min resolved ts of the store. +func (c *RaftCluster) GetStoreMinResolvedTS(storeID uint64) uint64 { + c.RLock() + defer c.RUnlock() + if !c.isInitialized() || !core.IsAvailableForMinResolvedTS(c.GetStore(storeID)) { + return math.MaxUint64 + } + return c.GetStore(storeID).GetMinResolvedTS() +} + // GetExternalTS returns the external timestamp. func (c *RaftCluster) GetExternalTS() uint64 { c.RLock() @@ -2482,14 +2523,14 @@ func (c *RaftCluster) SetStoreLimit(storeID uint64, typ storelimit.Type, ratePer // SetAllStoresLimit sets all store limit for a given type and rate. func (c *RaftCluster) SetAllStoresLimit(typ storelimit.Type, ratePerMin float64) error { old := c.opt.GetScheduleConfig().Clone() - oldAdd := config.DefaultStoreLimit.GetDefaultStoreLimit(storelimit.AddPeer) - oldRemove := config.DefaultStoreLimit.GetDefaultStoreLimit(storelimit.RemovePeer) + oldAdd := sc.DefaultStoreLimit.GetDefaultStoreLimit(storelimit.AddPeer) + oldRemove := sc.DefaultStoreLimit.GetDefaultStoreLimit(storelimit.RemovePeer) c.opt.SetAllStoresLimit(typ, ratePerMin) if err := c.opt.Persist(c.storage); err != nil { // roll back the store limit c.opt.SetScheduleConfig(old) - config.DefaultStoreLimit.SetDefaultStoreLimit(storelimit.AddPeer, oldAdd) - config.DefaultStoreLimit.SetDefaultStoreLimit(storelimit.RemovePeer, oldRemove) + sc.DefaultStoreLimit.SetDefaultStoreLimit(storelimit.AddPeer, oldAdd) + sc.DefaultStoreLimit.SetDefaultStoreLimit(storelimit.RemovePeer, oldRemove) log.Error("persist store limit meet error", errs.ZapError(err)) return err } @@ -2631,31 +2672,12 @@ func IsClientURL(addr string, etcdClient *clientv3.Client) bool { return false } -// cacheCluster include cache info to improve the performance. -type cacheCluster struct { - *RaftCluster - stores []*core.StoreInfo -} - -// GetStores returns store infos from cache -func (c *cacheCluster) GetStores() []*core.StoreInfo { - return c.stores -} - -// newCacheCluster constructor for cache -func newCacheCluster(c *RaftCluster) *cacheCluster { - return &cacheCluster{ - RaftCluster: c, - stores: c.GetStores(), - } -} - // GetPausedSchedulerDelayAt returns DelayAt of a paused scheduler func (c *RaftCluster) GetPausedSchedulerDelayAt(name string) (int64, error) { - return c.coordinator.getPausedSchedulerDelayAt(name) + return c.coordinator.GetSchedulersController().GetPausedSchedulerDelayAt(name) } // GetPausedSchedulerDelayUntil returns DelayUntil of a paused scheduler func (c *RaftCluster) GetPausedSchedulerDelayUntil(name string) (int64, error) { - return c.coordinator.getPausedSchedulerDelayUntil(name) + return c.coordinator.GetSchedulersController().GetPausedSchedulerDelayUntil(name) } diff --git a/server/cluster/cluster_test.go b/server/cluster/cluster_test.go index ca8f3d24e3a..959dba6ee74 100644 --- a/server/cluster/cluster_test.go +++ b/server/cluster/cluster_test.go @@ -16,6 +16,7 @@ package cluster import ( "context" + "encoding/json" "fmt" "math" "math/rand" @@ -23,23 +24,35 @@ import ( "testing" "time" + "github.com/docker/go-units" "github.com/pingcap/errors" "github.com/pingcap/failpoint" + "github.com/pingcap/kvproto/pkg/eraftpb" "github.com/pingcap/kvproto/pkg/metapb" "github.com/pingcap/kvproto/pkg/pdpb" "github.com/stretchr/testify/require" "github.com/tikv/pd/pkg/core" + "github.com/tikv/pd/pkg/core/constant" + "github.com/tikv/pd/pkg/core/storelimit" "github.com/tikv/pd/pkg/errs" "github.com/tikv/pd/pkg/id" + "github.com/tikv/pd/pkg/mock/mockhbstream" "github.com/tikv/pd/pkg/mock/mockid" "github.com/tikv/pd/pkg/progress" "github.com/tikv/pd/pkg/schedule" + sc "github.com/tikv/pd/pkg/schedule/config" + sche "github.com/tikv/pd/pkg/schedule/core" "github.com/tikv/pd/pkg/schedule/filter" + "github.com/tikv/pd/pkg/schedule/hbstream" "github.com/tikv/pd/pkg/schedule/labeler" + "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/placement" "github.com/tikv/pd/pkg/schedule/schedulers" "github.com/tikv/pd/pkg/statistics" "github.com/tikv/pd/pkg/storage" + "github.com/tikv/pd/pkg/utils/operatorutil" + "github.com/tikv/pd/pkg/utils/testutil" + "github.com/tikv/pd/pkg/utils/typeutil" "github.com/tikv/pd/pkg/versioninfo" "github.com/tikv/pd/server/config" ) @@ -50,6 +63,7 @@ func TestStoreHeartbeat(t *testing.T) { defer cancel() _, opt, err := newTestScheduleConfig() + opt.GetScheduleConfig().StoreLimitVersion = "v2" re.NoError(err) cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend(), core.NewBasicCluster()) @@ -61,7 +75,7 @@ func TestStoreHeartbeat(t *testing.T) { for _, region := range regions { re.NoError(cluster.putRegion(region)) } - re.Equal(int(n), cluster.GetRegionCount()) + re.Equal(int(n), cluster.GetTotalRegionCount()) for i, store := range stores { req := &pdpb.StoreHeartbeatRequest{} @@ -84,6 +98,7 @@ func TestStoreHeartbeat(t *testing.T) { s := cluster.GetStore(store.GetID()) re.NotEqual(int64(0), s.GetLastHeartbeatTS().UnixNano()) re.Equal(req.GetStats(), s.GetStoreStats()) + re.Equal("v2", cluster.GetStore(1).GetStoreLimit().Version()) storeMetasAfterHeartbeat = append(storeMetasAfterHeartbeat, s.GetMeta()) } @@ -92,7 +107,7 @@ func TestStoreHeartbeat(t *testing.T) { for i, store := range stores { tmp := &metapb.Store{} - ok, err := cluster.storage.LoadStore(store.GetID(), tmp) + ok, err := cluster.storage.LoadStoreMeta(store.GetID(), tmp) re.True(ok) re.NoError(err) re.Equal(storeMetasAfterHeartbeat[i], tmp) @@ -129,9 +144,11 @@ func TestStoreHeartbeat(t *testing.T) { }, PeerStats: []*pdpb.PeerStat{}, } + cluster.opt.GetScheduleConfig().StoreLimitVersion = "v1" re.NoError(cluster.HandleStoreHeartbeat(hotReq, hotResp)) re.NoError(cluster.HandleStoreHeartbeat(hotReq, hotResp)) re.NoError(cluster.HandleStoreHeartbeat(hotReq, hotResp)) + re.Equal("v1", cluster.GetStore(1).GetStoreLimit().Version()) time.Sleep(20 * time.Millisecond) storeStats := cluster.hotStat.RegionStats(statistics.Read, 3) re.Len(storeStats[1], 1) @@ -220,7 +237,7 @@ func TestSetOfflineStore(t *testing.T) { _, opt, err := newTestScheduleConfig() re.NoError(err) cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend(), core.NewBasicCluster()) - cluster.coordinator = newCoordinator(ctx, cluster, nil) + cluster.coordinator = schedule.NewCoordinator(ctx, cluster, nil) cluster.ruleManager = placement.NewRuleManager(storage.NewStorageWithMemoryBackend(), cluster, cluster.GetOpts()) if opt.IsPlacementRulesEnabled() { err := cluster.ruleManager.Initialize(opt.GetMaxReplicas(), opt.GetLocationLabels()) @@ -286,7 +303,7 @@ func TestSetOfflineWithReplica(t *testing.T) { _, opt, err := newTestScheduleConfig() re.NoError(err) cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend(), core.NewBasicCluster()) - cluster.coordinator = newCoordinator(ctx, cluster, nil) + cluster.coordinator = schedule.NewCoordinator(ctx, cluster, nil) // Put 4 stores. for _, store := range newTestStores(4, "2.0.0") { @@ -302,9 +319,9 @@ func TestSetOfflineWithReplica(t *testing.T) { re.NoError(cluster.RemoveStore(3, true)) } -func addEvictLeaderScheduler(cluster *RaftCluster, storeID uint64) (evictScheduler schedule.Scheduler, err error) { +func addEvictLeaderScheduler(cluster *RaftCluster, storeID uint64) (evictScheduler schedulers.Scheduler, err error) { args := []string{fmt.Sprintf("%d", storeID)} - evictScheduler, err = schedule.CreateScheduler(schedulers.EvictLeaderType, cluster.GetOperatorController(), cluster.storage, schedule.ConfigSliceDecoder(schedulers.EvictLeaderType, args)) + evictScheduler, err = schedulers.CreateScheduler(schedulers.EvictLeaderType, cluster.GetOperatorController(), cluster.storage, schedulers.ConfigSliceDecoder(schedulers.EvictLeaderType, args), cluster.GetCoordinator().GetSchedulersController().RemoveScheduler) if err != nil { return } @@ -325,7 +342,7 @@ func TestSetOfflineStoreWithEvictLeader(t *testing.T) { re.NoError(err) opt.SetMaxReplicas(1) cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend(), core.NewBasicCluster()) - cluster.coordinator = newCoordinator(ctx, cluster, nil) + cluster.coordinator = schedule.NewCoordinator(ctx, cluster, nil) // Put 3 stores. for _, store := range newTestStores(3, "2.0.0") { @@ -371,7 +388,7 @@ func TestReuseAddress(t *testing.T) { _, opt, err := newTestScheduleConfig() re.NoError(err) cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend(), core.NewBasicCluster()) - cluster.coordinator = newCoordinator(ctx, cluster, nil) + cluster.coordinator = schedule.NewCoordinator(ctx, cluster, nil) // Put 4 stores. for _, store := range newTestStores(4, "2.0.0") { re.NoError(cluster.PutStore(store.GetMeta())) @@ -417,7 +434,7 @@ func TestUpStore(t *testing.T) { _, opt, err := newTestScheduleConfig() re.NoError(err) cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend(), core.NewBasicCluster()) - cluster.coordinator = newCoordinator(ctx, cluster, nil) + cluster.coordinator = schedule.NewCoordinator(ctx, cluster, nil) cluster.ruleManager = placement.NewRuleManager(storage.NewStorageWithMemoryBackend(), cluster, cluster.GetOpts()) if opt.IsPlacementRulesEnabled() { err := cluster.ruleManager.Initialize(opt.GetMaxReplicas(), opt.GetLocationLabels()) @@ -462,7 +479,7 @@ func TestRemovingProcess(t *testing.T) { _, opt, err := newTestScheduleConfig() re.NoError(err) cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend(), core.NewBasicCluster()) - cluster.coordinator = newCoordinator(ctx, cluster, nil) + cluster.coordinator = schedule.NewCoordinator(ctx, cluster, nil) cluster.SetPrepared() // Put 5 stores. @@ -520,7 +537,7 @@ func TestDeleteStoreUpdatesClusterVersion(t *testing.T) { _, opt, err := newTestScheduleConfig() re.NoError(err) cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend(), core.NewBasicCluster()) - cluster.coordinator = newCoordinator(ctx, cluster, nil) + cluster.coordinator = schedule.NewCoordinator(ctx, cluster, nil) cluster.ruleManager = placement.NewRuleManager(storage.NewStorageWithMemoryBackend(), cluster, cluster.GetOpts()) if opt.IsPlacementRulesEnabled() { err := cluster.ruleManager.Initialize(opt.GetMaxReplicas(), opt.GetLocationLabels()) @@ -580,7 +597,7 @@ func TestRegionHeartbeatHotStat(t *testing.T) { _, opt, err := newTestScheduleConfig() re.NoError(err) cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend(), core.NewBasicCluster()) - cluster.coordinator = newCoordinator(ctx, cluster, nil) + cluster.coordinator = schedule.NewCoordinator(ctx, cluster, nil) newTestStores(4, "2.0.0") peers := []*metapb.Peer{ { @@ -642,7 +659,7 @@ func TestBucketHeartbeat(t *testing.T) { _, opt, err := newTestScheduleConfig() re.NoError(err) cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend(), core.NewBasicCluster()) - cluster.coordinator = newCoordinator(ctx, cluster, nil) + cluster.coordinator = schedule.NewCoordinator(ctx, cluster, nil) // case1: region is not exist buckets := &metapb.Buckets{ @@ -701,7 +718,7 @@ func TestRegionHeartbeat(t *testing.T) { _, opt, err := newTestScheduleConfig() re.NoError(err) cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend(), core.NewBasicCluster()) - cluster.coordinator = newCoordinator(ctx, cluster, nil) + cluster.coordinator = schedule.NewCoordinator(ctx, cluster, nil) n, np := uint64(3), uint64(3) cluster.wg.Add(1) go cluster.runUpdateStoreStats() @@ -936,7 +953,7 @@ func TestRegionFlowChanged(t *testing.T) { _, opt, err := newTestScheduleConfig() re.NoError(err) cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend(), core.NewBasicCluster()) - cluster.coordinator = newCoordinator(ctx, cluster, nil) + cluster.coordinator = schedule.NewCoordinator(ctx, cluster, nil) regions := []*core.RegionInfo{core.NewTestRegionInfo(1, 1, []byte{}, []byte{})} processRegions := func(regions []*core.RegionInfo) { for _, r := range regions { @@ -961,8 +978,12 @@ func TestRegionSizeChanged(t *testing.T) { _, opt, err := newTestScheduleConfig() re.NoError(err) cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend(), core.NewBasicCluster()) - cluster.coordinator = newCoordinator(ctx, cluster, nil) - cluster.regionStats = statistics.NewRegionStatistics(cluster.GetOpts(), cluster.ruleManager, cluster.storeConfigManager) + cluster.coordinator = schedule.NewCoordinator(ctx, cluster, nil) + cluster.regionStats = statistics.NewRegionStatistics( + cluster.GetBasicCluster(), + cluster.GetOpts(), + cluster.ruleManager, + cluster.storeConfigManager) region := newTestRegions(1, 3, 3)[0] cluster.opt.GetMaxMergeRegionKeys() curMaxMergeSize := int64(cluster.opt.GetMaxMergeRegionSize()) @@ -986,12 +1007,12 @@ func TestRegionSizeChanged(t *testing.T) { cluster.processRegionHeartbeat(region) re.False(cluster.regionStats.IsRegionStatsType(regionID, statistics.UndersizedRegion)) // Test MaxMergeRegionSize and MaxMergeRegionKeys change. - cluster.opt.SetMaxMergeRegionSize((uint64(curMaxMergeSize + 2))) - cluster.opt.SetMaxMergeRegionKeys((uint64(curMaxMergeKeys + 2))) + cluster.opt.SetMaxMergeRegionSize(uint64(curMaxMergeSize + 2)) + cluster.opt.SetMaxMergeRegionKeys(uint64(curMaxMergeKeys + 2)) cluster.processRegionHeartbeat(region) re.True(cluster.regionStats.IsRegionStatsType(regionID, statistics.UndersizedRegion)) - cluster.opt.SetMaxMergeRegionSize((uint64(curMaxMergeSize))) - cluster.opt.SetMaxMergeRegionKeys((uint64(curMaxMergeKeys))) + cluster.opt.SetMaxMergeRegionSize(uint64(curMaxMergeSize)) + cluster.opt.SetMaxMergeRegionKeys(uint64(curMaxMergeKeys)) cluster.processRegionHeartbeat(region) re.False(cluster.regionStats.IsRegionStatsType(regionID, statistics.UndersizedRegion)) } @@ -1004,7 +1025,7 @@ func TestConcurrentReportBucket(t *testing.T) { _, opt, err := newTestScheduleConfig() re.NoError(err) cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend(), core.NewBasicCluster()) - cluster.coordinator = newCoordinator(ctx, cluster, nil) + cluster.coordinator = schedule.NewCoordinator(ctx, cluster, nil) regions := []*core.RegionInfo{core.NewTestRegionInfo(1, 1, []byte{}, []byte{})} heartbeatRegions(re, cluster, regions) @@ -1034,7 +1055,7 @@ func TestConcurrentRegionHeartbeat(t *testing.T) { _, opt, err := newTestScheduleConfig() re.NoError(err) cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend(), core.NewBasicCluster()) - cluster.coordinator = newCoordinator(ctx, cluster, nil) + cluster.coordinator = schedule.NewCoordinator(ctx, cluster, nil) regions := []*core.RegionInfo{core.NewTestRegionInfo(1, 1, []byte{}, []byte{})} regions = core.SplitRegions(regions) @@ -1112,7 +1133,7 @@ func TestRegionLabelIsolationLevel(t *testing.T) { r := core.NewRegionInfo(region, peers[0]) re.NoError(cluster.putRegion(r)) - cluster.updateRegionsLabelLevelStats([]*core.RegionInfo{r}) + cluster.UpdateRegionsLabelLevelStats([]*core.RegionInfo{r}) counter := cluster.labelLevelStats.GetLabelCounter() re.Equal(0, counter["none"]) re.Equal(1, counter["zone"]) @@ -1154,7 +1175,7 @@ func TestHeartbeatSplit(t *testing.T) { _, opt, err := newTestScheduleConfig() re.NoError(err) cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend(), core.NewBasicCluster()) - cluster.coordinator = newCoordinator(ctx, cluster, nil) + cluster.coordinator = schedule.NewCoordinator(ctx, cluster, nil) // 1: [nil, nil) region1 := core.NewRegionInfo(&metapb.Region{Id: 1, RegionEpoch: &metapb.RegionEpoch{Version: 1, ConfVer: 1}}, nil) @@ -1198,7 +1219,7 @@ func TestRegionSplitAndMerge(t *testing.T) { _, opt, err := newTestScheduleConfig() re.NoError(err) cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend(), core.NewBasicCluster()) - cluster.coordinator = newCoordinator(ctx, cluster, nil) + cluster.coordinator = schedule.NewCoordinator(ctx, cluster, nil) regions := []*core.RegionInfo{core.NewTestRegionInfo(1, 1, []byte{}, []byte{})} @@ -1236,7 +1257,7 @@ func TestOfflineAndMerge(t *testing.T) { _, opt, err := newTestScheduleConfig() re.NoError(err) cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend(), core.NewBasicCluster()) - cluster.coordinator = newCoordinator(ctx, cluster, nil) + cluster.coordinator = schedule.NewCoordinator(ctx, cluster, nil) cluster.ruleManager = placement.NewRuleManager(storage.NewStorageWithMemoryBackend(), cluster, cluster.GetOpts()) if opt.IsPlacementRulesEnabled() { err := cluster.ruleManager.Initialize(opt.GetMaxReplicas(), opt.GetLocationLabels()) @@ -1244,8 +1265,12 @@ func TestOfflineAndMerge(t *testing.T) { panic(err) } } - cluster.regionStats = statistics.NewRegionStatistics(cluster.GetOpts(), cluster.ruleManager, cluster.storeConfigManager) - cluster.coordinator = newCoordinator(ctx, cluster, nil) + cluster.regionStats = statistics.NewRegionStatistics( + cluster.GetBasicCluster(), + cluster.GetOpts(), + cluster.ruleManager, + cluster.storeConfigManager) + cluster.coordinator = schedule.NewCoordinator(ctx, cluster, nil) // Put 4 stores. for _, store := range newTestStores(4, "5.0.0") { @@ -1284,13 +1309,13 @@ func TestOfflineAndMerge(t *testing.T) { regions = core.SplitRegions(regions) } heartbeatRegions(re, cluster, regions) - re.Len(cluster.GetOfflineRegionStatsByType(statistics.OfflinePeer), len(regions)) + re.Len(cluster.GetRegionStatsByType(statistics.OfflinePeer), len(regions)) // Merge. for i := 0; i < n; i++ { regions = core.MergeRegions(regions) heartbeatRegions(re, cluster, regions) - re.Len(cluster.GetOfflineRegionStatsByType(statistics.OfflinePeer), len(regions)) + re.Len(cluster.GetRegionStatsByType(statistics.OfflinePeer), len(regions)) } } @@ -1312,20 +1337,31 @@ func TestSyncConfig(t *testing.T) { whiteList []string maxRegionSize uint64 updated bool - }{{ - whiteList: []string{}, - maxRegionSize: uint64(144), - updated: false, - }, { - whiteList: []string{"127.0.0.1:5"}, - maxRegionSize: uint64(10), - updated: true, - }} + }{ + { + whiteList: []string{}, + maxRegionSize: uint64(144), + updated: false, + }, { + whiteList: []string{"127.0.0.1:5"}, + maxRegionSize: uint64(10), + updated: true, + }, + } for _, v := range testdata { tc.storeConfigManager = config.NewTestStoreConfigManager(v.whiteList) re.Equal(uint64(144), tc.GetStoreConfig().GetRegionMaxSize()) - re.Equal(v.updated, syncConfig(tc.storeConfigManager, tc.GetStores())) + success, switchRaftV2 := syncConfig(tc.storeConfigManager, tc.GetStores()) + re.Equal(v.updated, success) + if v.updated { + re.True(switchRaftV2) + tc.opt.UseRaftV2() + re.EqualValues(512, tc.opt.GetMaxMovableHotPeerSize()) + success, switchRaftV2 = syncConfig(tc.storeConfigManager, tc.GetStores()) + re.True(success) + re.False(switchRaftV2) + } re.Equal(v.maxRegionSize, tc.GetStoreConfig().GetRegionMaxSize()) } } @@ -1338,7 +1374,7 @@ func TestUpdateStorePendingPeerCount(t *testing.T) { _, opt, err := newTestScheduleConfig() re.NoError(err) tc := newTestCluster(ctx, opt) - tc.RaftCluster.coordinator = newCoordinator(ctx, tc.RaftCluster, nil) + tc.RaftCluster.coordinator = schedule.NewCoordinator(ctx, tc.RaftCluster, nil) stores := newTestStores(5, "2.0.0") for _, s := range stores { re.NoError(tc.putStoreLocked(s)) @@ -1486,8 +1522,12 @@ func TestCalculateStoreSize1(t *testing.T) { cfg.EnablePlacementRules = true opt.SetReplicationConfig(cfg) cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend(), core.NewBasicCluster()) - cluster.coordinator = newCoordinator(ctx, cluster, nil) - cluster.regionStats = statistics.NewRegionStatistics(cluster.GetOpts(), cluster.ruleManager, cluster.storeConfigManager) + cluster.coordinator = schedule.NewCoordinator(ctx, cluster, nil) + cluster.regionStats = statistics.NewRegionStatistics( + cluster.GetBasicCluster(), + cluster.GetOpts(), + cluster.ruleManager, + cluster.storeConfigManager) // Put 10 stores. for i, store := range newTestStores(10, "6.0.0") { @@ -1569,8 +1609,12 @@ func TestCalculateStoreSize2(t *testing.T) { opt.SetReplicationConfig(cfg) opt.SetMaxReplicas(3) cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend(), core.NewBasicCluster()) - cluster.coordinator = newCoordinator(ctx, cluster, nil) - cluster.regionStats = statistics.NewRegionStatistics(cluster.GetOpts(), cluster.ruleManager, cluster.storeConfigManager) + cluster.coordinator = schedule.NewCoordinator(ctx, cluster, nil) + cluster.regionStats = statistics.NewRegionStatistics( + cluster.GetBasicCluster(), + cluster.GetOpts(), + cluster.ruleManager, + cluster.storeConfigManager) // Put 10 stores. for i, store := range newTestStores(10, "6.0.0") { @@ -1835,11 +1879,119 @@ func TestAwakenStore(t *testing.T) { } } +func TestUpdateAndDeleteLabel(t *testing.T) { + re := require.New(t) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + _, opt, err := newTestScheduleConfig() + re.NoError(err) + cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend(), core.NewBasicCluster()) + stores := newTestStores(1, "6.5.1") + for _, store := range stores { + re.NoError(cluster.PutStore(store.GetMeta())) + } + re.Empty(cluster.GetStore(1).GetLabels()) + // Update label. + cluster.UpdateStoreLabels( + 1, + []*metapb.StoreLabel{ + {Key: "zone", Value: "zone1"}, + {Key: "host", Value: "host1"}, + }, + false, + ) + re.Equal( + []*metapb.StoreLabel{ + {Key: "zone", Value: "zone1"}, + {Key: "host", Value: "host1"}, + }, + cluster.GetStore(1).GetLabels(), + ) + // Update label again. + cluster.UpdateStoreLabels( + 1, + []*metapb.StoreLabel{ + {Key: "mode", Value: "readonly"}, + }, + false, + ) + // Update label with empty value. + cluster.UpdateStoreLabels( + 1, + []*metapb.StoreLabel{}, + false, + ) + re.Equal( + []*metapb.StoreLabel{ + {Key: "zone", Value: "zone1"}, + {Key: "host", Value: "host1"}, + {Key: "mode", Value: "readonly"}, + }, + cluster.GetStore(1).GetLabels(), + ) + // Delete label. + err = cluster.DeleteStoreLabel(1, "mode") + re.NoError(err) + re.Equal( + []*metapb.StoreLabel{ + {Key: "zone", Value: "zone1"}, + {Key: "host", Value: "host1"}, + }, + cluster.GetStore(1).GetLabels(), + ) + // Delete a non-exist label. + err = cluster.DeleteStoreLabel(1, "mode") + re.Error(err) + re.Equal( + []*metapb.StoreLabel{ + {Key: "zone", Value: "zone1"}, + {Key: "host", Value: "host1"}, + }, + cluster.GetStore(1).GetLabels(), + ) + // Update label without force. + cluster.UpdateStoreLabels( + 1, + []*metapb.StoreLabel{}, + false, + ) + re.Equal( + []*metapb.StoreLabel{ + {Key: "zone", Value: "zone1"}, + {Key: "host", Value: "host1"}, + }, + cluster.GetStore(1).GetLabels(), + ) + // Update label with force. + cluster.UpdateStoreLabels( + 1, + []*metapb.StoreLabel{}, + true, + ) + re.Empty(cluster.GetStore(1).GetLabels()) + // Update label first and then reboot the store. + cluster.UpdateStoreLabels( + 1, + []*metapb.StoreLabel{{Key: "mode", Value: "readonly"}}, + false, + ) + re.Equal([]*metapb.StoreLabel{{Key: "mode", Value: "readonly"}}, cluster.GetStore(1).GetLabels()) + // Mock the store doesn't have any label configured. + newStore := typeutil.DeepClone(cluster.GetStore(1).GetMeta(), core.StoreFactory) + newStore.Labels = nil + // Store rebooting will call PutStore. + err = cluster.PutStore(newStore) + re.NoError(err) + // Check the label after rebooting. + re.Equal([]*metapb.StoreLabel{{Key: "mode", Value: "readonly"}}, cluster.GetStore(1).GetLabels()) +} + type testCluster struct { *RaftCluster } -func newTestScheduleConfig() (*config.ScheduleConfig, *config.PersistOptions, error) { +func newTestScheduleConfig() (*sc.ScheduleConfig, *config.PersistOptions, error) { schedulers.Register() cfg := config.NewConfig() cfg.Schedule.TolerantSizeRatio = 5 @@ -1867,7 +2019,7 @@ func newTestRaftCluster( basicCluster *core.BasicCluster, ) *RaftCluster { rc := &RaftCluster{serverCtx: ctx} - rc.InitCluster(id, opt, s, basicCluster) + rc.InitCluster(id, opt, s, basicCluster, nil) rc.ruleManager = placement.NewRuleManager(storage.NewStorageWithMemoryBackend(), rc, opt) if opt.IsPlacementRulesEnabled() { err := rc.ruleManager.Initialize(opt.GetMaxReplicas(), opt.GetLocationLabels()) @@ -1976,7 +2128,7 @@ func checkRegions(re *require.Assertions, cache *core.BasicCluster, regions []*c } } - re.Equal(len(regions), cache.GetRegionCount()) + re.Equal(len(regions), cache.GetTotalRegionCount()) for id, count := range regionCount { re.Equal(count, cache.GetStoreRegionCount(id)) } @@ -2015,3 +2167,1439 @@ func checkStaleRegion(origin *metapb.Region, region *metapb.Region) error { return nil } + +func newTestOperator(regionID uint64, regionEpoch *metapb.RegionEpoch, kind operator.OpKind, steps ...operator.OpStep) *operator.Operator { + return operator.NewTestOperator(regionID, regionEpoch, kind, steps...) +} + +func (c *testCluster) AllocPeer(storeID uint64) (*metapb.Peer, error) { + id, err := c.AllocID() + if err != nil { + return nil, err + } + return &metapb.Peer{Id: id, StoreId: storeID}, nil +} + +func (c *testCluster) addRegionStore(storeID uint64, regionCount int, regionSizes ...uint64) error { + var regionSize uint64 + if len(regionSizes) == 0 { + regionSize = uint64(regionCount) * 10 + } else { + regionSize = regionSizes[0] + } + + stats := &pdpb.StoreStats{} + stats.Capacity = 100 * units.GiB + stats.UsedSize = regionSize * units.MiB + stats.Available = stats.Capacity - stats.UsedSize + newStore := core.NewStoreInfo(&metapb.Store{Id: storeID}, + core.SetStoreStats(stats), + core.SetRegionCount(regionCount), + core.SetRegionSize(int64(regionSize)), + core.SetLastHeartbeatTS(time.Now()), + ) + + c.SetStoreLimit(storeID, storelimit.AddPeer, 60) + c.SetStoreLimit(storeID, storelimit.RemovePeer, 60) + c.Lock() + defer c.Unlock() + return c.putStoreLocked(newStore) +} + +func (c *testCluster) addLeaderRegion(regionID uint64, leaderStoreID uint64, followerStoreIDs ...uint64) error { + region := newTestRegionMeta(regionID) + leader, _ := c.AllocPeer(leaderStoreID) + region.Peers = []*metapb.Peer{leader} + for _, followerStoreID := range followerStoreIDs { + peer, _ := c.AllocPeer(followerStoreID) + region.Peers = append(region.Peers, peer) + } + regionInfo := core.NewRegionInfo(region, leader, core.SetApproximateSize(10), core.SetApproximateKeys(10)) + return c.putRegion(regionInfo) +} + +func (c *testCluster) updateLeaderCount(storeID uint64, leaderCount int) error { + store := c.GetStore(storeID) + newStore := store.Clone( + core.SetLeaderCount(leaderCount), + core.SetLeaderSize(int64(leaderCount)*10), + ) + c.Lock() + defer c.Unlock() + return c.putStoreLocked(newStore) +} + +func (c *testCluster) addLeaderStore(storeID uint64, leaderCount int) error { + stats := &pdpb.StoreStats{} + newStore := core.NewStoreInfo(&metapb.Store{Id: storeID}, + core.SetStoreStats(stats), + core.SetLeaderCount(leaderCount), + core.SetLeaderSize(int64(leaderCount)*10), + core.SetLastHeartbeatTS(time.Now()), + ) + + c.SetStoreLimit(storeID, storelimit.AddPeer, 60) + c.SetStoreLimit(storeID, storelimit.RemovePeer, 60) + c.Lock() + defer c.Unlock() + return c.putStoreLocked(newStore) +} + +func (c *testCluster) setStoreDown(storeID uint64) error { + store := c.GetStore(storeID) + newStore := store.Clone( + core.UpStore(), + core.SetLastHeartbeatTS(typeutil.ZeroTime), + ) + c.Lock() + defer c.Unlock() + return c.putStoreLocked(newStore) +} + +func (c *testCluster) setStoreOffline(storeID uint64) error { + store := c.GetStore(storeID) + newStore := store.Clone(core.OfflineStore(false)) + c.Lock() + defer c.Unlock() + return c.putStoreLocked(newStore) +} + +func (c *testCluster) LoadRegion(regionID uint64, followerStoreIDs ...uint64) error { + // regions load from etcd will have no leader + region := newTestRegionMeta(regionID) + region.Peers = []*metapb.Peer{} + for _, id := range followerStoreIDs { + peer, _ := c.AllocPeer(id) + region.Peers = append(region.Peers, peer) + } + return c.putRegion(core.NewRegionInfo(region, nil)) +} + +func TestBasic(t *testing.T) { + re := require.New(t) + + tc, co, cleanup := prepare(nil, nil, nil, re) + defer cleanup() + oc := co.GetOperatorController() + + re.NoError(tc.addLeaderRegion(1, 1)) + + op1 := newTestOperator(1, tc.GetRegion(1).GetRegionEpoch(), operator.OpLeader) + oc.AddWaitingOperator(op1) + re.Equal(uint64(1), oc.OperatorCount(operator.OpLeader)) + re.Equal(op1.RegionID(), oc.GetOperator(1).RegionID()) + + // Region 1 already has an operator, cannot add another one. + op2 := newTestOperator(1, tc.GetRegion(1).GetRegionEpoch(), operator.OpRegion) + oc.AddWaitingOperator(op2) + re.Equal(uint64(0), oc.OperatorCount(operator.OpRegion)) + + // Remove the operator manually, then we can add a new operator. + re.True(oc.RemoveOperator(op1)) + op3 := newTestOperator(1, tc.GetRegion(1).GetRegionEpoch(), operator.OpRegion) + oc.AddWaitingOperator(op3) + re.Equal(uint64(1), oc.OperatorCount(operator.OpRegion)) + re.Equal(op3.RegionID(), oc.GetOperator(1).RegionID()) +} + +func TestDispatch(t *testing.T) { + re := require.New(t) + + tc, co, cleanup := prepare(nil, nil, nil, re) + defer cleanup() + co.GetPrepareChecker().SetPrepared() + // Transfer peer from store 4 to store 1. + re.NoError(tc.addRegionStore(4, 40)) + re.NoError(tc.addRegionStore(3, 30)) + re.NoError(tc.addRegionStore(2, 20)) + re.NoError(tc.addRegionStore(1, 10)) + re.NoError(tc.addLeaderRegion(1, 2, 3, 4)) + + // Transfer leader from store 4 to store 2. + re.NoError(tc.updateLeaderCount(4, 50)) + re.NoError(tc.updateLeaderCount(3, 50)) + re.NoError(tc.updateLeaderCount(2, 20)) + re.NoError(tc.updateLeaderCount(1, 10)) + re.NoError(tc.addLeaderRegion(2, 4, 3, 2)) + + go co.RunUntilStop() + + // Wait for schedule and turn off balance. + waitOperator(re, co, 1) + controller := co.GetSchedulersController() + operatorutil.CheckTransferPeer(re, co.GetOperatorController().GetOperator(1), operator.OpKind(0), 4, 1) + re.NoError(controller.RemoveScheduler(schedulers.BalanceRegionName)) + waitOperator(re, co, 2) + operatorutil.CheckTransferLeader(re, co.GetOperatorController().GetOperator(2), operator.OpKind(0), 4, 2) + re.NoError(controller.RemoveScheduler(schedulers.BalanceLeaderName)) + + stream := mockhbstream.NewHeartbeatStream() + + // Transfer peer. + region := tc.GetRegion(1).Clone() + re.NoError(dispatchHeartbeat(co, region, stream)) + region = waitAddLearner(re, stream, region, 1) + re.NoError(dispatchHeartbeat(co, region, stream)) + region = waitPromoteLearner(re, stream, region, 1) + re.NoError(dispatchHeartbeat(co, region, stream)) + region = waitRemovePeer(re, stream, region, 4) + re.NoError(dispatchHeartbeat(co, region, stream)) + re.NoError(dispatchHeartbeat(co, region, stream)) + waitNoResponse(re, stream) + + // Transfer leader. + region = tc.GetRegion(2).Clone() + re.NoError(dispatchHeartbeat(co, region, stream)) + waitTransferLeader(re, stream, region, 2) + re.NoError(dispatchHeartbeat(co, region, stream)) + waitNoResponse(re, stream) +} + +func dispatchHeartbeat(co *schedule.Coordinator, region *core.RegionInfo, stream hbstream.HeartbeatStream) error { + co.GetHeartbeatStreams().BindStream(region.GetLeader().GetStoreId(), stream) + if err := co.GetCluster().(*RaftCluster).putRegion(region.Clone()); err != nil { + return err + } + co.GetOperatorController().Dispatch(region, operator.DispatchFromHeartBeat, nil) + return nil +} + +func TestCollectMetricsConcurrent(t *testing.T) { + re := require.New(t) + + tc, co, cleanup := prepare(nil, func(tc *testCluster) { + tc.regionStats = statistics.NewRegionStatistics( + tc.GetBasicCluster(), + tc.GetOpts(), + nil, + tc.storeConfigManager) + }, func(co *schedule.Coordinator) { co.Run() }, re) + defer cleanup() + + // Make sure there are no problem when concurrent write and read + var wg sync.WaitGroup + count := 10 + wg.Add(count + 1) + for i := 0; i <= count; i++ { + go func(i int) { + defer wg.Done() + for j := 0; j < 1000; j++ { + re.NoError(tc.addRegionStore(uint64(i%5), rand.Intn(200))) + } + }(i) + } + controller := co.GetSchedulersController() + for i := 0; i < 1000; i++ { + co.CollectHotSpotMetrics() + controller.CollectSchedulerMetrics() + co.GetCluster().(*RaftCluster).collectClusterMetrics() + } + co.ResetHotSpotMetrics() + controller.ResetSchedulerMetrics() + co.GetCluster().(*RaftCluster).resetClusterMetrics() + wg.Wait() +} + +func TestCollectMetrics(t *testing.T) { + re := require.New(t) + + tc, co, cleanup := prepare(nil, func(tc *testCluster) { + tc.regionStats = statistics.NewRegionStatistics( + tc.GetBasicCluster(), + tc.GetOpts(), + nil, + tc.storeConfigManager) + }, func(co *schedule.Coordinator) { co.Run() }, re) + defer cleanup() + count := 10 + for i := 0; i <= count; i++ { + for k := 0; k < 200; k++ { + item := &statistics.HotPeerStat{ + StoreID: uint64(i % 5), + RegionID: uint64(i*1000 + k), + Loads: []float64{10, 20, 30}, + HotDegree: 10, + AntiCount: statistics.HotRegionAntiCount, // for write + } + tc.hotStat.HotCache.Update(item, statistics.Write) + } + } + controller := co.GetSchedulersController() + for i := 0; i < 1000; i++ { + co.CollectHotSpotMetrics() + controller.CollectSchedulerMetrics() + co.GetCluster().(*RaftCluster).collectClusterMetrics() + } + stores := co.GetCluster().GetStores() + regionStats := co.GetCluster().RegionWriteStats() + status1 := statistics.CollectHotPeerInfos(stores, regionStats) + status2 := statistics.GetHotStatus(stores, co.GetCluster().GetStoresLoads(), regionStats, statistics.Write, co.GetCluster().GetSchedulerConfig().IsTraceRegionFlow()) + for _, s := range status2.AsLeader { + s.Stats = nil + } + for _, s := range status2.AsPeer { + s.Stats = nil + } + re.Equal(status1, status2) + co.ResetHotSpotMetrics() + controller.ResetSchedulerMetrics() + co.GetCluster().(*RaftCluster).resetClusterMetrics() +} + +func prepare(setCfg func(*sc.ScheduleConfig), setTc func(*testCluster), run func(*schedule.Coordinator), re *require.Assertions) (*testCluster, *schedule.Coordinator, func()) { + ctx, cancel := context.WithCancel(context.Background()) + cfg, opt, err := newTestScheduleConfig() + re.NoError(err) + if setCfg != nil { + setCfg(cfg) + } + tc := newTestCluster(ctx, opt) + hbStreams := hbstream.NewTestHeartbeatStreams(ctx, tc.meta.GetId(), tc, true /* need to run */) + if setTc != nil { + setTc(tc) + } + co := schedule.NewCoordinator(ctx, tc.RaftCluster, hbStreams) + if run != nil { + run(co) + } + return tc, co, func() { + co.Stop() + co.GetSchedulersController().Wait() + co.GetWaitGroup().Wait() + hbStreams.Close() + cancel() + } +} + +func checkRegionAndOperator(re *require.Assertions, tc *testCluster, co *schedule.Coordinator, regionID uint64, expectAddOperator int) { + ops := co.GetCheckerController().CheckRegion(tc.GetRegion(regionID)) + if ops == nil { + re.Equal(0, expectAddOperator) + } else { + re.Equal(expectAddOperator, co.GetOperatorController().AddWaitingOperator(ops...)) + } +} + +func TestCheckRegion(t *testing.T) { + re := require.New(t) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + tc, co, cleanup := prepare(nil, nil, nil, re) + hbStreams, opt := co.GetHeartbeatStreams(), tc.opt + defer cleanup() + + re.NoError(tc.addRegionStore(4, 4)) + re.NoError(tc.addRegionStore(3, 3)) + re.NoError(tc.addRegionStore(2, 2)) + re.NoError(tc.addRegionStore(1, 1)) + re.NoError(tc.addLeaderRegion(1, 2, 3)) + checkRegionAndOperator(re, tc, co, 1, 1) + operatorutil.CheckAddPeer(re, co.GetOperatorController().GetOperator(1), operator.OpReplica, 1) + checkRegionAndOperator(re, tc, co, 1, 0) + + r := tc.GetRegion(1) + p := &metapb.Peer{Id: 1, StoreId: 1, Role: metapb.PeerRole_Learner} + r = r.Clone( + core.WithAddPeer(p), + core.WithPendingPeers(append(r.GetPendingPeers(), p)), + ) + re.NoError(tc.putRegion(r)) + checkRegionAndOperator(re, tc, co, 1, 0) + + tc = newTestCluster(ctx, opt) + co = schedule.NewCoordinator(ctx, tc.RaftCluster, hbStreams) + + re.NoError(tc.addRegionStore(4, 4)) + re.NoError(tc.addRegionStore(3, 3)) + re.NoError(tc.addRegionStore(2, 2)) + re.NoError(tc.addRegionStore(1, 1)) + re.NoError(tc.putRegion(r)) + checkRegionAndOperator(re, tc, co, 1, 0) + r = r.Clone(core.WithPendingPeers(nil)) + re.NoError(tc.putRegion(r)) + checkRegionAndOperator(re, tc, co, 1, 1) + op := co.GetOperatorController().GetOperator(1) + re.Equal(1, op.Len()) + re.Equal(uint64(1), op.Step(0).(operator.PromoteLearner).ToStore) + checkRegionAndOperator(re, tc, co, 1, 0) +} + +func TestCheckRegionWithScheduleDeny(t *testing.T) { + re := require.New(t) + + tc, co, cleanup := prepare(nil, nil, nil, re) + defer cleanup() + + re.NoError(tc.addRegionStore(4, 4)) + re.NoError(tc.addRegionStore(3, 3)) + re.NoError(tc.addRegionStore(2, 2)) + re.NoError(tc.addRegionStore(1, 1)) + re.NoError(tc.addLeaderRegion(1, 2, 3)) + region := tc.GetRegion(1) + re.NotNil(region) + // test with label schedule=deny + labelerManager := tc.GetRegionLabeler() + labelerManager.SetLabelRule(&labeler.LabelRule{ + ID: "schedulelabel", + Labels: []labeler.RegionLabel{{Key: "schedule", Value: "deny"}}, + RuleType: labeler.KeyRange, + Data: []interface{}{map[string]interface{}{"start_key": "", "end_key": ""}}, + }) + + // should allow to do rule checker + re.True(labelerManager.ScheduleDisabled(region)) + checkRegionAndOperator(re, tc, co, 1, 1) + + // should not allow to merge + tc.opt.SetSplitMergeInterval(time.Duration(0)) + re.NoError(tc.addLeaderRegion(2, 2, 3, 4)) + re.NoError(tc.addLeaderRegion(3, 2, 3, 4)) + region = tc.GetRegion(2) + re.True(labelerManager.ScheduleDisabled(region)) + checkRegionAndOperator(re, tc, co, 2, 0) + // delete label rule, should allow to do merge + labelerManager.DeleteLabelRule("schedulelabel") + re.False(labelerManager.ScheduleDisabled(region)) + checkRegionAndOperator(re, tc, co, 2, 2) +} + +func TestCheckerIsBusy(t *testing.T) { + re := require.New(t) + + tc, co, cleanup := prepare(func(cfg *sc.ScheduleConfig) { + cfg.ReplicaScheduleLimit = 0 // ensure replica checker is busy + cfg.MergeScheduleLimit = 10 + }, nil, nil, re) + defer cleanup() + + re.NoError(tc.addRegionStore(1, 0)) + num := 1 + typeutil.MaxUint64(tc.opt.GetReplicaScheduleLimit(), tc.opt.GetMergeScheduleLimit()) + var operatorKinds = []operator.OpKind{ + operator.OpReplica, operator.OpRegion | operator.OpMerge, + } + for i, operatorKind := range operatorKinds { + for j := uint64(0); j < num; j++ { + regionID := j + uint64(i+1)*num + re.NoError(tc.addLeaderRegion(regionID, 1)) + switch operatorKind { + case operator.OpReplica: + op := newTestOperator(regionID, tc.GetRegion(regionID).GetRegionEpoch(), operatorKind) + re.Equal(1, co.GetOperatorController().AddWaitingOperator(op)) + case operator.OpRegion | operator.OpMerge: + if regionID%2 == 1 { + ops, err := operator.CreateMergeRegionOperator("merge-region", co.GetCluster(), tc.GetRegion(regionID), tc.GetRegion(regionID-1), operator.OpMerge) + re.NoError(err) + re.Len(ops, co.GetOperatorController().AddWaitingOperator(ops...)) + } + } + } + } + checkRegionAndOperator(re, tc, co, num, 0) +} + +func TestMergeRegionCancelOneOperator(t *testing.T) { + re := require.New(t) + tc, co, cleanup := prepare(nil, nil, nil, re) + defer cleanup() + + source := core.NewRegionInfo( + &metapb.Region{ + Id: 1, + StartKey: []byte(""), + EndKey: []byte("a"), + }, + nil, + ) + target := core.NewRegionInfo( + &metapb.Region{ + Id: 2, + StartKey: []byte("a"), + EndKey: []byte("t"), + }, + nil, + ) + re.NoError(tc.putRegion(source)) + re.NoError(tc.putRegion(target)) + + // Cancel source region. + ops, err := operator.CreateMergeRegionOperator("merge-region", tc, source, target, operator.OpMerge) + re.NoError(err) + re.Len(ops, co.GetOperatorController().AddWaitingOperator(ops...)) + // Cancel source operator. + co.GetOperatorController().RemoveOperator(co.GetOperatorController().GetOperator(source.GetID())) + re.Len(co.GetOperatorController().GetOperators(), 0) + + // Cancel target region. + ops, err = operator.CreateMergeRegionOperator("merge-region", tc, source, target, operator.OpMerge) + re.NoError(err) + re.Len(ops, co.GetOperatorController().AddWaitingOperator(ops...)) + // Cancel target operator. + co.GetOperatorController().RemoveOperator(co.GetOperatorController().GetOperator(target.GetID())) + re.Len(co.GetOperatorController().GetOperators(), 0) +} + +func TestReplica(t *testing.T) { + re := require.New(t) + + tc, co, cleanup := prepare(func(cfg *sc.ScheduleConfig) { + // Turn off balance. + cfg.LeaderScheduleLimit = 0 + cfg.RegionScheduleLimit = 0 + }, nil, func(co *schedule.Coordinator) { co.Run() }, re) + defer cleanup() + + re.NoError(tc.addRegionStore(1, 1)) + re.NoError(tc.addRegionStore(2, 2)) + re.NoError(tc.addRegionStore(3, 3)) + re.NoError(tc.addRegionStore(4, 4)) + + stream := mockhbstream.NewHeartbeatStream() + + // Add peer to store 1. + re.NoError(tc.addLeaderRegion(1, 2, 3)) + region := tc.GetRegion(1) + re.NoError(dispatchHeartbeat(co, region, stream)) + region = waitAddLearner(re, stream, region, 1) + re.NoError(dispatchHeartbeat(co, region, stream)) + region = waitPromoteLearner(re, stream, region, 1) + re.NoError(dispatchHeartbeat(co, region, stream)) + waitNoResponse(re, stream) + + // Peer in store 3 is down, remove peer in store 3 and add peer to store 4. + re.NoError(tc.setStoreDown(3)) + downPeer := &pdpb.PeerStats{ + Peer: region.GetStorePeer(3), + DownSeconds: 24 * 60 * 60, + } + region = region.Clone( + core.WithDownPeers(append(region.GetDownPeers(), downPeer)), + ) + re.NoError(dispatchHeartbeat(co, region, stream)) + region = waitAddLearner(re, stream, region, 4) + re.NoError(dispatchHeartbeat(co, region, stream)) + region = waitPromoteLearner(re, stream, region, 4) + region = region.Clone(core.WithDownPeers(nil)) + re.NoError(dispatchHeartbeat(co, region, stream)) + waitNoResponse(re, stream) + + // Remove peer from store 4. + re.NoError(tc.addLeaderRegion(2, 1, 2, 3, 4)) + region = tc.GetRegion(2) + re.NoError(dispatchHeartbeat(co, region, stream)) + region = waitRemovePeer(re, stream, region, 4) + re.NoError(dispatchHeartbeat(co, region, stream)) + waitNoResponse(re, stream) + + // Remove offline peer directly when it's pending. + re.NoError(tc.addLeaderRegion(3, 1, 2, 3)) + re.NoError(tc.setStoreOffline(3)) + region = tc.GetRegion(3) + region = region.Clone(core.WithPendingPeers([]*metapb.Peer{region.GetStorePeer(3)})) + re.NoError(dispatchHeartbeat(co, region, stream)) + waitNoResponse(re, stream) +} + +func TestCheckCache(t *testing.T) { + re := require.New(t) + + tc, co, cleanup := prepare(func(cfg *sc.ScheduleConfig) { + // Turn off replica scheduling. + cfg.ReplicaScheduleLimit = 0 + }, nil, nil, re) + defer cleanup() + + re.NoError(tc.addRegionStore(1, 0)) + re.NoError(tc.addRegionStore(2, 0)) + re.NoError(tc.addRegionStore(3, 0)) + + // Add a peer with two replicas. + re.NoError(tc.addLeaderRegion(1, 2, 3)) + re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/schedule/break-patrol", `return`)) + + // case 1: operator cannot be created due to replica-schedule-limit restriction + co.GetWaitGroup().Add(1) + co.PatrolRegions() + re.Len(co.GetCheckerController().GetWaitingRegions(), 1) + + // cancel the replica-schedule-limit restriction + cfg := tc.GetScheduleConfig() + cfg.ReplicaScheduleLimit = 10 + tc.SetScheduleConfig(cfg) + co.GetWaitGroup().Add(1) + co.PatrolRegions() + oc := co.GetOperatorController() + re.Len(oc.GetOperators(), 1) + re.Empty(co.GetCheckerController().GetWaitingRegions()) + + // case 2: operator cannot be created due to store limit restriction + oc.RemoveOperator(oc.GetOperator(1)) + tc.SetStoreLimit(1, storelimit.AddPeer, 0) + co.GetWaitGroup().Add(1) + co.PatrolRegions() + re.Len(co.GetCheckerController().GetWaitingRegions(), 1) + + // cancel the store limit restriction + tc.SetStoreLimit(1, storelimit.AddPeer, 10) + time.Sleep(time.Second) + co.GetWaitGroup().Add(1) + co.PatrolRegions() + re.Len(oc.GetOperators(), 1) + re.Empty(co.GetCheckerController().GetWaitingRegions()) + + co.GetSchedulersController().Wait() + co.GetWaitGroup().Wait() + re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/schedule/break-patrol")) +} + +func TestPeerState(t *testing.T) { + re := require.New(t) + + tc, co, cleanup := prepare(nil, nil, func(co *schedule.Coordinator) { co.Run() }, re) + defer cleanup() + + // Transfer peer from store 4 to store 1. + re.NoError(tc.addRegionStore(1, 10)) + re.NoError(tc.addRegionStore(2, 10)) + re.NoError(tc.addRegionStore(3, 10)) + re.NoError(tc.addRegionStore(4, 40)) + re.NoError(tc.addLeaderRegion(1, 2, 3, 4)) + + stream := mockhbstream.NewHeartbeatStream() + + // Wait for schedule. + waitOperator(re, co, 1) + operatorutil.CheckTransferPeer(re, co.GetOperatorController().GetOperator(1), operator.OpKind(0), 4, 1) + + region := tc.GetRegion(1).Clone() + + // Add new peer. + re.NoError(dispatchHeartbeat(co, region, stream)) + region = waitAddLearner(re, stream, region, 1) + re.NoError(dispatchHeartbeat(co, region, stream)) + region = waitPromoteLearner(re, stream, region, 1) + + // If the new peer is pending, the operator will not finish. + region = region.Clone(core.WithPendingPeers(append(region.GetPendingPeers(), region.GetStorePeer(1)))) + re.NoError(dispatchHeartbeat(co, region, stream)) + waitNoResponse(re, stream) + re.NotNil(co.GetOperatorController().GetOperator(region.GetID())) + + // The new peer is not pending now, the operator will finish. + // And we will proceed to remove peer in store 4. + region = region.Clone(core.WithPendingPeers(nil)) + re.NoError(dispatchHeartbeat(co, region, stream)) + waitRemovePeer(re, stream, region, 4) + re.NoError(tc.addLeaderRegion(1, 1, 2, 3)) + region = tc.GetRegion(1).Clone() + re.NoError(dispatchHeartbeat(co, region, stream)) + waitNoResponse(re, stream) +} + +func TestShouldRun(t *testing.T) { + re := require.New(t) + + tc, co, cleanup := prepare(nil, nil, nil, re) + tc.RaftCluster.coordinator = co + defer cleanup() + + re.NoError(tc.addLeaderStore(1, 5)) + re.NoError(tc.addLeaderStore(2, 2)) + re.NoError(tc.addLeaderStore(3, 0)) + re.NoError(tc.addLeaderStore(4, 0)) + re.NoError(tc.LoadRegion(1, 1, 2, 3)) + re.NoError(tc.LoadRegion(2, 1, 2, 3)) + re.NoError(tc.LoadRegion(3, 1, 2, 3)) + re.NoError(tc.LoadRegion(4, 1, 2, 3)) + re.NoError(tc.LoadRegion(5, 1, 2, 3)) + re.NoError(tc.LoadRegion(6, 2, 1, 4)) + re.NoError(tc.LoadRegion(7, 2, 1, 4)) + re.False(co.ShouldRun()) + re.Equal(2, tc.GetStoreRegionCount(4)) + + testCases := []struct { + regionID uint64 + ShouldRun bool + }{ + {1, false}, + {2, false}, + {3, false}, + {4, false}, + {5, false}, + // store4 needs Collect two region + {6, false}, + {7, true}, + } + + for _, testCase := range testCases { + r := tc.GetRegion(testCase.regionID) + nr := r.Clone(core.WithLeader(r.GetPeers()[0])) + re.NoError(tc.processRegionHeartbeat(nr)) + re.Equal(testCase.ShouldRun, co.ShouldRun()) + } + nr := &metapb.Region{Id: 6, Peers: []*metapb.Peer{}} + newRegion := core.NewRegionInfo(nr, nil) + re.Error(tc.processRegionHeartbeat(newRegion)) + re.Equal(7, co.GetPrepareChecker().GetSum()) +} + +func TestShouldRunWithNonLeaderRegions(t *testing.T) { + re := require.New(t) + + tc, co, cleanup := prepare(nil, nil, nil, re) + tc.RaftCluster.coordinator = co + defer cleanup() + + re.NoError(tc.addLeaderStore(1, 10)) + re.NoError(tc.addLeaderStore(2, 0)) + re.NoError(tc.addLeaderStore(3, 0)) + for i := 0; i < 10; i++ { + re.NoError(tc.LoadRegion(uint64(i+1), 1, 2, 3)) + } + re.False(co.ShouldRun()) + re.Equal(10, tc.GetStoreRegionCount(1)) + + testCases := []struct { + regionID uint64 + ShouldRun bool + }{ + {1, false}, + {2, false}, + {3, false}, + {4, false}, + {5, false}, + {6, false}, + {7, false}, + {8, false}, + {9, true}, + } + + for _, testCase := range testCases { + r := tc.GetRegion(testCase.regionID) + nr := r.Clone(core.WithLeader(r.GetPeers()[0])) + re.NoError(tc.processRegionHeartbeat(nr)) + re.Equal(testCase.ShouldRun, co.ShouldRun()) + } + nr := &metapb.Region{Id: 9, Peers: []*metapb.Peer{}} + newRegion := core.NewRegionInfo(nr, nil) + re.Error(tc.processRegionHeartbeat(newRegion)) + re.Equal(9, co.GetPrepareChecker().GetSum()) + + // Now, after server is prepared, there exist some regions with no leader. + re.Equal(uint64(0), tc.GetRegion(10).GetLeader().GetStoreId()) +} + +func TestAddScheduler(t *testing.T) { + re := require.New(t) + + tc, co, cleanup := prepare(nil, nil, func(co *schedule.Coordinator) { co.Run() }, re) + defer cleanup() + controller := co.GetSchedulersController() + re.Len(controller.GetSchedulerNames(), len(sc.DefaultSchedulers)) + re.NoError(controller.RemoveScheduler(schedulers.BalanceLeaderName)) + re.NoError(controller.RemoveScheduler(schedulers.BalanceRegionName)) + re.NoError(controller.RemoveScheduler(schedulers.HotRegionName)) + re.NoError(controller.RemoveScheduler(schedulers.BalanceWitnessName)) + re.NoError(controller.RemoveScheduler(schedulers.TransferWitnessLeaderName)) + re.Empty(controller.GetSchedulerNames()) + + stream := mockhbstream.NewHeartbeatStream() + + // Add stores 1,2,3 + re.NoError(tc.addLeaderStore(1, 1)) + re.NoError(tc.addLeaderStore(2, 1)) + re.NoError(tc.addLeaderStore(3, 1)) + // Add regions 1 with leader in store 1 and followers in stores 2,3 + re.NoError(tc.addLeaderRegion(1, 1, 2, 3)) + // Add regions 2 with leader in store 2 and followers in stores 1,3 + re.NoError(tc.addLeaderRegion(2, 2, 1, 3)) + // Add regions 3 with leader in store 3 and followers in stores 1,2 + re.NoError(tc.addLeaderRegion(3, 3, 1, 2)) + + oc := co.GetOperatorController() + + // test ConfigJSONDecoder create + bl, err := schedulers.CreateScheduler(schedulers.BalanceLeaderType, oc, storage.NewStorageWithMemoryBackend(), schedulers.ConfigJSONDecoder([]byte("{}"))) + re.NoError(err) + conf, err := bl.EncodeConfig() + re.NoError(err) + data := make(map[string]interface{}) + err = json.Unmarshal(conf, &data) + re.NoError(err) + batch := data["batch"].(float64) + re.Equal(4, int(batch)) + gls, err := schedulers.CreateScheduler(schedulers.GrantLeaderType, oc, storage.NewStorageWithMemoryBackend(), schedulers.ConfigSliceDecoder(schedulers.GrantLeaderType, []string{"0"}), controller.RemoveScheduler) + re.NoError(err) + re.NotNil(controller.AddScheduler(gls)) + re.NotNil(controller.RemoveScheduler(gls.GetName())) + + gls, err = schedulers.CreateScheduler(schedulers.GrantLeaderType, oc, storage.NewStorageWithMemoryBackend(), schedulers.ConfigSliceDecoder(schedulers.GrantLeaderType, []string{"1"}), controller.RemoveScheduler) + re.NoError(err) + re.NoError(controller.AddScheduler(gls)) + + hb, err := schedulers.CreateScheduler(schedulers.HotRegionType, oc, storage.NewStorageWithMemoryBackend(), schedulers.ConfigJSONDecoder([]byte("{}"))) + re.NoError(err) + conf, err = hb.EncodeConfig() + re.NoError(err) + data = make(map[string]interface{}) + re.NoError(json.Unmarshal(conf, &data)) + re.Contains(data, "enable-for-tiflash") + re.Equal("true", data["enable-for-tiflash"].(string)) + + // Transfer all leaders to store 1. + waitOperator(re, co, 2) + region2 := tc.GetRegion(2) + re.NoError(dispatchHeartbeat(co, region2, stream)) + region2 = waitTransferLeader(re, stream, region2, 1) + re.NoError(dispatchHeartbeat(co, region2, stream)) + waitNoResponse(re, stream) + + waitOperator(re, co, 3) + region3 := tc.GetRegion(3) + re.NoError(dispatchHeartbeat(co, region3, stream)) + region3 = waitTransferLeader(re, stream, region3, 1) + re.NoError(dispatchHeartbeat(co, region3, stream)) + waitNoResponse(re, stream) +} + +func TestPersistScheduler(t *testing.T) { + re := require.New(t) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + tc, co, cleanup := prepare(nil, nil, func(co *schedule.Coordinator) { co.Run() }, re) + hbStreams := co.GetHeartbeatStreams() + defer cleanup() + defaultCount := len(sc.DefaultSchedulers) + // Add stores 1,2 + re.NoError(tc.addLeaderStore(1, 1)) + re.NoError(tc.addLeaderStore(2, 1)) + + controller := co.GetSchedulersController() + re.Len(controller.GetSchedulerNames(), defaultCount) + oc := co.GetOperatorController() + storage := tc.RaftCluster.storage + + gls1, err := schedulers.CreateScheduler(schedulers.GrantLeaderType, oc, storage, schedulers.ConfigSliceDecoder(schedulers.GrantLeaderType, []string{"1"}), controller.RemoveScheduler) + re.NoError(err) + re.NoError(controller.AddScheduler(gls1, "1")) + evict, err := schedulers.CreateScheduler(schedulers.EvictLeaderType, oc, storage, schedulers.ConfigSliceDecoder(schedulers.EvictLeaderType, []string{"2"}), controller.RemoveScheduler) + re.NoError(err) + re.NoError(controller.AddScheduler(evict, "2")) + re.Len(controller.GetSchedulerNames(), defaultCount+2) + sches, _, err := storage.LoadAllScheduleConfig() + re.NoError(err) + re.Len(sches, defaultCount+2) + + // remove 5 schedulers + re.NoError(controller.RemoveScheduler(schedulers.BalanceLeaderName)) + re.NoError(controller.RemoveScheduler(schedulers.BalanceRegionName)) + re.NoError(controller.RemoveScheduler(schedulers.HotRegionName)) + re.NoError(controller.RemoveScheduler(schedulers.BalanceWitnessName)) + re.NoError(controller.RemoveScheduler(schedulers.TransferWitnessLeaderName)) + re.Len(controller.GetSchedulerNames(), defaultCount-3) + re.NoError(co.GetCluster().GetPersistOptions().Persist(storage)) + co.Stop() + co.GetSchedulersController().Wait() + co.GetWaitGroup().Wait() + // make a new coordinator for testing + // whether the schedulers added or removed in dynamic way are recorded in opt + _, newOpt, err := newTestScheduleConfig() + re.NoError(err) + _, err = schedulers.CreateScheduler(schedulers.ShuffleRegionType, oc, storage, schedulers.ConfigJSONDecoder([]byte("null"))) + re.NoError(err) + // suppose we add a new default enable scheduler + sc.DefaultSchedulers = append(sc.DefaultSchedulers, sc.SchedulerConfig{Type: "shuffle-region"}) + defer func() { + sc.DefaultSchedulers = sc.DefaultSchedulers[:len(sc.DefaultSchedulers)-1] + }() + re.Len(newOpt.GetSchedulers(), defaultCount) + re.NoError(newOpt.Reload(storage)) + // only remains 3 items with independent config. + sches, _, err = storage.LoadAllScheduleConfig() + re.NoError(err) + re.Len(sches, 3) + + // option have 6 items because the default scheduler do not remove. + re.Len(newOpt.GetSchedulers(), defaultCount+3) + re.NoError(newOpt.Persist(storage)) + tc.RaftCluster.opt = newOpt + + co = schedule.NewCoordinator(ctx, tc.RaftCluster, hbStreams) + co.Run() + controller = co.GetSchedulersController() + re.Len(controller.GetSchedulerNames(), 3) + co.Stop() + co.GetSchedulersController().Wait() + co.GetWaitGroup().Wait() + // suppose restart PD again + _, newOpt, err = newTestScheduleConfig() + re.NoError(err) + re.NoError(newOpt.Reload(storage)) + tc.RaftCluster.opt = newOpt + co = schedule.NewCoordinator(ctx, tc.RaftCluster, hbStreams) + co.Run() + controller = co.GetSchedulersController() + re.Len(controller.GetSchedulerNames(), 3) + bls, err := schedulers.CreateScheduler(schedulers.BalanceLeaderType, oc, storage, schedulers.ConfigSliceDecoder(schedulers.BalanceLeaderType, []string{"", ""})) + re.NoError(err) + re.NoError(controller.AddScheduler(bls)) + brs, err := schedulers.CreateScheduler(schedulers.BalanceRegionType, oc, storage, schedulers.ConfigSliceDecoder(schedulers.BalanceRegionType, []string{"", ""})) + re.NoError(err) + re.NoError(controller.AddScheduler(brs)) + re.Len(controller.GetSchedulerNames(), defaultCount) + + // the scheduler option should contain 6 items + // the `hot scheduler` are disabled + re.Len(co.GetCluster().GetPersistOptions().GetSchedulers(), defaultCount+3) + re.NoError(controller.RemoveScheduler(schedulers.GrantLeaderName)) + // the scheduler that is not enable by default will be completely deleted + re.Len(co.GetCluster().GetPersistOptions().GetSchedulers(), defaultCount+2) + re.Len(controller.GetSchedulerNames(), 4) + re.NoError(co.GetCluster().GetPersistOptions().Persist(co.GetCluster().GetStorage())) + co.Stop() + co.GetSchedulersController().Wait() + co.GetWaitGroup().Wait() + _, newOpt, err = newTestScheduleConfig() + re.NoError(err) + re.NoError(newOpt.Reload(co.GetCluster().GetStorage())) + tc.RaftCluster.opt = newOpt + co = schedule.NewCoordinator(ctx, tc.RaftCluster, hbStreams) + + co.Run() + controller = co.GetSchedulersController() + re.Len(controller.GetSchedulerNames(), defaultCount-1) + re.NoError(controller.RemoveScheduler(schedulers.EvictLeaderName)) + re.Len(controller.GetSchedulerNames(), defaultCount-2) +} + +func TestRemoveScheduler(t *testing.T) { + re := require.New(t) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + tc, co, cleanup := prepare(func(cfg *sc.ScheduleConfig) { + cfg.ReplicaScheduleLimit = 0 + }, nil, func(co *schedule.Coordinator) { co.Run() }, re) + hbStreams := co.GetHeartbeatStreams() + defer cleanup() + + // Add stores 1,2 + re.NoError(tc.addLeaderStore(1, 1)) + re.NoError(tc.addLeaderStore(2, 1)) + defaultCount := len(sc.DefaultSchedulers) + controller := co.GetSchedulersController() + re.Len(controller.GetSchedulerNames(), defaultCount) + oc := co.GetOperatorController() + storage := tc.RaftCluster.storage + + gls1, err := schedulers.CreateScheduler(schedulers.GrantLeaderType, oc, storage, schedulers.ConfigSliceDecoder(schedulers.GrantLeaderType, []string{"1"}), controller.RemoveScheduler) + re.NoError(err) + re.NoError(controller.AddScheduler(gls1, "1")) + re.Len(controller.GetSchedulerNames(), defaultCount+1) + sches, _, err := storage.LoadAllScheduleConfig() + re.NoError(err) + re.Len(sches, defaultCount+1) + + // remove all schedulers + re.NoError(controller.RemoveScheduler(schedulers.BalanceLeaderName)) + re.NoError(controller.RemoveScheduler(schedulers.BalanceRegionName)) + re.NoError(controller.RemoveScheduler(schedulers.HotRegionName)) + re.NoError(controller.RemoveScheduler(schedulers.GrantLeaderName)) + re.NoError(controller.RemoveScheduler(schedulers.BalanceWitnessName)) + re.NoError(controller.RemoveScheduler(schedulers.TransferWitnessLeaderName)) + // all removed + sches, _, err = storage.LoadAllScheduleConfig() + re.NoError(err) + re.Empty(sches) + re.Empty(controller.GetSchedulerNames()) + re.NoError(co.GetCluster().GetPersistOptions().Persist(co.GetCluster().GetStorage())) + co.Stop() + co.GetSchedulersController().Wait() + co.GetWaitGroup().Wait() + + // suppose restart PD again + _, newOpt, err := newTestScheduleConfig() + re.NoError(err) + re.NoError(newOpt.Reload(tc.storage)) + tc.RaftCluster.opt = newOpt + co = schedule.NewCoordinator(ctx, tc.RaftCluster, hbStreams) + co.Run() + re.Empty(controller.GetSchedulerNames()) + // the option remains default scheduler + re.Len(co.GetCluster().GetPersistOptions().GetSchedulers(), defaultCount) + co.Stop() + co.GetSchedulersController().Wait() + co.GetWaitGroup().Wait() +} + +func TestRestart(t *testing.T) { + re := require.New(t) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + tc, co, cleanup := prepare(func(cfg *sc.ScheduleConfig) { + // Turn off balance, we test add replica only. + cfg.LeaderScheduleLimit = 0 + cfg.RegionScheduleLimit = 0 + }, nil, func(co *schedule.Coordinator) { co.Run() }, re) + hbStreams := co.GetHeartbeatStreams() + defer cleanup() + + // Add 3 stores (1, 2, 3) and a region with 1 replica on store 1. + re.NoError(tc.addRegionStore(1, 1)) + re.NoError(tc.addRegionStore(2, 2)) + re.NoError(tc.addRegionStore(3, 3)) + re.NoError(tc.addLeaderRegion(1, 1)) + region := tc.GetRegion(1) + co.GetPrepareChecker().Collect(region) + + // Add 1 replica on store 2. + stream := mockhbstream.NewHeartbeatStream() + re.NoError(dispatchHeartbeat(co, region, stream)) + region = waitAddLearner(re, stream, region, 2) + re.NoError(dispatchHeartbeat(co, region, stream)) + region = waitPromoteLearner(re, stream, region, 2) + co.Stop() + co.GetSchedulersController().Wait() + co.GetWaitGroup().Wait() + + // Recreate coordinator then add another replica on store 3. + co = schedule.NewCoordinator(ctx, tc.RaftCluster, hbStreams) + co.GetPrepareChecker().Collect(region) + co.Run() + re.NoError(dispatchHeartbeat(co, region, stream)) + region = waitAddLearner(re, stream, region, 3) + re.NoError(dispatchHeartbeat(co, region, stream)) + waitPromoteLearner(re, stream, region, 3) +} + +func TestPauseScheduler(t *testing.T) { + re := require.New(t) + + _, co, cleanup := prepare(nil, nil, func(co *schedule.Coordinator) { co.Run() }, re) + defer cleanup() + controller := co.GetSchedulersController() + _, err := controller.IsSchedulerAllowed("test") + re.Error(err) + controller.PauseOrResumeScheduler(schedulers.BalanceLeaderName, 60) + paused, _ := controller.IsSchedulerPaused(schedulers.BalanceLeaderName) + re.True(paused) + pausedAt, err := controller.GetPausedSchedulerDelayAt(schedulers.BalanceLeaderName) + re.NoError(err) + resumeAt, err := controller.GetPausedSchedulerDelayUntil(schedulers.BalanceLeaderName) + re.NoError(err) + re.Equal(int64(60), resumeAt-pausedAt) + allowed, _ := controller.IsSchedulerAllowed(schedulers.BalanceLeaderName) + re.False(allowed) +} + +func BenchmarkPatrolRegion(b *testing.B) { + re := require.New(b) + + mergeLimit := uint64(4100) + regionNum := 10000 + + tc, co, cleanup := prepare(func(cfg *sc.ScheduleConfig) { + cfg.MergeScheduleLimit = mergeLimit + }, nil, nil, re) + defer cleanup() + + tc.opt.SetSplitMergeInterval(time.Duration(0)) + for i := 1; i < 4; i++ { + if err := tc.addRegionStore(uint64(i), regionNum, 96); err != nil { + return + } + } + for i := 0; i < regionNum; i++ { + if err := tc.addLeaderRegion(uint64(i), 1, 2, 3); err != nil { + return + } + } + + listen := make(chan int) + go func() { + oc := co.GetOperatorController() + listen <- 0 + for { + if oc.OperatorCount(operator.OpMerge) == mergeLimit { + co.Stop() + return + } + } + }() + <-listen + + co.GetWaitGroup().Add(1) + b.ResetTimer() + co.PatrolRegions() +} + +func waitOperator(re *require.Assertions, co *schedule.Coordinator, regionID uint64) { + testutil.Eventually(re, func() bool { + return co.GetOperatorController().GetOperator(regionID) != nil + }) +} + +func TestOperatorCount(t *testing.T) { + re := require.New(t) + + tc, co, cleanup := prepare(nil, nil, nil, re) + defer cleanup() + oc := co.GetOperatorController() + re.Equal(uint64(0), oc.OperatorCount(operator.OpLeader)) + re.Equal(uint64(0), oc.OperatorCount(operator.OpRegion)) + + re.NoError(tc.addLeaderRegion(1, 1)) + re.NoError(tc.addLeaderRegion(2, 2)) + { + op1 := newTestOperator(1, tc.GetRegion(1).GetRegionEpoch(), operator.OpLeader) + oc.AddWaitingOperator(op1) + re.Equal(uint64(1), oc.OperatorCount(operator.OpLeader)) // 1:leader + op2 := newTestOperator(2, tc.GetRegion(2).GetRegionEpoch(), operator.OpLeader) + oc.AddWaitingOperator(op2) + re.Equal(uint64(2), oc.OperatorCount(operator.OpLeader)) // 1:leader, 2:leader + re.True(oc.RemoveOperator(op1)) + re.Equal(uint64(1), oc.OperatorCount(operator.OpLeader)) // 2:leader + } + + { + op1 := newTestOperator(1, tc.GetRegion(1).GetRegionEpoch(), operator.OpRegion) + oc.AddWaitingOperator(op1) + re.Equal(uint64(1), oc.OperatorCount(operator.OpRegion)) // 1:region 2:leader + re.Equal(uint64(1), oc.OperatorCount(operator.OpLeader)) + op2 := newTestOperator(2, tc.GetRegion(2).GetRegionEpoch(), operator.OpRegion) + op2.SetPriorityLevel(constant.High) + oc.AddWaitingOperator(op2) + re.Equal(uint64(2), oc.OperatorCount(operator.OpRegion)) // 1:region 2:region + re.Equal(uint64(0), oc.OperatorCount(operator.OpLeader)) + } +} + +func TestStoreOverloaded(t *testing.T) { + re := require.New(t) + + tc, co, cleanup := prepare(nil, nil, nil, re) + defer cleanup() + oc := co.GetOperatorController() + lb, err := schedulers.CreateScheduler(schedulers.BalanceRegionType, oc, tc.storage, schedulers.ConfigSliceDecoder(schedulers.BalanceRegionType, []string{"", ""})) + re.NoError(err) + opt := tc.GetOpts() + re.NoError(tc.addRegionStore(4, 100)) + re.NoError(tc.addRegionStore(3, 100)) + re.NoError(tc.addRegionStore(2, 100)) + re.NoError(tc.addRegionStore(1, 10)) + re.NoError(tc.addLeaderRegion(1, 2, 3, 4)) + region := tc.GetRegion(1).Clone(core.SetApproximateSize(60)) + tc.putRegion(region) + start := time.Now() + { + ops, _ := lb.Schedule(tc, false /* dryRun */) + re.Len(ops, 1) + op1 := ops[0] + re.NotNil(op1) + re.True(oc.AddOperator(op1)) + re.True(oc.RemoveOperator(op1)) + } + for { + time.Sleep(time.Millisecond * 10) + ops, _ := lb.Schedule(tc, false /* dryRun */) + if time.Since(start) > time.Second { + break + } + re.Empty(ops) + } + + // reset all stores' limit + // scheduling one time needs 1/10 seconds + opt.SetAllStoresLimit(storelimit.AddPeer, 600) + opt.SetAllStoresLimit(storelimit.RemovePeer, 600) + time.Sleep(time.Second) + for i := 0; i < 10; i++ { + ops, _ := lb.Schedule(tc, false /* dryRun */) + re.Len(ops, 1) + op := ops[0] + re.True(oc.AddOperator(op)) + re.True(oc.RemoveOperator(op)) + } + // sleep 1 seconds to make sure that the token is filled up + time.Sleep(time.Second) + for i := 0; i < 100; i++ { + ops, _ := lb.Schedule(tc, false /* dryRun */) + re.Greater(len(ops), 0) + } +} + +func TestStoreOverloadedWithReplace(t *testing.T) { + re := require.New(t) + + tc, co, cleanup := prepare(nil, nil, nil, re) + defer cleanup() + oc := co.GetOperatorController() + lb, err := schedulers.CreateScheduler(schedulers.BalanceRegionType, oc, tc.storage, schedulers.ConfigSliceDecoder(schedulers.BalanceRegionType, []string{"", ""})) + re.NoError(err) + + re.NoError(tc.addRegionStore(4, 100)) + re.NoError(tc.addRegionStore(3, 100)) + re.NoError(tc.addRegionStore(2, 100)) + re.NoError(tc.addRegionStore(1, 10)) + re.NoError(tc.addLeaderRegion(1, 2, 3, 4)) + re.NoError(tc.addLeaderRegion(2, 1, 3, 4)) + region := tc.GetRegion(1).Clone(core.SetApproximateSize(60)) + tc.putRegion(region) + region = tc.GetRegion(2).Clone(core.SetApproximateSize(60)) + tc.putRegion(region) + op1 := newTestOperator(1, tc.GetRegion(1).GetRegionEpoch(), operator.OpRegion, operator.AddPeer{ToStore: 1, PeerID: 1}) + re.True(oc.AddOperator(op1)) + op2 := newTestOperator(1, tc.GetRegion(1).GetRegionEpoch(), operator.OpRegion, operator.AddPeer{ToStore: 2, PeerID: 2}) + op2.SetPriorityLevel(constant.High) + re.True(oc.AddOperator(op2)) + op3 := newTestOperator(1, tc.GetRegion(2).GetRegionEpoch(), operator.OpRegion, operator.AddPeer{ToStore: 1, PeerID: 3}) + re.False(oc.AddOperator(op3)) + ops, _ := lb.Schedule(tc, false /* dryRun */) + re.Empty(ops) + // sleep 2 seconds to make sure that token is filled up + time.Sleep(2 * time.Second) + ops, _ = lb.Schedule(tc, false /* dryRun */) + re.Greater(len(ops), 0) +} + +func TestDownStoreLimit(t *testing.T) { + re := require.New(t) + + tc, co, cleanup := prepare(nil, nil, nil, re) + defer cleanup() + oc := co.GetOperatorController() + rc := co.GetCheckerController().GetRuleChecker() + + tc.addRegionStore(1, 100) + tc.addRegionStore(2, 100) + tc.addRegionStore(3, 100) + tc.addLeaderRegion(1, 1, 2, 3) + + region := tc.GetRegion(1) + tc.setStoreDown(1) + tc.SetStoreLimit(1, storelimit.RemovePeer, 1) + + region = region.Clone(core.WithDownPeers([]*pdpb.PeerStats{ + { + Peer: region.GetStorePeer(1), + DownSeconds: 24 * 60 * 60, + }, + }), core.SetApproximateSize(1)) + tc.putRegion(region) + for i := uint64(1); i < 20; i++ { + tc.addRegionStore(i+3, 100) + op := rc.Check(region) + re.NotNil(op) + re.True(oc.AddOperator(op)) + oc.RemoveOperator(op) + } + + region = region.Clone(core.SetApproximateSize(100)) + tc.putRegion(region) + for i := uint64(20); i < 25; i++ { + tc.addRegionStore(i+3, 100) + op := rc.Check(region) + re.NotNil(op) + re.True(oc.AddOperator(op)) + oc.RemoveOperator(op) + } +} + +// FIXME: remove after move into schedulers package +type mockLimitScheduler struct { + schedulers.Scheduler + limit uint64 + counter *operator.Controller + kind operator.OpKind +} + +func (s *mockLimitScheduler) IsScheduleAllowed(cluster sche.SchedulerCluster) bool { + return s.counter.OperatorCount(s.kind) < s.limit +} + +func TestController(t *testing.T) { + re := require.New(t) + + tc, co, cleanup := prepare(nil, nil, nil, re) + defer cleanup() + oc := co.GetOperatorController() + + re.NoError(tc.addLeaderRegion(1, 1)) + re.NoError(tc.addLeaderRegion(2, 2)) + scheduler, err := schedulers.CreateScheduler(schedulers.BalanceLeaderType, oc, storage.NewStorageWithMemoryBackend(), schedulers.ConfigSliceDecoder(schedulers.BalanceLeaderType, []string{"", ""})) + re.NoError(err) + lb := &mockLimitScheduler{ + Scheduler: scheduler, + counter: oc, + kind: operator.OpLeader, + } + + sc := schedulers.NewScheduleController(tc.ctx, co.GetCluster(), co.GetOperatorController(), lb) + + for i := schedulers.MinScheduleInterval; sc.GetInterval() != schedulers.MaxScheduleInterval; i = sc.GetNextInterval(i) { + re.Equal(i, sc.GetInterval()) + re.Empty(sc.Schedule(false)) + } + // limit = 2 + lb.limit = 2 + // count = 0 + { + re.True(sc.AllowSchedule(false)) + op1 := newTestOperator(1, tc.GetRegion(1).GetRegionEpoch(), operator.OpLeader) + re.Equal(1, oc.AddWaitingOperator(op1)) + // count = 1 + re.True(sc.AllowSchedule(false)) + op2 := newTestOperator(2, tc.GetRegion(2).GetRegionEpoch(), operator.OpLeader) + re.Equal(1, oc.AddWaitingOperator(op2)) + // count = 2 + re.False(sc.AllowSchedule(false)) + re.True(oc.RemoveOperator(op1)) + // count = 1 + re.True(sc.AllowSchedule(false)) + } + + op11 := newTestOperator(1, tc.GetRegion(1).GetRegionEpoch(), operator.OpLeader) + // add a PriorityKind operator will remove old operator + { + op3 := newTestOperator(2, tc.GetRegion(2).GetRegionEpoch(), operator.OpHotRegion) + op3.SetPriorityLevel(constant.High) + re.Equal(1, oc.AddWaitingOperator(op11)) + re.False(sc.AllowSchedule(false)) + re.Equal(1, oc.AddWaitingOperator(op3)) + re.True(sc.AllowSchedule(false)) + re.True(oc.RemoveOperator(op3)) + } + + // add a admin operator will remove old operator + { + op2 := newTestOperator(2, tc.GetRegion(2).GetRegionEpoch(), operator.OpLeader) + re.Equal(1, oc.AddWaitingOperator(op2)) + re.False(sc.AllowSchedule(false)) + op4 := newTestOperator(2, tc.GetRegion(2).GetRegionEpoch(), operator.OpAdmin) + op4.SetPriorityLevel(constant.High) + re.Equal(1, oc.AddWaitingOperator(op4)) + re.True(sc.AllowSchedule(false)) + re.True(oc.RemoveOperator(op4)) + } + + // test wrong region id. + { + op5 := newTestOperator(3, &metapb.RegionEpoch{}, operator.OpHotRegion) + re.Equal(0, oc.AddWaitingOperator(op5)) + } + + // test wrong region epoch. + re.True(oc.RemoveOperator(op11)) + epoch := &metapb.RegionEpoch{ + Version: tc.GetRegion(1).GetRegionEpoch().GetVersion() + 1, + ConfVer: tc.GetRegion(1).GetRegionEpoch().GetConfVer(), + } + { + op6 := newTestOperator(1, epoch, operator.OpLeader) + re.Equal(0, oc.AddWaitingOperator(op6)) + } + epoch.Version-- + { + op6 := newTestOperator(1, epoch, operator.OpLeader) + re.Equal(1, oc.AddWaitingOperator(op6)) + re.True(oc.RemoveOperator(op6)) + } +} + +func TestInterval(t *testing.T) { + re := require.New(t) + + tc, co, cleanup := prepare(nil, nil, nil, re) + defer cleanup() + + lb, err := schedulers.CreateScheduler(schedulers.BalanceLeaderType, co.GetOperatorController(), storage.NewStorageWithMemoryBackend(), schedulers.ConfigSliceDecoder(schedulers.BalanceLeaderType, []string{"", ""})) + re.NoError(err) + sc := schedulers.NewScheduleController(tc.ctx, co.GetCluster(), co.GetOperatorController(), lb) + + // If no operator for x seconds, the next check should be in x/2 seconds. + idleSeconds := []int{5, 10, 20, 30, 60} + for _, n := range idleSeconds { + sc.SetInterval(schedulers.MinScheduleInterval) + for totalSleep := time.Duration(0); totalSleep <= time.Second*time.Duration(n); totalSleep += sc.GetInterval() { + re.Empty(sc.Schedule(false)) + } + re.Less(sc.GetInterval(), time.Second*time.Duration(n/2)) + } +} + +func waitAddLearner(re *require.Assertions, stream mockhbstream.HeartbeatStream, region *core.RegionInfo, storeID uint64) *core.RegionInfo { + var res *pdpb.RegionHeartbeatResponse + testutil.Eventually(re, func() bool { + if res = stream.Recv(); res != nil { + return res.GetRegionId() == region.GetID() && + res.GetChangePeer().GetChangeType() == eraftpb.ConfChangeType_AddLearnerNode && + res.GetChangePeer().GetPeer().GetStoreId() == storeID + } + return false + }) + return region.Clone( + core.WithAddPeer(res.GetChangePeer().GetPeer()), + core.WithIncConfVer(), + ) +} + +func waitPromoteLearner(re *require.Assertions, stream mockhbstream.HeartbeatStream, region *core.RegionInfo, storeID uint64) *core.RegionInfo { + var res *pdpb.RegionHeartbeatResponse + testutil.Eventually(re, func() bool { + if res = stream.Recv(); res != nil { + return res.GetRegionId() == region.GetID() && + res.GetChangePeer().GetChangeType() == eraftpb.ConfChangeType_AddNode && + res.GetChangePeer().GetPeer().GetStoreId() == storeID + } + return false + }) + // Remove learner than add voter. + return region.Clone( + core.WithRemoveStorePeer(storeID), + core.WithAddPeer(res.GetChangePeer().GetPeer()), + ) +} + +func waitRemovePeer(re *require.Assertions, stream mockhbstream.HeartbeatStream, region *core.RegionInfo, storeID uint64) *core.RegionInfo { + var res *pdpb.RegionHeartbeatResponse + testutil.Eventually(re, func() bool { + if res = stream.Recv(); res != nil { + return res.GetRegionId() == region.GetID() && + res.GetChangePeer().GetChangeType() == eraftpb.ConfChangeType_RemoveNode && + res.GetChangePeer().GetPeer().GetStoreId() == storeID + } + return false + }) + return region.Clone( + core.WithRemoveStorePeer(storeID), + core.WithIncConfVer(), + ) +} + +func waitTransferLeader(re *require.Assertions, stream mockhbstream.HeartbeatStream, region *core.RegionInfo, storeID uint64) *core.RegionInfo { + var res *pdpb.RegionHeartbeatResponse + testutil.Eventually(re, func() bool { + if res = stream.Recv(); res != nil { + if res.GetRegionId() == region.GetID() { + for _, peer := range append(res.GetTransferLeader().GetPeers(), res.GetTransferLeader().GetPeer()) { + if peer.GetStoreId() == storeID { + return true + } + } + } + } + return false + }) + return region.Clone( + core.WithLeader(region.GetStorePeer(storeID)), + ) +} + +func waitNoResponse(re *require.Assertions, stream mockhbstream.HeartbeatStream) { + testutil.Eventually(re, func() bool { + res := stream.Recv() + return res == nil + }) +} diff --git a/server/cluster/cluster_worker.go b/server/cluster/cluster_worker.go index fd0acfe7466..51781bde7f6 100644 --- a/server/cluster/cluster_worker.go +++ b/server/cluster/cluster_worker.go @@ -23,7 +23,7 @@ import ( "github.com/pingcap/log" "github.com/tikv/pd/pkg/core" "github.com/tikv/pd/pkg/errs" - "github.com/tikv/pd/pkg/schedule" + "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/statistics/buckets" "github.com/tikv/pd/pkg/utils/logutil" "github.com/tikv/pd/pkg/utils/typeutil" @@ -37,14 +37,14 @@ func (c *RaftCluster) HandleRegionHeartbeat(region *core.RegionInfo) error { return err } - c.coordinator.opController.Dispatch(region, schedule.DispatchFromHeartBeat) + c.coordinator.GetOperatorController().Dispatch(region, operator.DispatchFromHeartBeat, c.coordinator.RecordOpStepWithTTL) return nil } // HandleAskSplit handles the split request. func (c *RaftCluster) HandleAskSplit(request *pdpb.AskSplitRequest) (*pdpb.AskSplitResponse, error) { - if c.GetUnsafeRecoveryController().IsRunning() { - return nil, errs.ErrUnsafeRecoveryIsRunning.FastGenByArgs() + if c.isSchedulingHalted() { + return nil, errs.ErrSchedulingIsHalted.FastGenByArgs() } if !c.opt.IsTikvRegionSplitEnabled() { return nil, errs.ErrSchedulerTiKVSplitDisabled.FastGenByArgs() @@ -86,6 +86,10 @@ func (c *RaftCluster) HandleAskSplit(request *pdpb.AskSplitRequest) (*pdpb.AskSp return split, nil } +func (c *RaftCluster) isSchedulingHalted() bool { + return c.opt.IsSchedulingHalted() +} + // ValidRequestRegion is used to decide if the region is valid. func (c *RaftCluster) ValidRequestRegion(reqRegion *metapb.Region) error { startKey := reqRegion.GetStartKey() @@ -105,8 +109,8 @@ func (c *RaftCluster) ValidRequestRegion(reqRegion *metapb.Region) error { // HandleAskBatchSplit handles the batch split request. func (c *RaftCluster) HandleAskBatchSplit(request *pdpb.AskBatchSplitRequest) (*pdpb.AskBatchSplitResponse, error) { - if c.GetUnsafeRecoveryController().IsRunning() { - return nil, errs.ErrUnsafeRecoveryIsRunning.FastGenByArgs() + if c.isSchedulingHalted() { + return nil, errs.ErrSchedulingIsHalted.FastGenByArgs() } if !c.opt.IsTikvRegionSplitEnabled() { return nil, errs.ErrSchedulerTiKVSplitDisabled.FastGenByArgs() diff --git a/server/cluster/coordinator_test.go b/server/cluster/coordinator_test.go deleted file mode 100644 index 5a1b85b286a..00000000000 --- a/server/cluster/coordinator_test.go +++ /dev/null @@ -1,1408 +0,0 @@ -// Copyright 2016 TiKV Project Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package cluster - -import ( - "context" - "encoding/json" - "math/rand" - "sync" - "testing" - "time" - - "github.com/docker/go-units" - "github.com/pingcap/failpoint" - "github.com/pingcap/kvproto/pkg/eraftpb" - "github.com/pingcap/kvproto/pkg/metapb" - "github.com/pingcap/kvproto/pkg/pdpb" - "github.com/stretchr/testify/require" - "github.com/tikv/pd/pkg/core" - "github.com/tikv/pd/pkg/core/constant" - "github.com/tikv/pd/pkg/core/storelimit" - "github.com/tikv/pd/pkg/mock/mockhbstream" - "github.com/tikv/pd/pkg/schedule" - "github.com/tikv/pd/pkg/schedule/hbstream" - "github.com/tikv/pd/pkg/schedule/labeler" - "github.com/tikv/pd/pkg/schedule/operator" - "github.com/tikv/pd/pkg/schedule/schedulers" - "github.com/tikv/pd/pkg/statistics" - "github.com/tikv/pd/pkg/storage" - "github.com/tikv/pd/pkg/utils/testutil" - "github.com/tikv/pd/pkg/utils/typeutil" - "github.com/tikv/pd/server/config" -) - -func newTestOperator(regionID uint64, regionEpoch *metapb.RegionEpoch, kind operator.OpKind, steps ...operator.OpStep) *operator.Operator { - return operator.NewTestOperator(regionID, regionEpoch, kind, steps...) -} - -func (c *testCluster) AllocPeer(storeID uint64) (*metapb.Peer, error) { - id, err := c.GetAllocator().Alloc() - if err != nil { - return nil, err - } - return &metapb.Peer{Id: id, StoreId: storeID}, nil -} - -func (c *testCluster) addRegionStore(storeID uint64, regionCount int, regionSizes ...uint64) error { - var regionSize uint64 - if len(regionSizes) == 0 { - regionSize = uint64(regionCount) * 10 - } else { - regionSize = regionSizes[0] - } - - stats := &pdpb.StoreStats{} - stats.Capacity = 100 * units.GiB - stats.UsedSize = regionSize * units.MiB - stats.Available = stats.Capacity - stats.UsedSize - newStore := core.NewStoreInfo(&metapb.Store{Id: storeID}, - core.SetStoreStats(stats), - core.SetRegionCount(regionCount), - core.SetRegionSize(int64(regionSize)), - core.SetLastHeartbeatTS(time.Now()), - ) - - c.SetStoreLimit(storeID, storelimit.AddPeer, 60) - c.SetStoreLimit(storeID, storelimit.RemovePeer, 60) - c.Lock() - defer c.Unlock() - return c.putStoreLocked(newStore) -} - -func (c *testCluster) addLeaderRegion(regionID uint64, leaderStoreID uint64, followerStoreIDs ...uint64) error { - region := newTestRegionMeta(regionID) - leader, _ := c.AllocPeer(leaderStoreID) - region.Peers = []*metapb.Peer{leader} - for _, followerStoreID := range followerStoreIDs { - peer, _ := c.AllocPeer(followerStoreID) - region.Peers = append(region.Peers, peer) - } - regionInfo := core.NewRegionInfo(region, leader, core.SetApproximateSize(10), core.SetApproximateKeys(10)) - return c.putRegion(regionInfo) -} - -func (c *testCluster) updateLeaderCount(storeID uint64, leaderCount int) error { - store := c.GetStore(storeID) - newStore := store.Clone( - core.SetLeaderCount(leaderCount), - core.SetLeaderSize(int64(leaderCount)*10), - ) - c.Lock() - defer c.Unlock() - return c.putStoreLocked(newStore) -} - -func (c *testCluster) addLeaderStore(storeID uint64, leaderCount int) error { - stats := &pdpb.StoreStats{} - newStore := core.NewStoreInfo(&metapb.Store{Id: storeID}, - core.SetStoreStats(stats), - core.SetLeaderCount(leaderCount), - core.SetLeaderSize(int64(leaderCount)*10), - core.SetLastHeartbeatTS(time.Now()), - ) - - c.SetStoreLimit(storeID, storelimit.AddPeer, 60) - c.SetStoreLimit(storeID, storelimit.RemovePeer, 60) - c.Lock() - defer c.Unlock() - return c.putStoreLocked(newStore) -} - -func (c *testCluster) setStoreDown(storeID uint64) error { - store := c.GetStore(storeID) - newStore := store.Clone( - core.UpStore(), - core.SetLastHeartbeatTS(typeutil.ZeroTime), - ) - c.Lock() - defer c.Unlock() - return c.putStoreLocked(newStore) -} - -func (c *testCluster) setStoreOffline(storeID uint64) error { - store := c.GetStore(storeID) - newStore := store.Clone(core.OfflineStore(false)) - c.Lock() - defer c.Unlock() - return c.putStoreLocked(newStore) -} - -func (c *testCluster) LoadRegion(regionID uint64, followerStoreIDs ...uint64) error { - // regions load from etcd will have no leader - region := newTestRegionMeta(regionID) - region.Peers = []*metapb.Peer{} - for _, id := range followerStoreIDs { - peer, _ := c.AllocPeer(id) - region.Peers = append(region.Peers, peer) - } - return c.putRegion(core.NewRegionInfo(region, nil)) -} - -func TestBasic(t *testing.T) { - re := require.New(t) - - tc, co, cleanup := prepare(nil, nil, nil, re) - defer cleanup() - oc := co.opController - - re.NoError(tc.addLeaderRegion(1, 1)) - - op1 := newTestOperator(1, tc.GetRegion(1).GetRegionEpoch(), operator.OpLeader) - oc.AddWaitingOperator(op1) - re.Equal(uint64(1), oc.OperatorCount(operator.OpLeader)) - re.Equal(op1.RegionID(), oc.GetOperator(1).RegionID()) - - // Region 1 already has an operator, cannot add another one. - op2 := newTestOperator(1, tc.GetRegion(1).GetRegionEpoch(), operator.OpRegion) - oc.AddWaitingOperator(op2) - re.Equal(uint64(0), oc.OperatorCount(operator.OpRegion)) - - // Remove the operator manually, then we can add a new operator. - re.True(oc.RemoveOperator(op1)) - op3 := newTestOperator(1, tc.GetRegion(1).GetRegionEpoch(), operator.OpRegion) - oc.AddWaitingOperator(op3) - re.Equal(uint64(1), oc.OperatorCount(operator.OpRegion)) - re.Equal(op3.RegionID(), oc.GetOperator(1).RegionID()) -} - -func TestDispatch(t *testing.T) { - re := require.New(t) - - tc, co, cleanup := prepare(nil, nil, nil, re) - defer cleanup() - co.prepareChecker.prepared = true - // Transfer peer from store 4 to store 1. - re.NoError(tc.addRegionStore(4, 40)) - re.NoError(tc.addRegionStore(3, 30)) - re.NoError(tc.addRegionStore(2, 20)) - re.NoError(tc.addRegionStore(1, 10)) - re.NoError(tc.addLeaderRegion(1, 2, 3, 4)) - - // Transfer leader from store 4 to store 2. - re.NoError(tc.updateLeaderCount(4, 50)) - re.NoError(tc.updateLeaderCount(3, 50)) - re.NoError(tc.updateLeaderCount(2, 20)) - re.NoError(tc.updateLeaderCount(1, 10)) - re.NoError(tc.addLeaderRegion(2, 4, 3, 2)) - - go co.runUntilStop() - - // Wait for schedule and turn off balance. - waitOperator(re, co, 1) - testutil.CheckTransferPeer(re, co.opController.GetOperator(1), operator.OpKind(0), 4, 1) - re.NoError(co.removeScheduler(schedulers.BalanceRegionName)) - waitOperator(re, co, 2) - testutil.CheckTransferLeader(re, co.opController.GetOperator(2), operator.OpKind(0), 4, 2) - re.NoError(co.removeScheduler(schedulers.BalanceLeaderName)) - - stream := mockhbstream.NewHeartbeatStream() - - // Transfer peer. - region := tc.GetRegion(1).Clone() - re.NoError(dispatchHeartbeat(co, region, stream)) - region = waitAddLearner(re, stream, region, 1) - re.NoError(dispatchHeartbeat(co, region, stream)) - region = waitPromoteLearner(re, stream, region, 1) - re.NoError(dispatchHeartbeat(co, region, stream)) - region = waitRemovePeer(re, stream, region, 4) - re.NoError(dispatchHeartbeat(co, region, stream)) - re.NoError(dispatchHeartbeat(co, region, stream)) - waitNoResponse(re, stream) - - // Transfer leader. - region = tc.GetRegion(2).Clone() - re.NoError(dispatchHeartbeat(co, region, stream)) - waitTransferLeader(re, stream, region, 2) - re.NoError(dispatchHeartbeat(co, region, stream)) - waitNoResponse(re, stream) -} - -func dispatchHeartbeat(co *coordinator, region *core.RegionInfo, stream hbstream.HeartbeatStream) error { - co.hbStreams.BindStream(region.GetLeader().GetStoreId(), stream) - if err := co.cluster.putRegion(region.Clone()); err != nil { - return err - } - co.opController.Dispatch(region, schedule.DispatchFromHeartBeat) - return nil -} - -func TestCollectMetricsConcurrent(t *testing.T) { - re := require.New(t) - - tc, co, cleanup := prepare(nil, func(tc *testCluster) { - tc.regionStats = statistics.NewRegionStatistics(tc.GetOpts(), nil, tc.storeConfigManager) - }, func(co *coordinator) { co.run() }, re) - defer cleanup() - - // Make sure there are no problem when concurrent write and read - var wg sync.WaitGroup - count := 10 - wg.Add(count + 1) - for i := 0; i <= count; i++ { - go func(i int) { - defer wg.Done() - for j := 0; j < 1000; j++ { - re.NoError(tc.addRegionStore(uint64(i%5), rand.Intn(200))) - } - }(i) - } - for i := 0; i < 1000; i++ { - co.collectHotSpotMetrics() - co.collectSchedulerMetrics() - co.cluster.collectClusterMetrics() - } - co.resetHotSpotMetrics() - co.resetSchedulerMetrics() - co.cluster.resetClusterMetrics() - wg.Wait() -} - -func TestCollectMetrics(t *testing.T) { - re := require.New(t) - - tc, co, cleanup := prepare(nil, func(tc *testCluster) { - tc.regionStats = statistics.NewRegionStatistics(tc.GetOpts(), nil, tc.storeConfigManager) - }, func(co *coordinator) { co.run() }, re) - defer cleanup() - count := 10 - for i := 0; i <= count; i++ { - for k := 0; k < 200; k++ { - item := &statistics.HotPeerStat{ - StoreID: uint64(i % 5), - RegionID: uint64(i*1000 + k), - Loads: []float64{10, 20, 30}, - HotDegree: 10, - AntiCount: statistics.HotRegionAntiCount, // for write - } - tc.hotStat.HotCache.Update(item, statistics.Write) - } - } - for i := 0; i < 1000; i++ { - co.collectHotSpotMetrics() - co.collectSchedulerMetrics() - co.cluster.collectClusterMetrics() - } - stores := co.cluster.GetStores() - regionStats := co.cluster.RegionWriteStats() - status1 := statistics.CollectHotPeerInfos(stores, regionStats) - status2 := statistics.GetHotStatus(stores, co.cluster.GetStoresLoads(), regionStats, statistics.Write, co.cluster.GetOpts().IsTraceRegionFlow()) - for _, s := range status2.AsLeader { - s.Stats = nil - } - for _, s := range status2.AsPeer { - s.Stats = nil - } - re.Equal(status1, status2) - co.resetHotSpotMetrics() - co.resetSchedulerMetrics() - co.cluster.resetClusterMetrics() -} - -func prepare(setCfg func(*config.ScheduleConfig), setTc func(*testCluster), run func(*coordinator), re *require.Assertions) (*testCluster, *coordinator, func()) { - ctx, cancel := context.WithCancel(context.Background()) - cfg, opt, err := newTestScheduleConfig() - re.NoError(err) - if setCfg != nil { - setCfg(cfg) - } - tc := newTestCluster(ctx, opt) - hbStreams := hbstream.NewTestHeartbeatStreams(ctx, tc.meta.GetId(), tc, true /* need to run */) - if setTc != nil { - setTc(tc) - } - co := newCoordinator(ctx, tc.RaftCluster, hbStreams) - if run != nil { - run(co) - } - return tc, co, func() { - co.stop() - co.wg.Wait() - hbStreams.Close() - cancel() - } -} - -func checkRegionAndOperator(re *require.Assertions, tc *testCluster, co *coordinator, regionID uint64, expectAddOperator int) { - ops := co.checkers.CheckRegion(tc.GetRegion(regionID)) - if ops == nil { - re.Equal(0, expectAddOperator) - } else { - re.Equal(expectAddOperator, co.opController.AddWaitingOperator(ops...)) - } -} - -func TestCheckRegion(t *testing.T) { - re := require.New(t) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - tc, co, cleanup := prepare(nil, nil, nil, re) - hbStreams, opt := co.hbStreams, tc.opt - defer cleanup() - - re.NoError(tc.addRegionStore(4, 4)) - re.NoError(tc.addRegionStore(3, 3)) - re.NoError(tc.addRegionStore(2, 2)) - re.NoError(tc.addRegionStore(1, 1)) - re.NoError(tc.addLeaderRegion(1, 2, 3)) - checkRegionAndOperator(re, tc, co, 1, 1) - testutil.CheckAddPeer(re, co.opController.GetOperator(1), operator.OpReplica, 1) - checkRegionAndOperator(re, tc, co, 1, 0) - - r := tc.GetRegion(1) - p := &metapb.Peer{Id: 1, StoreId: 1, Role: metapb.PeerRole_Learner} - r = r.Clone( - core.WithAddPeer(p), - core.WithPendingPeers(append(r.GetPendingPeers(), p)), - ) - re.NoError(tc.putRegion(r)) - checkRegionAndOperator(re, tc, co, 1, 0) - - tc = newTestCluster(ctx, opt) - co = newCoordinator(ctx, tc.RaftCluster, hbStreams) - - re.NoError(tc.addRegionStore(4, 4)) - re.NoError(tc.addRegionStore(3, 3)) - re.NoError(tc.addRegionStore(2, 2)) - re.NoError(tc.addRegionStore(1, 1)) - re.NoError(tc.putRegion(r)) - checkRegionAndOperator(re, tc, co, 1, 0) - r = r.Clone(core.WithPendingPeers(nil)) - re.NoError(tc.putRegion(r)) - checkRegionAndOperator(re, tc, co, 1, 1) - op := co.opController.GetOperator(1) - re.Equal(1, op.Len()) - re.Equal(uint64(1), op.Step(0).(operator.PromoteLearner).ToStore) - checkRegionAndOperator(re, tc, co, 1, 0) -} - -func TestCheckRegionWithScheduleDeny(t *testing.T) { - re := require.New(t) - - tc, co, cleanup := prepare(nil, nil, nil, re) - - defer cleanup() - - re.NoError(tc.addRegionStore(4, 4)) - re.NoError(tc.addRegionStore(3, 3)) - re.NoError(tc.addRegionStore(2, 2)) - re.NoError(tc.addRegionStore(1, 1)) - re.NoError(tc.addLeaderRegion(1, 2, 3)) - region := tc.GetRegion(1) - re.NotNil(region) - // test with label schedule=deny - labelerManager := tc.GetRegionLabeler() - labelerManager.SetLabelRule(&labeler.LabelRule{ - ID: "schedulelabel", - Labels: []labeler.RegionLabel{{Key: "schedule", Value: "deny"}}, - RuleType: labeler.KeyRange, - Data: []interface{}{map[string]interface{}{"start_key": "", "end_key": ""}}, - }) - - re.True(labelerManager.ScheduleDisabled(region)) - checkRegionAndOperator(re, tc, co, 1, 0) - labelerManager.DeleteLabelRule("schedulelabel") - re.False(labelerManager.ScheduleDisabled(region)) - checkRegionAndOperator(re, tc, co, 1, 1) -} - -func TestCheckerIsBusy(t *testing.T) { - re := require.New(t) - - tc, co, cleanup := prepare(func(cfg *config.ScheduleConfig) { - cfg.ReplicaScheduleLimit = 0 // ensure replica checker is busy - cfg.MergeScheduleLimit = 10 - }, nil, nil, re) - defer cleanup() - - re.NoError(tc.addRegionStore(1, 0)) - num := 1 + typeutil.MaxUint64(tc.opt.GetReplicaScheduleLimit(), tc.opt.GetMergeScheduleLimit()) - var operatorKinds = []operator.OpKind{ - operator.OpReplica, operator.OpRegion | operator.OpMerge, - } - for i, operatorKind := range operatorKinds { - for j := uint64(0); j < num; j++ { - regionID := j + uint64(i+1)*num - re.NoError(tc.addLeaderRegion(regionID, 1)) - switch operatorKind { - case operator.OpReplica: - op := newTestOperator(regionID, tc.GetRegion(regionID).GetRegionEpoch(), operatorKind) - re.Equal(1, co.opController.AddWaitingOperator(op)) - case operator.OpRegion | operator.OpMerge: - if regionID%2 == 1 { - ops, err := operator.CreateMergeRegionOperator("merge-region", co.cluster, tc.GetRegion(regionID), tc.GetRegion(regionID-1), operator.OpMerge) - re.NoError(err) - re.Len(ops, co.opController.AddWaitingOperator(ops...)) - } - } - } - } - checkRegionAndOperator(re, tc, co, num, 0) -} - -func TestReplica(t *testing.T) { - re := require.New(t) - - tc, co, cleanup := prepare(func(cfg *config.ScheduleConfig) { - // Turn off balance. - cfg.LeaderScheduleLimit = 0 - cfg.RegionScheduleLimit = 0 - }, nil, func(co *coordinator) { co.run() }, re) - defer cleanup() - - re.NoError(tc.addRegionStore(1, 1)) - re.NoError(tc.addRegionStore(2, 2)) - re.NoError(tc.addRegionStore(3, 3)) - re.NoError(tc.addRegionStore(4, 4)) - - stream := mockhbstream.NewHeartbeatStream() - - // Add peer to store 1. - re.NoError(tc.addLeaderRegion(1, 2, 3)) - region := tc.GetRegion(1) - re.NoError(dispatchHeartbeat(co, region, stream)) - region = waitAddLearner(re, stream, region, 1) - re.NoError(dispatchHeartbeat(co, region, stream)) - region = waitPromoteLearner(re, stream, region, 1) - re.NoError(dispatchHeartbeat(co, region, stream)) - waitNoResponse(re, stream) - - // Peer in store 3 is down, remove peer in store 3 and add peer to store 4. - re.NoError(tc.setStoreDown(3)) - downPeer := &pdpb.PeerStats{ - Peer: region.GetStorePeer(3), - DownSeconds: 24 * 60 * 60, - } - region = region.Clone( - core.WithDownPeers(append(region.GetDownPeers(), downPeer)), - ) - re.NoError(dispatchHeartbeat(co, region, stream)) - region = waitAddLearner(re, stream, region, 4) - re.NoError(dispatchHeartbeat(co, region, stream)) - region = waitPromoteLearner(re, stream, region, 4) - region = region.Clone(core.WithDownPeers(nil)) - re.NoError(dispatchHeartbeat(co, region, stream)) - waitNoResponse(re, stream) - - // Remove peer from store 4. - re.NoError(tc.addLeaderRegion(2, 1, 2, 3, 4)) - region = tc.GetRegion(2) - re.NoError(dispatchHeartbeat(co, region, stream)) - region = waitRemovePeer(re, stream, region, 4) - re.NoError(dispatchHeartbeat(co, region, stream)) - waitNoResponse(re, stream) - - // Remove offline peer directly when it's pending. - re.NoError(tc.addLeaderRegion(3, 1, 2, 3)) - re.NoError(tc.setStoreOffline(3)) - region = tc.GetRegion(3) - region = region.Clone(core.WithPendingPeers([]*metapb.Peer{region.GetStorePeer(3)})) - re.NoError(dispatchHeartbeat(co, region, stream)) - waitNoResponse(re, stream) -} - -func TestCheckCache(t *testing.T) { - re := require.New(t) - - tc, co, cleanup := prepare(func(cfg *config.ScheduleConfig) { - // Turn off replica scheduling. - cfg.ReplicaScheduleLimit = 0 - }, nil, nil, re) - defer cleanup() - - re.NoError(tc.addRegionStore(1, 0)) - re.NoError(tc.addRegionStore(2, 0)) - re.NoError(tc.addRegionStore(3, 0)) - - // Add a peer with two replicas. - re.NoError(tc.addLeaderRegion(1, 2, 3)) - re.NoError(failpoint.Enable("github.com/tikv/pd/server/cluster/break-patrol", `return`)) - - // case 1: operator cannot be created due to replica-schedule-limit restriction - co.wg.Add(1) - co.patrolRegions() - re.Len(co.checkers.GetWaitingRegions(), 1) - - // cancel the replica-schedule-limit restriction - cfg := tc.GetScheduleConfig() - cfg.ReplicaScheduleLimit = 10 - tc.SetScheduleConfig(cfg) - co.wg.Add(1) - co.patrolRegions() - oc := co.opController - re.Len(oc.GetOperators(), 1) - re.Empty(co.checkers.GetWaitingRegions()) - - // case 2: operator cannot be created due to store limit restriction - oc.RemoveOperator(oc.GetOperator(1)) - tc.SetStoreLimit(1, storelimit.AddPeer, 0) - co.wg.Add(1) - co.patrolRegions() - re.Len(co.checkers.GetWaitingRegions(), 1) - - // cancel the store limit restriction - tc.SetStoreLimit(1, storelimit.AddPeer, 10) - time.Sleep(time.Second) - co.wg.Add(1) - co.patrolRegions() - re.Len(oc.GetOperators(), 1) - re.Empty(co.checkers.GetWaitingRegions()) - - co.wg.Wait() - re.NoError(failpoint.Disable("github.com/tikv/pd/server/cluster/break-patrol")) -} - -func TestPeerState(t *testing.T) { - re := require.New(t) - - tc, co, cleanup := prepare(nil, nil, func(co *coordinator) { co.run() }, re) - defer cleanup() - - // Transfer peer from store 4 to store 1. - re.NoError(tc.addRegionStore(1, 10)) - re.NoError(tc.addRegionStore(2, 10)) - re.NoError(tc.addRegionStore(3, 10)) - re.NoError(tc.addRegionStore(4, 40)) - re.NoError(tc.addLeaderRegion(1, 2, 3, 4)) - - stream := mockhbstream.NewHeartbeatStream() - - // Wait for schedule. - waitOperator(re, co, 1) - testutil.CheckTransferPeer(re, co.opController.GetOperator(1), operator.OpKind(0), 4, 1) - - region := tc.GetRegion(1).Clone() - - // Add new peer. - re.NoError(dispatchHeartbeat(co, region, stream)) - region = waitAddLearner(re, stream, region, 1) - re.NoError(dispatchHeartbeat(co, region, stream)) - region = waitPromoteLearner(re, stream, region, 1) - - // If the new peer is pending, the operator will not finish. - region = region.Clone(core.WithPendingPeers(append(region.GetPendingPeers(), region.GetStorePeer(1)))) - re.NoError(dispatchHeartbeat(co, region, stream)) - waitNoResponse(re, stream) - re.NotNil(co.opController.GetOperator(region.GetID())) - - // The new peer is not pending now, the operator will finish. - // And we will proceed to remove peer in store 4. - region = region.Clone(core.WithPendingPeers(nil)) - re.NoError(dispatchHeartbeat(co, region, stream)) - waitRemovePeer(re, stream, region, 4) - re.NoError(tc.addLeaderRegion(1, 1, 2, 3)) - region = tc.GetRegion(1).Clone() - re.NoError(dispatchHeartbeat(co, region, stream)) - waitNoResponse(re, stream) -} - -func TestShouldRun(t *testing.T) { - re := require.New(t) - - tc, co, cleanup := prepare(nil, nil, nil, re) - tc.RaftCluster.coordinator = co - defer cleanup() - - re.NoError(tc.addLeaderStore(1, 5)) - re.NoError(tc.addLeaderStore(2, 2)) - re.NoError(tc.addLeaderStore(3, 0)) - re.NoError(tc.addLeaderStore(4, 0)) - re.NoError(tc.LoadRegion(1, 1, 2, 3)) - re.NoError(tc.LoadRegion(2, 1, 2, 3)) - re.NoError(tc.LoadRegion(3, 1, 2, 3)) - re.NoError(tc.LoadRegion(4, 1, 2, 3)) - re.NoError(tc.LoadRegion(5, 1, 2, 3)) - re.NoError(tc.LoadRegion(6, 2, 1, 4)) - re.NoError(tc.LoadRegion(7, 2, 1, 4)) - re.False(co.shouldRun()) - re.Equal(2, tc.GetStoreRegionCount(4)) - - testCases := []struct { - regionID uint64 - shouldRun bool - }{ - {1, false}, - {2, false}, - {3, false}, - {4, false}, - {5, false}, - // store4 needs collect two region - {6, false}, - {7, true}, - } - - for _, testCase := range testCases { - r := tc.GetRegion(testCase.regionID) - nr := r.Clone(core.WithLeader(r.GetPeers()[0])) - re.NoError(tc.processRegionHeartbeat(nr)) - re.Equal(testCase.shouldRun, co.shouldRun()) - } - nr := &metapb.Region{Id: 6, Peers: []*metapb.Peer{}} - newRegion := core.NewRegionInfo(nr, nil) - re.Error(tc.processRegionHeartbeat(newRegion)) - re.Equal(7, co.prepareChecker.sum) -} - -func TestShouldRunWithNonLeaderRegions(t *testing.T) { - re := require.New(t) - - tc, co, cleanup := prepare(nil, nil, nil, re) - tc.RaftCluster.coordinator = co - defer cleanup() - - re.NoError(tc.addLeaderStore(1, 10)) - re.NoError(tc.addLeaderStore(2, 0)) - re.NoError(tc.addLeaderStore(3, 0)) - for i := 0; i < 10; i++ { - re.NoError(tc.LoadRegion(uint64(i+1), 1, 2, 3)) - } - re.False(co.shouldRun()) - re.Equal(10, tc.GetStoreRegionCount(1)) - - testCases := []struct { - regionID uint64 - shouldRun bool - }{ - {1, false}, - {2, false}, - {3, false}, - {4, false}, - {5, false}, - {6, false}, - {7, false}, - {8, false}, - {9, true}, - } - - for _, testCase := range testCases { - r := tc.GetRegion(testCase.regionID) - nr := r.Clone(core.WithLeader(r.GetPeers()[0])) - re.NoError(tc.processRegionHeartbeat(nr)) - re.Equal(testCase.shouldRun, co.shouldRun()) - } - nr := &metapb.Region{Id: 9, Peers: []*metapb.Peer{}} - newRegion := core.NewRegionInfo(nr, nil) - re.Error(tc.processRegionHeartbeat(newRegion)) - re.Equal(9, co.prepareChecker.sum) - - // Now, after server is prepared, there exist some regions with no leader. - re.Equal(uint64(0), tc.GetRegion(10).GetLeader().GetStoreId()) -} - -func TestAddScheduler(t *testing.T) { - re := require.New(t) - - tc, co, cleanup := prepare(nil, nil, func(co *coordinator) { co.run() }, re) - defer cleanup() - - re.Len(co.schedulers, len(config.DefaultSchedulers)) - re.NoError(co.removeScheduler(schedulers.BalanceLeaderName)) - re.NoError(co.removeScheduler(schedulers.BalanceRegionName)) - re.NoError(co.removeScheduler(schedulers.HotRegionName)) - re.NoError(co.removeScheduler(schedulers.SplitBucketName)) - re.NoError(co.removeScheduler(schedulers.BalanceWitnessName)) - re.NoError(co.removeScheduler(schedulers.TransferWitnessLeaderName)) - re.Empty(co.schedulers) - - stream := mockhbstream.NewHeartbeatStream() - - // Add stores 1,2,3 - re.NoError(tc.addLeaderStore(1, 1)) - re.NoError(tc.addLeaderStore(2, 1)) - re.NoError(tc.addLeaderStore(3, 1)) - // Add regions 1 with leader in store 1 and followers in stores 2,3 - re.NoError(tc.addLeaderRegion(1, 1, 2, 3)) - // Add regions 2 with leader in store 2 and followers in stores 1,3 - re.NoError(tc.addLeaderRegion(2, 2, 1, 3)) - // Add regions 3 with leader in store 3 and followers in stores 1,2 - re.NoError(tc.addLeaderRegion(3, 3, 1, 2)) - - oc := co.opController - - // test ConfigJSONDecoder create - bl, err := schedule.CreateScheduler(schedulers.BalanceLeaderType, oc, storage.NewStorageWithMemoryBackend(), schedule.ConfigJSONDecoder([]byte("{}"))) - re.NoError(err) - conf, err := bl.EncodeConfig() - re.NoError(err) - data := make(map[string]interface{}) - err = json.Unmarshal(conf, &data) - re.NoError(err) - batch := data["batch"].(float64) - re.Equal(4, int(batch)) - - gls, err := schedule.CreateScheduler(schedulers.GrantLeaderType, oc, storage.NewStorageWithMemoryBackend(), schedule.ConfigSliceDecoder(schedulers.GrantLeaderType, []string{"0"})) - re.NoError(err) - re.NotNil(co.addScheduler(gls)) - re.NotNil(co.removeScheduler(gls.GetName())) - - gls, err = schedule.CreateScheduler(schedulers.GrantLeaderType, oc, storage.NewStorageWithMemoryBackend(), schedule.ConfigSliceDecoder(schedulers.GrantLeaderType, []string{"1"})) - re.NoError(err) - re.NoError(co.addScheduler(gls)) - - hb, err := schedule.CreateScheduler(schedulers.HotRegionType, oc, storage.NewStorageWithMemoryBackend(), schedule.ConfigJSONDecoder([]byte("{}"))) - re.NoError(err) - conf, err = hb.EncodeConfig() - re.NoError(err) - data = make(map[string]interface{}) - re.NoError(json.Unmarshal(conf, &data)) - re.Contains(data, "enable-for-tiflash") - re.Equal("true", data["enable-for-tiflash"].(string)) - - // Transfer all leaders to store 1. - waitOperator(re, co, 2) - region2 := tc.GetRegion(2) - re.NoError(dispatchHeartbeat(co, region2, stream)) - region2 = waitTransferLeader(re, stream, region2, 1) - re.NoError(dispatchHeartbeat(co, region2, stream)) - waitNoResponse(re, stream) - - waitOperator(re, co, 3) - region3 := tc.GetRegion(3) - re.NoError(dispatchHeartbeat(co, region3, stream)) - region3 = waitTransferLeader(re, stream, region3, 1) - re.NoError(dispatchHeartbeat(co, region3, stream)) - waitNoResponse(re, stream) -} - -func TestPersistScheduler(t *testing.T) { - re := require.New(t) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - tc, co, cleanup := prepare(nil, nil, func(co *coordinator) { co.run() }, re) - hbStreams := co.hbStreams - defer cleanup() - - // Add stores 1,2 - re.NoError(tc.addLeaderStore(1, 1)) - re.NoError(tc.addLeaderStore(2, 1)) - - re.Len(co.schedulers, 6) - oc := co.opController - storage := tc.RaftCluster.storage - - gls1, err := schedule.CreateScheduler(schedulers.GrantLeaderType, oc, storage, schedule.ConfigSliceDecoder(schedulers.GrantLeaderType, []string{"1"})) - re.NoError(err) - re.NoError(co.addScheduler(gls1, "1")) - evict, err := schedule.CreateScheduler(schedulers.EvictLeaderType, oc, storage, schedule.ConfigSliceDecoder(schedulers.EvictLeaderType, []string{"2"})) - re.NoError(err) - re.NoError(co.addScheduler(evict, "2")) - re.Len(co.schedulers, 8) - sches, _, err := storage.LoadAllScheduleConfig() - re.NoError(err) - re.Len(sches, 8) - re.NoError(co.removeScheduler(schedulers.BalanceLeaderName)) - re.NoError(co.removeScheduler(schedulers.BalanceRegionName)) - re.NoError(co.removeScheduler(schedulers.HotRegionName)) - re.NoError(co.removeScheduler(schedulers.SplitBucketName)) - re.NoError(co.removeScheduler(schedulers.BalanceWitnessName)) - re.NoError(co.removeScheduler(schedulers.TransferWitnessLeaderName)) - re.Len(co.schedulers, 2) - re.NoError(co.cluster.opt.Persist(storage)) - co.stop() - co.wg.Wait() - // make a new coordinator for testing - // whether the schedulers added or removed in dynamic way are recorded in opt - _, newOpt, err := newTestScheduleConfig() - re.NoError(err) - _, err = schedule.CreateScheduler(schedulers.ShuffleRegionType, oc, storage, schedule.ConfigJSONDecoder([]byte("null"))) - re.NoError(err) - // suppose we add a new default enable scheduler - config.DefaultSchedulers = append(config.DefaultSchedulers, config.SchedulerConfig{Type: "shuffle-region"}) - defer func() { - config.DefaultSchedulers = config.DefaultSchedulers[:len(config.DefaultSchedulers)-1] - }() - re.Len(newOpt.GetSchedulers(), 6) - re.NoError(newOpt.Reload(storage)) - // only remains 3 items with independent config. - sches, _, err = storage.LoadAllScheduleConfig() - re.NoError(err) - re.Len(sches, 3) - - // option have 6 items because the default scheduler do not remove. - re.Len(newOpt.GetSchedulers(), 9) - re.NoError(newOpt.Persist(storage)) - tc.RaftCluster.opt = newOpt - - co = newCoordinator(ctx, tc.RaftCluster, hbStreams) - co.run() - re.Len(co.schedulers, 3) - co.stop() - co.wg.Wait() - // suppose restart PD again - _, newOpt, err = newTestScheduleConfig() - re.NoError(err) - re.NoError(newOpt.Reload(storage)) - tc.RaftCluster.opt = newOpt - co = newCoordinator(ctx, tc.RaftCluster, hbStreams) - co.run() - re.Len(co.schedulers, 3) - bls, err := schedule.CreateScheduler(schedulers.BalanceLeaderType, oc, storage, schedule.ConfigSliceDecoder(schedulers.BalanceLeaderType, []string{"", ""})) - re.NoError(err) - re.NoError(co.addScheduler(bls)) - brs, err := schedule.CreateScheduler(schedulers.BalanceRegionType, oc, storage, schedule.ConfigSliceDecoder(schedulers.BalanceRegionType, []string{"", ""})) - re.NoError(err) - re.NoError(co.addScheduler(brs)) - re.Len(co.schedulers, 5) - - // the scheduler option should contain 6 items - // the `hot scheduler` are disabled - re.Len(co.cluster.opt.GetSchedulers(), 9) - re.NoError(co.removeScheduler(schedulers.GrantLeaderName)) - // the scheduler that is not enable by default will be completely deleted - re.Len(co.cluster.opt.GetSchedulers(), 8) - re.Len(co.schedulers, 4) - re.NoError(co.cluster.opt.Persist(co.cluster.storage)) - co.stop() - co.wg.Wait() - _, newOpt, err = newTestScheduleConfig() - re.NoError(err) - re.NoError(newOpt.Reload(co.cluster.storage)) - tc.RaftCluster.opt = newOpt - co = newCoordinator(ctx, tc.RaftCluster, hbStreams) - - co.run() - re.Len(co.schedulers, 4) - re.NoError(co.removeScheduler(schedulers.EvictLeaderName)) - re.Len(co.schedulers, 3) -} - -func TestRemoveScheduler(t *testing.T) { - re := require.New(t) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - tc, co, cleanup := prepare(func(cfg *config.ScheduleConfig) { - cfg.ReplicaScheduleLimit = 0 - }, nil, func(co *coordinator) { co.run() }, re) - hbStreams := co.hbStreams - defer cleanup() - - // Add stores 1,2 - re.NoError(tc.addLeaderStore(1, 1)) - re.NoError(tc.addLeaderStore(2, 1)) - - re.Len(co.schedulers, 6) - oc := co.opController - storage := tc.RaftCluster.storage - - gls1, err := schedule.CreateScheduler(schedulers.GrantLeaderType, oc, storage, schedule.ConfigSliceDecoder(schedulers.GrantLeaderType, []string{"1"})) - re.NoError(err) - re.NoError(co.addScheduler(gls1, "1")) - re.Len(co.schedulers, 7) - sches, _, err := storage.LoadAllScheduleConfig() - re.NoError(err) - re.Len(sches, 7) - - // remove all schedulers - re.NoError(co.removeScheduler(schedulers.BalanceLeaderName)) - re.NoError(co.removeScheduler(schedulers.BalanceRegionName)) - re.NoError(co.removeScheduler(schedulers.HotRegionName)) - re.NoError(co.removeScheduler(schedulers.GrantLeaderName)) - re.NoError(co.removeScheduler(schedulers.SplitBucketName)) - re.NoError(co.removeScheduler(schedulers.BalanceWitnessName)) - re.NoError(co.removeScheduler(schedulers.TransferWitnessLeaderName)) - // all removed - sches, _, err = storage.LoadAllScheduleConfig() - re.NoError(err) - re.Empty(sches) - re.Empty(co.schedulers) - re.NoError(co.cluster.opt.Persist(co.cluster.storage)) - co.stop() - co.wg.Wait() - - // suppose restart PD again - _, newOpt, err := newTestScheduleConfig() - re.NoError(err) - re.NoError(newOpt.Reload(tc.storage)) - tc.RaftCluster.opt = newOpt - co = newCoordinator(ctx, tc.RaftCluster, hbStreams) - co.run() - re.Empty(co.schedulers) - // the option remains default scheduler - re.Len(co.cluster.opt.GetSchedulers(), 6) - co.stop() - co.wg.Wait() -} - -func TestRestart(t *testing.T) { - re := require.New(t) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - tc, co, cleanup := prepare(func(cfg *config.ScheduleConfig) { - // Turn off balance, we test add replica only. - cfg.LeaderScheduleLimit = 0 - cfg.RegionScheduleLimit = 0 - }, nil, func(co *coordinator) { co.run() }, re) - hbStreams := co.hbStreams - defer cleanup() - - // Add 3 stores (1, 2, 3) and a region with 1 replica on store 1. - re.NoError(tc.addRegionStore(1, 1)) - re.NoError(tc.addRegionStore(2, 2)) - re.NoError(tc.addRegionStore(3, 3)) - re.NoError(tc.addLeaderRegion(1, 1)) - region := tc.GetRegion(1) - co.prepareChecker.collect(region) - - // Add 1 replica on store 2. - stream := mockhbstream.NewHeartbeatStream() - re.NoError(dispatchHeartbeat(co, region, stream)) - region = waitAddLearner(re, stream, region, 2) - re.NoError(dispatchHeartbeat(co, region, stream)) - region = waitPromoteLearner(re, stream, region, 2) - co.stop() - co.wg.Wait() - - // Recreate coordinator then add another replica on store 3. - co = newCoordinator(ctx, tc.RaftCluster, hbStreams) - co.prepareChecker.collect(region) - co.run() - re.NoError(dispatchHeartbeat(co, region, stream)) - region = waitAddLearner(re, stream, region, 3) - re.NoError(dispatchHeartbeat(co, region, stream)) - waitPromoteLearner(re, stream, region, 3) -} - -func TestPauseScheduler(t *testing.T) { - re := require.New(t) - - _, co, cleanup := prepare(nil, nil, func(co *coordinator) { co.run() }, re) - defer cleanup() - _, err := co.isSchedulerAllowed("test") - re.Error(err) - co.pauseOrResumeScheduler(schedulers.BalanceLeaderName, 60) - paused, _ := co.isSchedulerPaused(schedulers.BalanceLeaderName) - re.True(paused) - pausedAt, err := co.getPausedSchedulerDelayAt(schedulers.BalanceLeaderName) - re.NoError(err) - resumeAt, err := co.getPausedSchedulerDelayUntil(schedulers.BalanceLeaderName) - re.NoError(err) - re.Equal(int64(60), resumeAt-pausedAt) - allowed, _ := co.isSchedulerAllowed(schedulers.BalanceLeaderName) - re.False(allowed) -} - -func BenchmarkPatrolRegion(b *testing.B) { - re := require.New(b) - - mergeLimit := uint64(4100) - regionNum := 10000 - - tc, co, cleanup := prepare(func(cfg *config.ScheduleConfig) { - cfg.MergeScheduleLimit = mergeLimit - }, nil, nil, re) - defer cleanup() - - tc.opt.SetSplitMergeInterval(time.Duration(0)) - for i := 1; i < 4; i++ { - if err := tc.addRegionStore(uint64(i), regionNum, 96); err != nil { - return - } - } - for i := 0; i < regionNum; i++ { - if err := tc.addLeaderRegion(uint64(i), 1, 2, 3); err != nil { - return - } - } - - listen := make(chan int) - go func() { - oc := co.opController - listen <- 0 - for { - if oc.OperatorCount(operator.OpMerge) == mergeLimit { - co.cancel() - return - } - } - }() - <-listen - - co.wg.Add(1) - b.ResetTimer() - co.patrolRegions() -} - -func waitOperator(re *require.Assertions, co *coordinator, regionID uint64) { - testutil.Eventually(re, func() bool { - return co.opController.GetOperator(regionID) != nil - }) -} - -func TestOperatorCount(t *testing.T) { - re := require.New(t) - - tc, co, cleanup := prepare(nil, nil, nil, re) - defer cleanup() - oc := co.opController - re.Equal(uint64(0), oc.OperatorCount(operator.OpLeader)) - re.Equal(uint64(0), oc.OperatorCount(operator.OpRegion)) - - re.NoError(tc.addLeaderRegion(1, 1)) - re.NoError(tc.addLeaderRegion(2, 2)) - { - op1 := newTestOperator(1, tc.GetRegion(1).GetRegionEpoch(), operator.OpLeader) - oc.AddWaitingOperator(op1) - re.Equal(uint64(1), oc.OperatorCount(operator.OpLeader)) // 1:leader - op2 := newTestOperator(2, tc.GetRegion(2).GetRegionEpoch(), operator.OpLeader) - oc.AddWaitingOperator(op2) - re.Equal(uint64(2), oc.OperatorCount(operator.OpLeader)) // 1:leader, 2:leader - re.True(oc.RemoveOperator(op1)) - re.Equal(uint64(1), oc.OperatorCount(operator.OpLeader)) // 2:leader - } - - { - op1 := newTestOperator(1, tc.GetRegion(1).GetRegionEpoch(), operator.OpRegion) - oc.AddWaitingOperator(op1) - re.Equal(uint64(1), oc.OperatorCount(operator.OpRegion)) // 1:region 2:leader - re.Equal(uint64(1), oc.OperatorCount(operator.OpLeader)) - op2 := newTestOperator(2, tc.GetRegion(2).GetRegionEpoch(), operator.OpRegion) - op2.SetPriorityLevel(constant.High) - oc.AddWaitingOperator(op2) - re.Equal(uint64(2), oc.OperatorCount(operator.OpRegion)) // 1:region 2:region - re.Equal(uint64(0), oc.OperatorCount(operator.OpLeader)) - } -} - -func TestStoreOverloaded(t *testing.T) { - re := require.New(t) - - tc, co, cleanup := prepare(nil, nil, nil, re) - defer cleanup() - oc := co.opController - lb, err := schedule.CreateScheduler(schedulers.BalanceRegionType, oc, tc.storage, schedule.ConfigSliceDecoder(schedulers.BalanceRegionType, []string{"", ""})) - re.NoError(err) - opt := tc.GetOpts() - re.NoError(tc.addRegionStore(4, 100)) - re.NoError(tc.addRegionStore(3, 100)) - re.NoError(tc.addRegionStore(2, 100)) - re.NoError(tc.addRegionStore(1, 10)) - re.NoError(tc.addLeaderRegion(1, 2, 3, 4)) - region := tc.GetRegion(1).Clone(core.SetApproximateSize(60)) - tc.putRegion(region) - start := time.Now() - { - ops, _ := lb.Schedule(tc, false /* dryRun */) - re.Len(ops, 1) - op1 := ops[0] - re.NotNil(op1) - re.True(oc.AddOperator(op1)) - re.True(oc.RemoveOperator(op1)) - } - for { - time.Sleep(time.Millisecond * 10) - ops, _ := lb.Schedule(tc, false /* dryRun */) - if time.Since(start) > time.Second { - break - } - re.Empty(ops) - } - - // reset all stores' limit - // scheduling one time needs 1/10 seconds - opt.SetAllStoresLimit(storelimit.AddPeer, 600) - opt.SetAllStoresLimit(storelimit.RemovePeer, 600) - time.Sleep(time.Second) - for i := 0; i < 10; i++ { - ops, _ := lb.Schedule(tc, false /* dryRun */) - re.Len(ops, 1) - op := ops[0] - re.True(oc.AddOperator(op)) - re.True(oc.RemoveOperator(op)) - } - // sleep 1 seconds to make sure that the token is filled up - time.Sleep(time.Second) - for i := 0; i < 100; i++ { - ops, _ := lb.Schedule(tc, false /* dryRun */) - re.Greater(len(ops), 0) - } -} - -func TestStoreOverloadedWithReplace(t *testing.T) { - re := require.New(t) - - tc, co, cleanup := prepare(nil, nil, nil, re) - defer cleanup() - oc := co.opController - lb, err := schedule.CreateScheduler(schedulers.BalanceRegionType, oc, tc.storage, schedule.ConfigSliceDecoder(schedulers.BalanceRegionType, []string{"", ""})) - re.NoError(err) - - re.NoError(tc.addRegionStore(4, 100)) - re.NoError(tc.addRegionStore(3, 100)) - re.NoError(tc.addRegionStore(2, 100)) - re.NoError(tc.addRegionStore(1, 10)) - re.NoError(tc.addLeaderRegion(1, 2, 3, 4)) - re.NoError(tc.addLeaderRegion(2, 1, 3, 4)) - region := tc.GetRegion(1).Clone(core.SetApproximateSize(60)) - tc.putRegion(region) - region = tc.GetRegion(2).Clone(core.SetApproximateSize(60)) - tc.putRegion(region) - op1 := newTestOperator(1, tc.GetRegion(1).GetRegionEpoch(), operator.OpRegion, operator.AddPeer{ToStore: 1, PeerID: 1}) - re.True(oc.AddOperator(op1)) - op2 := newTestOperator(1, tc.GetRegion(1).GetRegionEpoch(), operator.OpRegion, operator.AddPeer{ToStore: 2, PeerID: 2}) - op2.SetPriorityLevel(constant.High) - re.True(oc.AddOperator(op2)) - op3 := newTestOperator(1, tc.GetRegion(2).GetRegionEpoch(), operator.OpRegion, operator.AddPeer{ToStore: 1, PeerID: 3}) - re.False(oc.AddOperator(op3)) - ops, _ := lb.Schedule(tc, false /* dryRun */) - re.Empty(ops) - // sleep 2 seconds to make sure that token is filled up - time.Sleep(2 * time.Second) - ops, _ = lb.Schedule(tc, false /* dryRun */) - re.Greater(len(ops), 0) -} - -func TestDownStoreLimit(t *testing.T) { - re := require.New(t) - - tc, co, cleanup := prepare(nil, nil, nil, re) - defer cleanup() - oc := co.opController - rc := co.checkers.GetRuleChecker() - - tc.addRegionStore(1, 100) - tc.addRegionStore(2, 100) - tc.addRegionStore(3, 100) - tc.addLeaderRegion(1, 1, 2, 3) - - region := tc.GetRegion(1) - tc.setStoreDown(1) - tc.SetStoreLimit(1, storelimit.RemovePeer, 1) - - region = region.Clone(core.WithDownPeers([]*pdpb.PeerStats{ - { - Peer: region.GetStorePeer(1), - DownSeconds: 24 * 60 * 60, - }, - }), core.SetApproximateSize(1)) - tc.putRegion(region) - for i := uint64(1); i < 20; i++ { - tc.addRegionStore(i+3, 100) - op := rc.Check(region) - re.NotNil(op) - re.True(oc.AddOperator(op)) - oc.RemoveOperator(op) - } - - region = region.Clone(core.SetApproximateSize(100)) - tc.putRegion(region) - for i := uint64(20); i < 25; i++ { - tc.addRegionStore(i+3, 100) - op := rc.Check(region) - re.NotNil(op) - re.True(oc.AddOperator(op)) - oc.RemoveOperator(op) - } -} - -// FIXME: remove after move into schedulers package -type mockLimitScheduler struct { - schedule.Scheduler - limit uint64 - counter *schedule.OperatorController - kind operator.OpKind -} - -func (s *mockLimitScheduler) IsScheduleAllowed(cluster schedule.Cluster) bool { - return s.counter.OperatorCount(s.kind) < s.limit -} - -func TestController(t *testing.T) { - re := require.New(t) - - tc, co, cleanup := prepare(nil, nil, nil, re) - defer cleanup() - oc := co.opController - - re.NoError(tc.addLeaderRegion(1, 1)) - re.NoError(tc.addLeaderRegion(2, 2)) - scheduler, err := schedule.CreateScheduler(schedulers.BalanceLeaderType, oc, storage.NewStorageWithMemoryBackend(), schedule.ConfigSliceDecoder(schedulers.BalanceLeaderType, []string{"", ""})) - re.NoError(err) - lb := &mockLimitScheduler{ - Scheduler: scheduler, - counter: oc, - kind: operator.OpLeader, - } - - sc := newScheduleController(co, lb) - - for i := schedulers.MinScheduleInterval; sc.GetInterval() != schedulers.MaxScheduleInterval; i = sc.GetNextInterval(i) { - re.Equal(i, sc.GetInterval()) - re.Empty(sc.Schedule(false)) - } - // limit = 2 - lb.limit = 2 - // count = 0 - { - re.True(sc.AllowSchedule(false)) - op1 := newTestOperator(1, tc.GetRegion(1).GetRegionEpoch(), operator.OpLeader) - re.Equal(1, oc.AddWaitingOperator(op1)) - // count = 1 - re.True(sc.AllowSchedule(false)) - op2 := newTestOperator(2, tc.GetRegion(2).GetRegionEpoch(), operator.OpLeader) - re.Equal(1, oc.AddWaitingOperator(op2)) - // count = 2 - re.False(sc.AllowSchedule(false)) - re.True(oc.RemoveOperator(op1)) - // count = 1 - re.True(sc.AllowSchedule(false)) - } - - op11 := newTestOperator(1, tc.GetRegion(1).GetRegionEpoch(), operator.OpLeader) - // add a PriorityKind operator will remove old operator - { - op3 := newTestOperator(2, tc.GetRegion(2).GetRegionEpoch(), operator.OpHotRegion) - op3.SetPriorityLevel(constant.High) - re.Equal(1, oc.AddWaitingOperator(op11)) - re.False(sc.AllowSchedule(false)) - re.Equal(1, oc.AddWaitingOperator(op3)) - re.True(sc.AllowSchedule(false)) - re.True(oc.RemoveOperator(op3)) - } - - // add a admin operator will remove old operator - { - op2 := newTestOperator(2, tc.GetRegion(2).GetRegionEpoch(), operator.OpLeader) - re.Equal(1, oc.AddWaitingOperator(op2)) - re.False(sc.AllowSchedule(false)) - op4 := newTestOperator(2, tc.GetRegion(2).GetRegionEpoch(), operator.OpAdmin) - op4.SetPriorityLevel(constant.High) - re.Equal(1, oc.AddWaitingOperator(op4)) - re.True(sc.AllowSchedule(false)) - re.True(oc.RemoveOperator(op4)) - } - - // test wrong region id. - { - op5 := newTestOperator(3, &metapb.RegionEpoch{}, operator.OpHotRegion) - re.Equal(0, oc.AddWaitingOperator(op5)) - } - - // test wrong region epoch. - re.True(oc.RemoveOperator(op11)) - epoch := &metapb.RegionEpoch{ - Version: tc.GetRegion(1).GetRegionEpoch().GetVersion() + 1, - ConfVer: tc.GetRegion(1).GetRegionEpoch().GetConfVer(), - } - { - op6 := newTestOperator(1, epoch, operator.OpLeader) - re.Equal(0, oc.AddWaitingOperator(op6)) - } - epoch.Version-- - { - op6 := newTestOperator(1, epoch, operator.OpLeader) - re.Equal(1, oc.AddWaitingOperator(op6)) - re.True(oc.RemoveOperator(op6)) - } -} - -func TestInterval(t *testing.T) { - re := require.New(t) - - _, co, cleanup := prepare(nil, nil, nil, re) - defer cleanup() - - lb, err := schedule.CreateScheduler(schedulers.BalanceLeaderType, co.opController, storage.NewStorageWithMemoryBackend(), schedule.ConfigSliceDecoder(schedulers.BalanceLeaderType, []string{"", ""})) - re.NoError(err) - sc := newScheduleController(co, lb) - - // If no operator for x seconds, the next check should be in x/2 seconds. - idleSeconds := []int{5, 10, 20, 30, 60} - for _, n := range idleSeconds { - sc.nextInterval = schedulers.MinScheduleInterval - for totalSleep := time.Duration(0); totalSleep <= time.Second*time.Duration(n); totalSleep += sc.GetInterval() { - re.Empty(sc.Schedule(false)) - } - re.Less(sc.GetInterval(), time.Second*time.Duration(n/2)) - } -} - -func waitAddLearner(re *require.Assertions, stream mockhbstream.HeartbeatStream, region *core.RegionInfo, storeID uint64) *core.RegionInfo { - var res *pdpb.RegionHeartbeatResponse - testutil.Eventually(re, func() bool { - if res = stream.Recv(); res != nil { - return res.GetRegionId() == region.GetID() && - res.GetChangePeer().GetChangeType() == eraftpb.ConfChangeType_AddLearnerNode && - res.GetChangePeer().GetPeer().GetStoreId() == storeID - } - return false - }) - return region.Clone( - core.WithAddPeer(res.GetChangePeer().GetPeer()), - core.WithIncConfVer(), - ) -} - -func waitPromoteLearner(re *require.Assertions, stream mockhbstream.HeartbeatStream, region *core.RegionInfo, storeID uint64) *core.RegionInfo { - var res *pdpb.RegionHeartbeatResponse - testutil.Eventually(re, func() bool { - if res = stream.Recv(); res != nil { - return res.GetRegionId() == region.GetID() && - res.GetChangePeer().GetChangeType() == eraftpb.ConfChangeType_AddNode && - res.GetChangePeer().GetPeer().GetStoreId() == storeID - } - return false - }) - // Remove learner than add voter. - return region.Clone( - core.WithRemoveStorePeer(storeID), - core.WithAddPeer(res.GetChangePeer().GetPeer()), - ) -} - -func waitRemovePeer(re *require.Assertions, stream mockhbstream.HeartbeatStream, region *core.RegionInfo, storeID uint64) *core.RegionInfo { - var res *pdpb.RegionHeartbeatResponse - testutil.Eventually(re, func() bool { - if res = stream.Recv(); res != nil { - return res.GetRegionId() == region.GetID() && - res.GetChangePeer().GetChangeType() == eraftpb.ConfChangeType_RemoveNode && - res.GetChangePeer().GetPeer().GetStoreId() == storeID - } - return false - }) - return region.Clone( - core.WithRemoveStorePeer(storeID), - core.WithIncConfVer(), - ) -} - -func waitTransferLeader(re *require.Assertions, stream mockhbstream.HeartbeatStream, region *core.RegionInfo, storeID uint64) *core.RegionInfo { - var res *pdpb.RegionHeartbeatResponse - testutil.Eventually(re, func() bool { - if res = stream.Recv(); res != nil { - if res.GetRegionId() == region.GetID() { - for _, peer := range append(res.GetTransferLeader().GetPeers(), res.GetTransferLeader().GetPeer()) { - if peer.GetStoreId() == storeID { - return true - } - } - } - } - return false - }) - return region.Clone( - core.WithLeader(region.GetStorePeer(storeID)), - ) -} - -func waitNoResponse(re *require.Assertions, stream mockhbstream.HeartbeatStream) { - testutil.Eventually(re, func() bool { - res := stream.Recv() - return res == nil - }) -} diff --git a/server/cluster/metrics.go b/server/cluster/metrics.go index 8c0bceb94ca..e43fe595f70 100644 --- a/server/cluster/metrics.go +++ b/server/cluster/metrics.go @@ -41,38 +41,6 @@ var ( Help: "Counter of the bucket event", }, []string{"event"}) - schedulerStatusGauge = prometheus.NewGaugeVec( - prometheus.GaugeOpts{ - Namespace: "pd", - Subsystem: "scheduler", - Name: "status", - Help: "Status of the scheduler.", - }, []string{"kind", "type"}) - - hotSpotStatusGauge = prometheus.NewGaugeVec( - prometheus.GaugeOpts{ - Namespace: "pd", - Subsystem: "hotspot", - Name: "status", - Help: "Status of the hotspot.", - }, []string{"address", "store", "type"}) - - hotPendingSum = prometheus.NewGaugeVec( - prometheus.GaugeOpts{ - Namespace: "pd", - Subsystem: "scheduler", - Name: "hot_pending_sum", - Help: "Pending influence sum of store in hot region scheduler.", - }, []string{"store", "rw", "dim"}) - - patrolCheckRegionsGauge = prometheus.NewGauge( - prometheus.GaugeOpts{ - Namespace: "pd", - Subsystem: "checker", - Name: "patrol_regions_time", - Help: "Time spent of patrol checks region.", - }) - updateStoreStatsGauge = prometheus.NewGauge( prometheus.GaugeOpts{ Namespace: "pd", @@ -96,14 +64,6 @@ var ( Help: "Current state of the cluster", }, []string{"state"}) - regionListGauge = prometheus.NewGaugeVec( - prometheus.GaugeOpts{ - Namespace: "pd", - Subsystem: "checker", - Name: "region_list", - Help: "Number of region in waiting list", - }, []string{"type"}) - storesProgressGauge = prometheus.NewGaugeVec( prometheus.GaugeOpts{ Namespace: "pd", @@ -140,12 +100,8 @@ var ( func init() { prometheus.MustRegister(regionEventCounter) prometheus.MustRegister(healthStatusGauge) - prometheus.MustRegister(schedulerStatusGauge) - prometheus.MustRegister(hotSpotStatusGauge) - prometheus.MustRegister(patrolCheckRegionsGauge) prometheus.MustRegister(clusterStateCPUGauge) prometheus.MustRegister(clusterStateCurrent) - prometheus.MustRegister(regionListGauge) prometheus.MustRegister(bucketEventCounter) prometheus.MustRegister(storesProgressGauge) prometheus.MustRegister(storesSpeedGauge) diff --git a/server/cluster/store_limiter.go b/server/cluster/store_limiter.go index f4b317ffe22..4e77590ddc5 100644 --- a/server/cluster/store_limiter.go +++ b/server/cluster/store_limiter.go @@ -15,12 +15,9 @@ package cluster import ( - "github.com/pingcap/kvproto/pkg/pdpb" - "github.com/pingcap/log" "github.com/tikv/pd/pkg/core/storelimit" "github.com/tikv/pd/pkg/utils/syncutil" "github.com/tikv/pd/server/config" - "go.uber.org/zap" ) // StoreLimiter adjust the store limit dynamically @@ -47,57 +44,6 @@ func NewStoreLimiter(opt *config.PersistOptions) *StoreLimiter { } } -// Collect the store statistics and update the cluster state -func (s *StoreLimiter) Collect(stats *pdpb.StoreStats) { - s.m.Lock() - defer s.m.Unlock() - - log.Debug("collected statistics", zap.Reflect("stats", stats)) - s.state.Collect((*StatEntry)(stats)) - - state := s.state.State() - ratePeerAdd := s.calculateRate(storelimit.AddPeer, state) - ratePeerRemove := s.calculateRate(storelimit.RemovePeer, state) - - if ratePeerAdd > 0 || ratePeerRemove > 0 { - if ratePeerAdd > 0 { - s.opt.SetAllStoresLimit(storelimit.AddPeer, ratePeerAdd) - log.Info("change store region add limit for cluster", zap.Stringer("state", state), zap.Float64("rate", ratePeerAdd)) - } - if ratePeerRemove > 0 { - s.opt.SetAllStoresLimit(storelimit.RemovePeer, ratePeerRemove) - log.Info("change store region remove limit for cluster", zap.Stringer("state", state), zap.Float64("rate", ratePeerRemove)) - } - s.current = state - collectClusterStateCurrent(state) - } -} - -func collectClusterStateCurrent(state LoadState) { - for i := LoadStateNone; i <= LoadStateHigh; i++ { - if i == state { - clusterStateCurrent.WithLabelValues(state.String()).Set(1) - continue - } - clusterStateCurrent.WithLabelValues(i.String()).Set(0) - } -} - -func (s *StoreLimiter) calculateRate(limitType storelimit.Type, state LoadState) float64 { - rate := float64(0) - switch state { - case LoadStateIdle: - rate = float64(s.scene[limitType].Idle) - case LoadStateLow: - rate = float64(s.scene[limitType].Low) - case LoadStateNormal: - rate = float64(s.scene[limitType].Normal) - case LoadStateHigh: - rate = float64(s.scene[limitType].High) - } - return rate -} - // ReplaceStoreLimitScene replaces the store limit values for different scenes func (s *StoreLimiter) ReplaceStoreLimitScene(scene *storelimit.Scene, limitType storelimit.Type) { s.m.Lock() diff --git a/server/cluster/store_limiter_test.go b/server/cluster/store_limiter_test.go index ab5fd3d88bd..5d03d7cb0f1 100644 --- a/server/cluster/store_limiter_test.go +++ b/server/cluster/store_limiter_test.go @@ -17,21 +17,11 @@ package cluster import ( "testing" - "github.com/pingcap/kvproto/pkg/pdpb" "github.com/stretchr/testify/require" "github.com/tikv/pd/pkg/core/storelimit" "github.com/tikv/pd/pkg/mock/mockconfig" ) -func TestCollect(t *testing.T) { - re := require.New(t) - - limiter := NewStoreLimiter(mockconfig.NewTestOptions()) - - limiter.Collect(&pdpb.StoreStats{}) - re.Equal(int64(1), limiter.state.cst.total) -} - func TestStoreLimitScene(t *testing.T) { re := require.New(t) diff --git a/server/config/config.go b/server/config/config.go index 6c936fcab6e..5b9088ac8ea 100644 --- a/server/config/config.go +++ b/server/config/config.go @@ -25,23 +25,20 @@ import ( "strings" "time" + "github.com/BurntSushi/toml" + "github.com/coreos/go-semver/semver" "github.com/docker/go-units" + "github.com/pingcap/errors" + "github.com/pingcap/log" "github.com/spf13/pflag" - "github.com/tikv/pd/pkg/core/storelimit" "github.com/tikv/pd/pkg/errs" - rm "github.com/tikv/pd/pkg/mcs/resource_manager/server" + rm "github.com/tikv/pd/pkg/mcs/resourcemanager/server" + sc "github.com/tikv/pd/pkg/schedule/config" "github.com/tikv/pd/pkg/utils/configutil" "github.com/tikv/pd/pkg/utils/grpcutil" "github.com/tikv/pd/pkg/utils/metricutil" - "github.com/tikv/pd/pkg/utils/syncutil" "github.com/tikv/pd/pkg/utils/typeutil" "github.com/tikv/pd/pkg/versioninfo" - - "github.com/BurntSushi/toml" - "github.com/coreos/go-semver/semver" - "github.com/pingcap/errors" - "github.com/pingcap/kvproto/pkg/metapb" - "github.com/pingcap/log" "go.etcd.io/etcd/embed" "go.etcd.io/etcd/pkg/transport" "go.uber.org/zap" @@ -80,6 +77,15 @@ type Config struct { LogFileDeprecated string `toml:"log-file" json:"log-file,omitempty"` LogLevelDeprecated string `toml:"log-level" json:"log-level,omitempty"` + // MaxConcurrentTSOProxyStreamings is the maximum number of concurrent TSO proxy streaming process routines allowed. + // Exceeding this limit will result in an error being returned to the client when a new client starts a TSO streaming. + // Set this to 0 will disable TSO Proxy. + // Set this to the negative value to disable the limit. + MaxConcurrentTSOProxyStreamings int `toml:"max-concurrent-tso-proxy-streamings" json:"max-concurrent-tso-proxy-streamings"` + // TSOProxyRecvFromClientTimeout is the timeout for the TSO proxy to receive a tso request from a client via grpc TSO stream. + // After the timeout, the TSO proxy will close the grpc TSO stream. + TSOProxyRecvFromClientTimeout typeutil.Duration `toml:"tso-proxy-recv-from-client-timeout" json:"tso-proxy-recv-from-client-timeout"` + // TSOSaveInterval is the interval to save timestamp. TSOSaveInterval typeutil.Duration `toml:"tso-save-interval" json:"tso-save-interval"` @@ -98,9 +104,9 @@ type Config struct { Metric metricutil.MetricConfig `toml:"metric" json:"metric"` - Schedule ScheduleConfig `toml:"schedule" json:"schedule"` + Schedule sc.ScheduleConfig `toml:"schedule" json:"schedule"` - Replication ReplicationConfig `toml:"replication" json:"replication"` + Replication sc.ReplicationConfig `toml:"replication" json:"replication"` PDServerCfg PDServerConfig `toml:"pd-server" json:"pd-server"` @@ -159,7 +165,7 @@ type Config struct { Keyspace KeyspaceConfig `toml:"keyspace" json:"keyspace"` - RequestUnit rm.RequestUnitConfig `toml:"request-unit" json:"request-unit"` + Controller rm.ControllerConfig `toml:"controller" json:"controller"` } // NewConfig creates a new config. @@ -208,16 +214,18 @@ const ( // DefaultMinResolvedTSPersistenceInterval is the default value of min resolved ts persistent interval. DefaultMinResolvedTSPersistenceInterval = time.Second - defaultStrictlyMatchLabel = false - defaultEnablePlacementRules = true - defaultEnableGRPCGateway = true - defaultDisableErrorVerbose = true - defaultEnableWitness = false + defaultEnableGRPCGateway = true + defaultDisableErrorVerbose = true + defaultEnableWitness = false + defaultHaltScheduling = false defaultDashboardAddress = "auto" defaultDRWaitStoreTimeout = time.Minute + defaultMaxConcurrentTSOProxyStreamings = 5000 + defaultTSOProxyRecvFromClientTimeout = 1 * time.Hour + defaultTSOSaveInterval = time.Duration(defaultLeaderLease) * time.Second // defaultTSOUpdatePhysicalInterval is the default value of the config `TSOUpdatePhysicalInterval`. defaultTSOUpdatePhysicalInterval = 50 * time.Millisecond @@ -226,8 +234,6 @@ const ( defaultLogFormat = "text" - defaultMaxMovableHotPeerSize = int64(512) - defaultServerMemoryLimit = 0 minServerMemoryLimit = 0 maxServerMemoryLimit = 0.99 @@ -238,6 +244,11 @@ const ( defaultGCTunerThreshold = 0.6 minGCTunerThreshold = 0 maxGCTunerThreshold = 0.9 + + defaultWaitRegionSplitTimeout = 30 * time.Second + defaultCheckRegionSplitInterval = 50 * time.Millisecond + minCheckRegionSplitInterval = 1 * time.Millisecond + maxCheckRegionSplitInterval = 100 * time.Millisecond ) // Special keys for Labels @@ -249,11 +260,6 @@ const ( var ( defaultEnableTelemetry = false defaultRuntimeServices = []string{} - defaultLocationLabels = []string{} - // DefaultStoreLimit is the default store limit of add peer and remove peer. - DefaultStoreLimit = StoreLimit{AddPeer: 15, RemovePeer: 15} - // DefaultTiFlashStoreLimit is the default TiFlash store limit of add peer and remove peer. - DefaultTiFlashStoreLimit = StoreLimit{AddPeer: 30, RemovePeer: 30} ) func init() { @@ -266,50 +272,6 @@ func initByLDFlags(edition string) { } } -// StoreLimit is the default limit of adding peer and removing peer when putting stores. -type StoreLimit struct { - mu syncutil.RWMutex - // AddPeer is the default rate of adding peers for store limit (per minute). - AddPeer float64 - // RemovePeer is the default rate of removing peers for store limit (per minute). - RemovePeer float64 -} - -// SetDefaultStoreLimit sets the default store limit for a given type. -func (sl *StoreLimit) SetDefaultStoreLimit(typ storelimit.Type, ratePerMin float64) { - sl.mu.Lock() - defer sl.mu.Unlock() - switch typ { - case storelimit.AddPeer: - sl.AddPeer = ratePerMin - case storelimit.RemovePeer: - sl.RemovePeer = ratePerMin - } -} - -// GetDefaultStoreLimit gets the default store limit for a given type. -func (sl *StoreLimit) GetDefaultStoreLimit(typ storelimit.Type) float64 { - sl.mu.RLock() - defer sl.mu.RUnlock() - switch typ { - case storelimit.AddPeer: - return sl.AddPeer - case storelimit.RemovePeer: - return sl.RemovePeer - default: - panic("invalid type") - } -} - -func adjustSchedulers(v *SchedulerConfigs, defValue SchedulerConfigs) { - if len(*v) == 0 { - // Make a copy to avoid changing DefaultSchedulers unexpectedly. - // When reloading from storage, the config is passed to json.Unmarshal. - // Without clone, the DefaultSchedulers could be overwritten. - *v = append(defValue[:0:0], defValue...) - } -} - // Parse parses flag definitions from the argument list. func (c *Config) Parse(flagSet *pflag.FlagSet) error { // Load config file if specified. @@ -345,21 +307,21 @@ func (c *Config) Parse(flagSet *pflag.FlagSet) error { } // ignore the error check here - configutil.AdjustCommandlineString(flagSet, &c.Log.Level, "log-level") - configutil.AdjustCommandlineString(flagSet, &c.Log.File.Filename, "log-file") - configutil.AdjustCommandlineString(flagSet, &c.Name, "name") - configutil.AdjustCommandlineString(flagSet, &c.DataDir, "data-dir") - configutil.AdjustCommandlineString(flagSet, &c.ClientUrls, "client-urls") - configutil.AdjustCommandlineString(flagSet, &c.AdvertiseClientUrls, "advertise-client-urls") - configutil.AdjustCommandlineString(flagSet, &c.PeerUrls, "peer-urls") - configutil.AdjustCommandlineString(flagSet, &c.AdvertisePeerUrls, "advertise-peer-urls") - configutil.AdjustCommandlineString(flagSet, &c.InitialCluster, "initial-cluster") - configutil.AdjustCommandlineString(flagSet, &c.Join, "join") - configutil.AdjustCommandlineString(flagSet, &c.Metric.PushAddress, "metrics-addr") - configutil.AdjustCommandlineString(flagSet, &c.Security.CAPath, "cacert") - configutil.AdjustCommandlineString(flagSet, &c.Security.CertPath, "cert") - configutil.AdjustCommandlineString(flagSet, &c.Security.KeyPath, "key") - configutil.AdjustCommandlineBool(flagSet, &c.ForceNewCluster, "force-new-cluster") + configutil.AdjustCommandLineString(flagSet, &c.Log.Level, "log-level") + configutil.AdjustCommandLineString(flagSet, &c.Log.File.Filename, "log-file") + configutil.AdjustCommandLineString(flagSet, &c.Name, "name") + configutil.AdjustCommandLineString(flagSet, &c.DataDir, "data-dir") + configutil.AdjustCommandLineString(flagSet, &c.ClientUrls, "client-urls") + configutil.AdjustCommandLineString(flagSet, &c.AdvertiseClientUrls, "advertise-client-urls") + configutil.AdjustCommandLineString(flagSet, &c.PeerUrls, "peer-urls") + configutil.AdjustCommandLineString(flagSet, &c.AdvertisePeerUrls, "advertise-peer-urls") + configutil.AdjustCommandLineString(flagSet, &c.InitialCluster, "initial-cluster") + configutil.AdjustCommandLineString(flagSet, &c.Join, "join") + configutil.AdjustCommandLineString(flagSet, &c.Metric.PushAddress, "metrics-addr") + configutil.AdjustCommandLineString(flagSet, &c.Security.CAPath, "cacert") + configutil.AdjustCommandLineString(flagSet, &c.Security.CertPath, "cert") + configutil.AdjustCommandLineString(flagSet, &c.Security.KeyPath, "key") + configutil.AdjustCommandLineBool(flagSet, &c.ForceNewCluster, "force-new-cluster") return c.Adjust(meta, false) } @@ -436,10 +398,11 @@ func (c *Config) Adjust(meta *toml.MetaData, reloading bool) error { } } - configutil.AdjustInt64(&c.LeaderLease, defaultLeaderLease) + configutil.AdjustInt(&c.MaxConcurrentTSOProxyStreamings, defaultMaxConcurrentTSOProxyStreamings) + configutil.AdjustDuration(&c.TSOProxyRecvFromClientTimeout, defaultTSOProxyRecvFromClientTimeout) + configutil.AdjustInt64(&c.LeaderLease, defaultLeaderLease) configutil.AdjustDuration(&c.TSOSaveInterval, defaultTSOSaveInterval) - configutil.AdjustDuration(&c.TSOUpdatePhysicalInterval, defaultTSOUpdatePhysicalInterval) if c.TSOUpdatePhysicalInterval.Duration > maxTSOUpdatePhysicalInterval { @@ -469,10 +432,10 @@ func (c *Config) Adjust(meta *toml.MetaData, reloading bool) error { configutil.AdjustString(&c.Metric.PushJob, c.Name) - if err := c.Schedule.adjust(configMetaData.Child("schedule"), reloading); err != nil { + if err := c.Schedule.Adjust(configMetaData.Child("schedule"), reloading); err != nil { return err } - if err := c.Replication.adjust(configMetaData.Child("replication")); err != nil { + if err := c.Replication.Adjust(configMetaData.Child("replication")); err != nil { return err } @@ -496,13 +459,15 @@ func (c *Config) Adjust(meta *toml.MetaData, reloading bool) error { c.ReplicationMode.adjust(configMetaData.Child("replication-mode")) + c.Keyspace.adjust(configMetaData.Child("keyspace")) + c.Security.Encryption.Adjust() if len(c.Log.Format) == 0 { c.Log.Format = defaultLogFormat } - c.RequestUnit.Adjust() + c.Controller.Adjust(configMetaData.Child("controller")) return nil } @@ -527,522 +492,6 @@ func (c *Config) String() string { return string(data) } -// ScheduleConfig is the schedule configuration. -// NOTE: This type is exported by HTTP API. Please pay more attention when modifying it. -type ScheduleConfig struct { - // If the snapshot count of one store is greater than this value, - // it will never be used as a source or target store. - MaxSnapshotCount uint64 `toml:"max-snapshot-count" json:"max-snapshot-count"` - MaxPendingPeerCount uint64 `toml:"max-pending-peer-count" json:"max-pending-peer-count"` - // If both the size of region is smaller than MaxMergeRegionSize - // and the number of rows in region is smaller than MaxMergeRegionKeys, - // it will try to merge with adjacent regions. - MaxMergeRegionSize uint64 `toml:"max-merge-region-size" json:"max-merge-region-size"` - MaxMergeRegionKeys uint64 `toml:"max-merge-region-keys" json:"max-merge-region-keys"` - // SplitMergeInterval is the minimum interval time to permit merge after split. - SplitMergeInterval typeutil.Duration `toml:"split-merge-interval" json:"split-merge-interval"` - // SwitchWitnessInterval is the minimum interval that allows a peer to become a witness again after it is promoted to non-witness. - SwitchWitnessInterval typeutil.Duration `toml:"switch-witness-interval" json:"swtich-witness-interval"` - // EnableOneWayMerge is the option to enable one way merge. This means a Region can only be merged into the next region of it. - EnableOneWayMerge bool `toml:"enable-one-way-merge" json:"enable-one-way-merge,string"` - // EnableCrossTableMerge is the option to enable cross table merge. This means two Regions can be merged with different table IDs. - // This option only works when key type is "table". - EnableCrossTableMerge bool `toml:"enable-cross-table-merge" json:"enable-cross-table-merge,string"` - // PatrolRegionInterval is the interval for scanning region during patrol. - PatrolRegionInterval typeutil.Duration `toml:"patrol-region-interval" json:"patrol-region-interval"` - // MaxStoreDownTime is the max duration after which - // a store will be considered to be down if it hasn't reported heartbeats. - MaxStoreDownTime typeutil.Duration `toml:"max-store-down-time" json:"max-store-down-time"` - // MaxStorePreparingTime is the max duration after which - // a store will be considered to be preparing. - MaxStorePreparingTime typeutil.Duration `toml:"max-store-preparing-time" json:"max-store-preparing-time"` - // LeaderScheduleLimit is the max coexist leader schedules. - LeaderScheduleLimit uint64 `toml:"leader-schedule-limit" json:"leader-schedule-limit"` - // LeaderSchedulePolicy is the option to balance leader, there are some policies supported: ["count", "size"], default: "count" - LeaderSchedulePolicy string `toml:"leader-schedule-policy" json:"leader-schedule-policy"` - // RegionScheduleLimit is the max coexist region schedules. - RegionScheduleLimit uint64 `toml:"region-schedule-limit" json:"region-schedule-limit"` - // WitnessScheduleLimit is the max coexist witness schedules. - WitnessScheduleLimit uint64 `toml:"witness-schedule-limit" json:"witness-schedule-limit"` - // ReplicaScheduleLimit is the max coexist replica schedules. - ReplicaScheduleLimit uint64 `toml:"replica-schedule-limit" json:"replica-schedule-limit"` - // MergeScheduleLimit is the max coexist merge schedules. - MergeScheduleLimit uint64 `toml:"merge-schedule-limit" json:"merge-schedule-limit"` - // HotRegionScheduleLimit is the max coexist hot region schedules. - HotRegionScheduleLimit uint64 `toml:"hot-region-schedule-limit" json:"hot-region-schedule-limit"` - // HotRegionCacheHitThreshold is the cache hits threshold of the hot region. - // If the number of times a region hits the hot cache is greater than this - // threshold, it is considered a hot region. - HotRegionCacheHitsThreshold uint64 `toml:"hot-region-cache-hits-threshold" json:"hot-region-cache-hits-threshold"` - // StoreBalanceRate is the maximum of balance rate for each store. - // WARN: StoreBalanceRate is deprecated. - StoreBalanceRate float64 `toml:"store-balance-rate" json:"store-balance-rate,omitempty"` - // StoreLimit is the limit of scheduling for stores. - StoreLimit map[uint64]StoreLimitConfig `toml:"store-limit" json:"store-limit"` - // TolerantSizeRatio is the ratio of buffer size for balance scheduler. - TolerantSizeRatio float64 `toml:"tolerant-size-ratio" json:"tolerant-size-ratio"` - // - // high space stage transition stage low space stage - // |--------------------|-----------------------------|-------------------------| - // ^ ^ ^ ^ - // 0 HighSpaceRatio * capacity LowSpaceRatio * capacity capacity - // - // LowSpaceRatio is the lowest usage ratio of store which regraded as low space. - // When in low space, store region score increases to very large and varies inversely with available size. - LowSpaceRatio float64 `toml:"low-space-ratio" json:"low-space-ratio"` - // HighSpaceRatio is the highest usage ratio of store which regraded as high space. - // High space means there is a lot of spare capacity, and store region score varies directly with used size. - HighSpaceRatio float64 `toml:"high-space-ratio" json:"high-space-ratio"` - // RegionScoreFormulaVersion is used to control the formula used to calculate region score. - RegionScoreFormulaVersion string `toml:"region-score-formula-version" json:"region-score-formula-version"` - // SchedulerMaxWaitingOperator is the max coexist operators for each scheduler. - SchedulerMaxWaitingOperator uint64 `toml:"scheduler-max-waiting-operator" json:"scheduler-max-waiting-operator"` - // WARN: DisableLearner is deprecated. - // DisableLearner is the option to disable using AddLearnerNode instead of AddNode. - DisableLearner bool `toml:"disable-raft-learner" json:"disable-raft-learner,string,omitempty"` - // DisableRemoveDownReplica is the option to prevent replica checker from - // removing down replicas. - // WARN: DisableRemoveDownReplica is deprecated. - DisableRemoveDownReplica bool `toml:"disable-remove-down-replica" json:"disable-remove-down-replica,string,omitempty"` - // DisableReplaceOfflineReplica is the option to prevent replica checker from - // replacing offline replicas. - // WARN: DisableReplaceOfflineReplica is deprecated. - DisableReplaceOfflineReplica bool `toml:"disable-replace-offline-replica" json:"disable-replace-offline-replica,string,omitempty"` - // DisableMakeUpReplica is the option to prevent replica checker from making up - // replicas when replica count is less than expected. - // WARN: DisableMakeUpReplica is deprecated. - DisableMakeUpReplica bool `toml:"disable-make-up-replica" json:"disable-make-up-replica,string,omitempty"` - // DisableRemoveExtraReplica is the option to prevent replica checker from - // removing extra replicas. - // WARN: DisableRemoveExtraReplica is deprecated. - DisableRemoveExtraReplica bool `toml:"disable-remove-extra-replica" json:"disable-remove-extra-replica,string,omitempty"` - // DisableLocationReplacement is the option to prevent replica checker from - // moving replica to a better location. - // WARN: DisableLocationReplacement is deprecated. - DisableLocationReplacement bool `toml:"disable-location-replacement" json:"disable-location-replacement,string,omitempty"` - - // EnableRemoveDownReplica is the option to enable replica checker to remove down replica. - EnableRemoveDownReplica bool `toml:"enable-remove-down-replica" json:"enable-remove-down-replica,string"` - // EnableReplaceOfflineReplica is the option to enable replica checker to replace offline replica. - EnableReplaceOfflineReplica bool `toml:"enable-replace-offline-replica" json:"enable-replace-offline-replica,string"` - // EnableMakeUpReplica is the option to enable replica checker to make up replica. - EnableMakeUpReplica bool `toml:"enable-make-up-replica" json:"enable-make-up-replica,string"` - // EnableRemoveExtraReplica is the option to enable replica checker to remove extra replica. - EnableRemoveExtraReplica bool `toml:"enable-remove-extra-replica" json:"enable-remove-extra-replica,string"` - // EnableLocationReplacement is the option to enable replica checker to move replica to a better location. - EnableLocationReplacement bool `toml:"enable-location-replacement" json:"enable-location-replacement,string"` - // EnableDebugMetrics is the option to enable debug metrics. - EnableDebugMetrics bool `toml:"enable-debug-metrics" json:"enable-debug-metrics,string"` - // EnableJointConsensus is the option to enable using joint consensus as a operator step. - EnableJointConsensus bool `toml:"enable-joint-consensus" json:"enable-joint-consensus,string"` - // EnableTiKVSplitRegion is the option to enable tikv split region. - // on ebs-based BR we need to disable it with TTL - EnableTiKVSplitRegion bool `toml:"enable-tikv-split-region" json:"enable-tikv-split-region,string"` - - // Schedulers support for loading customized schedulers - Schedulers SchedulerConfigs `toml:"schedulers" json:"schedulers-v2"` // json v2 is for the sake of compatible upgrade - - // Only used to display - SchedulersPayload map[string]interface{} `toml:"schedulers-payload" json:"schedulers-payload"` - - // StoreLimitMode can be auto or manual, when set to auto, - // PD tries to change the store limit values according to - // the load state of the cluster dynamically. User can - // overwrite the auto-tuned value by pd-ctl, when the value - // is overwritten, the value is fixed until it is deleted. - // Default: manual - StoreLimitMode string `toml:"store-limit-mode" json:"store-limit-mode"` - - // Controls the time interval between write hot regions info into leveldb. - HotRegionsWriteInterval typeutil.Duration `toml:"hot-regions-write-interval" json:"hot-regions-write-interval"` - - // The day of hot regions data to be reserved. 0 means close. - HotRegionsReservedDays uint64 `toml:"hot-regions-reserved-days" json:"hot-regions-reserved-days"` - - // MaxMovableHotPeerSize is the threshold of region size for balance hot region and split bucket scheduler. - // Hot region must be split before moved if it's region size is greater than MaxMovableHotPeerSize. - MaxMovableHotPeerSize int64 `toml:"max-movable-hot-peer-size" json:"max-movable-hot-peer-size,omitempty"` - - // EnableDiagnostic is the the option to enable using diagnostic - EnableDiagnostic bool `toml:"enable-diagnostic" json:"enable-diagnostic,string"` - - // EnableWitness is the option to enable using witness - EnableWitness bool `toml:"enable-witness" json:"enable-witness,string"` - - // SlowStoreEvictingAffectedStoreRatioThreshold is the affected ratio threshold when judging a store is slow - // A store's slowness must affected more than `store-count * SlowStoreEvictingAffectedStoreRatioThreshold` to trigger evicting. - SlowStoreEvictingAffectedStoreRatioThreshold float64 `toml:"slow-store-evicting-affected-store-ratio-threshold" json:"slow-store-evicting-affected-store-ratio-threshold,omitempty"` -} - -// Clone returns a cloned scheduling configuration. -func (c *ScheduleConfig) Clone() *ScheduleConfig { - schedulers := append(c.Schedulers[:0:0], c.Schedulers...) - var storeLimit map[uint64]StoreLimitConfig - if c.StoreLimit != nil { - storeLimit = make(map[uint64]StoreLimitConfig, len(c.StoreLimit)) - for k, v := range c.StoreLimit { - storeLimit[k] = v - } - } - cfg := *c - cfg.StoreLimit = storeLimit - cfg.Schedulers = schedulers - cfg.SchedulersPayload = nil - return &cfg -} - -const ( - defaultMaxReplicas = 3 - defaultMaxSnapshotCount = 64 - defaultMaxPendingPeerCount = 64 - defaultMaxMergeRegionSize = 20 - defaultSplitMergeInterval = time.Hour - defaultSwitchWitnessInterval = time.Hour - defaultEnableDiagnostic = false - defaultPatrolRegionInterval = 10 * time.Millisecond - defaultMaxStoreDownTime = 30 * time.Minute - defaultLeaderScheduleLimit = 4 - defaultRegionScheduleLimit = 2048 - defaultWitnessScheduleLimit = 4 - defaultReplicaScheduleLimit = 64 - defaultMergeScheduleLimit = 8 - defaultHotRegionScheduleLimit = 4 - defaultTolerantSizeRatio = 0 - defaultLowSpaceRatio = 0.8 - defaultHighSpaceRatio = 0.7 - defaultRegionScoreFormulaVersion = "v2" - // defaultHotRegionCacheHitsThreshold is the low hit number threshold of the - // hot region. - defaultHotRegionCacheHitsThreshold = 3 - defaultSchedulerMaxWaitingOperator = 5 - defaultLeaderSchedulePolicy = "count" - defaultStoreLimitMode = "manual" - defaultEnableJointConsensus = true - defaultEnableTiKVSplitRegion = true - defaultEnableCrossTableMerge = true - defaultHotRegionsWriteInterval = 10 * time.Minute - defaultHotRegionsReservedDays = 7 - // It means we skip the preparing stage after the 48 hours no matter if the store has finished preparing stage. - defaultMaxStorePreparingTime = 48 * time.Hour - // When a slow store affected more than 30% of total stores, it will trigger evicting. - defaultSlowStoreEvictingAffectedStoreRatioThreshold = 0.3 -) - -func (c *ScheduleConfig) adjust(meta *configutil.ConfigMetaData, reloading bool) error { - if !meta.IsDefined("max-snapshot-count") { - configutil.AdjustUint64(&c.MaxSnapshotCount, defaultMaxSnapshotCount) - } - if !meta.IsDefined("max-pending-peer-count") { - configutil.AdjustUint64(&c.MaxPendingPeerCount, defaultMaxPendingPeerCount) - } - if !meta.IsDefined("max-merge-region-size") { - configutil.AdjustUint64(&c.MaxMergeRegionSize, defaultMaxMergeRegionSize) - } - configutil.AdjustDuration(&c.SplitMergeInterval, defaultSplitMergeInterval) - configutil.AdjustDuration(&c.SwitchWitnessInterval, defaultSwitchWitnessInterval) - configutil.AdjustDuration(&c.PatrolRegionInterval, defaultPatrolRegionInterval) - configutil.AdjustDuration(&c.MaxStoreDownTime, defaultMaxStoreDownTime) - configutil.AdjustDuration(&c.HotRegionsWriteInterval, defaultHotRegionsWriteInterval) - configutil.AdjustDuration(&c.MaxStorePreparingTime, defaultMaxStorePreparingTime) - if !meta.IsDefined("leader-schedule-limit") { - configutil.AdjustUint64(&c.LeaderScheduleLimit, defaultLeaderScheduleLimit) - } - if !meta.IsDefined("region-schedule-limit") { - configutil.AdjustUint64(&c.RegionScheduleLimit, defaultRegionScheduleLimit) - } - if !meta.IsDefined("witness-schedule-limit") { - configutil.AdjustUint64(&c.WitnessScheduleLimit, defaultWitnessScheduleLimit) - } - if !meta.IsDefined("replica-schedule-limit") { - configutil.AdjustUint64(&c.ReplicaScheduleLimit, defaultReplicaScheduleLimit) - } - if !meta.IsDefined("merge-schedule-limit") { - configutil.AdjustUint64(&c.MergeScheduleLimit, defaultMergeScheduleLimit) - } - if !meta.IsDefined("hot-region-schedule-limit") { - configutil.AdjustUint64(&c.HotRegionScheduleLimit, defaultHotRegionScheduleLimit) - } - if !meta.IsDefined("hot-region-cache-hits-threshold") { - configutil.AdjustUint64(&c.HotRegionCacheHitsThreshold, defaultHotRegionCacheHitsThreshold) - } - if !meta.IsDefined("tolerant-size-ratio") { - configutil.AdjustFloat64(&c.TolerantSizeRatio, defaultTolerantSizeRatio) - } - if !meta.IsDefined("scheduler-max-waiting-operator") { - configutil.AdjustUint64(&c.SchedulerMaxWaitingOperator, defaultSchedulerMaxWaitingOperator) - } - if !meta.IsDefined("leader-schedule-policy") { - configutil.AdjustString(&c.LeaderSchedulePolicy, defaultLeaderSchedulePolicy) - } - if !meta.IsDefined("store-limit-mode") { - configutil.AdjustString(&c.StoreLimitMode, defaultStoreLimitMode) - } - if !meta.IsDefined("enable-joint-consensus") { - c.EnableJointConsensus = defaultEnableJointConsensus - } - if !meta.IsDefined("enable-tikv-split-region") { - c.EnableTiKVSplitRegion = defaultEnableTiKVSplitRegion - } - if !meta.IsDefined("enable-cross-table-merge") { - c.EnableCrossTableMerge = defaultEnableCrossTableMerge - } - configutil.AdjustFloat64(&c.LowSpaceRatio, defaultLowSpaceRatio) - configutil.AdjustFloat64(&c.HighSpaceRatio, defaultHighSpaceRatio) - if !meta.IsDefined("enable-diagnostic") { - c.EnableDiagnostic = defaultEnableDiagnostic - } - - if !meta.IsDefined("enable-witness") { - c.EnableWitness = defaultEnableWitness - } - - // new cluster:v2, old cluster:v1 - if !meta.IsDefined("region-score-formula-version") && !reloading { - configutil.AdjustString(&c.RegionScoreFormulaVersion, defaultRegionScoreFormulaVersion) - } - - adjustSchedulers(&c.Schedulers, DefaultSchedulers) - - for k, b := range c.migrateConfigurationMap() { - v, err := c.parseDeprecatedFlag(meta, k, *b[0], *b[1]) - if err != nil { - return err - } - *b[0], *b[1] = false, v // reset old flag false to make it ignored when marshal to JSON - } - - if c.StoreBalanceRate != 0 { - DefaultStoreLimit = StoreLimit{AddPeer: c.StoreBalanceRate, RemovePeer: c.StoreBalanceRate} - c.StoreBalanceRate = 0 - } - - if c.StoreLimit == nil { - c.StoreLimit = make(map[uint64]StoreLimitConfig) - } - - if !meta.IsDefined("hot-regions-reserved-days") { - configutil.AdjustUint64(&c.HotRegionsReservedDays, defaultHotRegionsReservedDays) - } - - if !meta.IsDefined("SlowStoreEvictingAffectedStoreRatioThreshold") { - configutil.AdjustFloat64(&c.SlowStoreEvictingAffectedStoreRatioThreshold, defaultSlowStoreEvictingAffectedStoreRatioThreshold) - } - return c.Validate() -} - -func (c *ScheduleConfig) migrateConfigurationMap() map[string][2]*bool { - return map[string][2]*bool{ - "remove-down-replica": {&c.DisableRemoveDownReplica, &c.EnableRemoveDownReplica}, - "replace-offline-replica": {&c.DisableReplaceOfflineReplica, &c.EnableReplaceOfflineReplica}, - "make-up-replica": {&c.DisableMakeUpReplica, &c.EnableMakeUpReplica}, - "remove-extra-replica": {&c.DisableRemoveExtraReplica, &c.EnableRemoveExtraReplica}, - "location-replacement": {&c.DisableLocationReplacement, &c.EnableLocationReplacement}, - } -} - -// GetMaxMergeRegionKeys returns the max merge keys. -// it should keep consistent with tikv: https://github.com/tikv/tikv/pull/12484 -func (c *ScheduleConfig) GetMaxMergeRegionKeys() uint64 { - if keys := c.MaxMergeRegionKeys; keys != 0 { - return keys - } - return c.MaxMergeRegionSize * 10000 -} - -func (c *ScheduleConfig) parseDeprecatedFlag(meta *configutil.ConfigMetaData, name string, old, new bool) (bool, error) { - oldName, newName := "disable-"+name, "enable-"+name - defineOld, defineNew := meta.IsDefined(oldName), meta.IsDefined(newName) - switch { - case defineNew && defineOld: - if new == old { - return false, errors.Errorf("config item %s and %s(deprecated) are conflict", newName, oldName) - } - return new, nil - case defineNew && !defineOld: - return new, nil - case !defineNew && defineOld: - return !old, nil // use !disable-* - case !defineNew && !defineOld: - return true, nil // use default value true - } - return false, nil // unreachable. -} - -// MigrateDeprecatedFlags updates new flags according to deprecated flags. -func (c *ScheduleConfig) MigrateDeprecatedFlags() { - c.DisableLearner = false - if c.StoreBalanceRate != 0 { - DefaultStoreLimit = StoreLimit{AddPeer: c.StoreBalanceRate, RemovePeer: c.StoreBalanceRate} - c.StoreBalanceRate = 0 - } - for _, b := range c.migrateConfigurationMap() { - // If old=false (previously disabled), set both old and new to false. - if *b[0] { - *b[0], *b[1] = false, false - } - } -} - -// Validate is used to validate if some scheduling configurations are right. -func (c *ScheduleConfig) Validate() error { - if c.TolerantSizeRatio < 0 { - return errors.New("tolerant-size-ratio should be non-negative") - } - if c.LowSpaceRatio < 0 || c.LowSpaceRatio > 1 { - return errors.New("low-space-ratio should between 0 and 1") - } - if c.HighSpaceRatio < 0 || c.HighSpaceRatio > 1 { - return errors.New("high-space-ratio should between 0 and 1") - } - if c.LowSpaceRatio <= c.HighSpaceRatio { - return errors.New("low-space-ratio should be larger than high-space-ratio") - } - if c.LeaderSchedulePolicy != "count" && c.LeaderSchedulePolicy != "size" { - return errors.Errorf("leader-schedule-policy %v is invalid", c.LeaderSchedulePolicy) - } - if c.SlowStoreEvictingAffectedStoreRatioThreshold == 0 { - return errors.Errorf("slow-store-evicting-affected-store-ratio-threshold is not set") - } - return nil -} - -// Deprecated is used to find if there is an option has been deprecated. -func (c *ScheduleConfig) Deprecated() error { - if c.DisableLearner { - return errors.New("disable-raft-learner has already been deprecated") - } - if c.DisableRemoveDownReplica { - return errors.New("disable-remove-down-replica has already been deprecated") - } - if c.DisableReplaceOfflineReplica { - return errors.New("disable-replace-offline-replica has already been deprecated") - } - if c.DisableMakeUpReplica { - return errors.New("disable-make-up-replica has already been deprecated") - } - if c.DisableRemoveExtraReplica { - return errors.New("disable-remove-extra-replica has already been deprecated") - } - if c.DisableLocationReplacement { - return errors.New("disable-location-replacement has already been deprecated") - } - if c.StoreBalanceRate != 0 { - return errors.New("store-balance-rate has already been deprecated") - } - return nil -} - -// StoreLimitConfig is a config about scheduling rate limit of different types for a store. -type StoreLimitConfig struct { - AddPeer float64 `toml:"add-peer" json:"add-peer"` - RemovePeer float64 `toml:"remove-peer" json:"remove-peer"` -} - -// SchedulerConfigs is a slice of customized scheduler configuration. -type SchedulerConfigs []SchedulerConfig - -// SchedulerConfig is customized scheduler configuration -type SchedulerConfig struct { - Type string `toml:"type" json:"type"` - Args []string `toml:"args" json:"args"` - Disable bool `toml:"disable" json:"disable"` - ArgsPayload string `toml:"args-payload" json:"args-payload"` -} - -// DefaultSchedulers are the schedulers be created by default. -// If these schedulers are not in the persistent configuration, they -// will be created automatically when reloading. -var DefaultSchedulers = SchedulerConfigs{ - {Type: "balance-region"}, - {Type: "balance-leader"}, - {Type: "balance-witness"}, - {Type: "hot-region"}, - {Type: "split-bucket"}, - {Type: "transfer-witness-leader"}, -} - -// IsDefaultScheduler checks whether the scheduler is enable by default. -func IsDefaultScheduler(typ string) bool { - for _, c := range DefaultSchedulers { - if typ == c.Type { - return true - } - } - return false -} - -// ReplicationConfig is the replication configuration. -// NOTE: This type is exported by HTTP API. Please pay more attention when modifying it. -type ReplicationConfig struct { - // MaxReplicas is the number of replicas for each region. - MaxReplicas uint64 `toml:"max-replicas" json:"max-replicas"` - - // The label keys specified the location of a store. - // The placement priorities is implied by the order of label keys. - // For example, ["zone", "rack"] means that we should place replicas to - // different zones first, then to different racks if we don't have enough zones. - LocationLabels typeutil.StringSlice `toml:"location-labels" json:"location-labels"` - // StrictlyMatchLabel strictly checks if the label of TiKV is matched with LocationLabels. - StrictlyMatchLabel bool `toml:"strictly-match-label" json:"strictly-match-label,string"` - - // When PlacementRules feature is enabled. MaxReplicas, LocationLabels and IsolationLabels are not used any more. - EnablePlacementRules bool `toml:"enable-placement-rules" json:"enable-placement-rules,string"` - - // EnablePlacementRuleCache controls whether use cache during rule checker - EnablePlacementRulesCache bool `toml:"enable-placement-rules-cache" json:"enable-placement-rules-cache,string"` - - // IsolationLevel is used to isolate replicas explicitly and forcibly if it's not empty. - // Its value must be empty or one of LocationLabels. - // Example: - // location-labels = ["zone", "rack", "host"] - // isolation-level = "zone" - // With configuration like above, PD ensure that all replicas be placed in different zones. - // Even if a zone is down, PD will not try to make up replicas in other zone - // because other zones already have replicas on it. - IsolationLevel string `toml:"isolation-level" json:"isolation-level"` -} - -// Clone makes a deep copy of the config. -func (c *ReplicationConfig) Clone() *ReplicationConfig { - locationLabels := append(c.LocationLabels[:0:0], c.LocationLabels...) - cfg := *c - cfg.LocationLabels = locationLabels - return &cfg -} - -// Validate is used to validate if some replication configurations are right. -func (c *ReplicationConfig) Validate() error { - foundIsolationLevel := false - for _, label := range c.LocationLabels { - err := ValidateLabels([]*metapb.StoreLabel{{Key: label}}) - if err != nil { - return err - } - // IsolationLevel should be empty or one of LocationLabels - if !foundIsolationLevel && label == c.IsolationLevel { - foundIsolationLevel = true - } - } - if c.IsolationLevel != "" && !foundIsolationLevel { - return errors.New("isolation-level must be one of location-labels or empty") - } - return nil -} - -func (c *ReplicationConfig) adjust(meta *configutil.ConfigMetaData) error { - configutil.AdjustUint64(&c.MaxReplicas, defaultMaxReplicas) - if !meta.IsDefined("enable-placement-rules") { - c.EnablePlacementRules = defaultEnablePlacementRules - } - if !meta.IsDefined("strictly-match-label") { - c.StrictlyMatchLabel = defaultStrictlyMatchLabel - } - if !meta.IsDefined("location-labels") { - c.LocationLabels = defaultLocationLabels - } - return c.Validate() -} - // PDServerConfig is the configuration for pd server. // NOTE: This type is exported by HTTP API. Please pay more attention when modifying it. type PDServerConfig struct { @@ -1053,7 +502,7 @@ type PDServerConfig struct { // KeyType is option to specify the type of keys. // There are some types supported: ["table", "raw", "txn"], default: "table" KeyType string `toml:"key-type" json:"key-type"` - // RuntimeServices is the running the running extension services. + // RuntimeServices is the running extension services. RuntimeServices typeutil.StringSlice `toml:"runtime-services" json:"runtime-services"` // MetricStorage is the cluster metric storage. // Currently we use prometheus as metric storage, we may use PD/TiKV as metric storage later. @@ -1071,10 +520,12 @@ type PDServerConfig struct { ServerMemoryLimit float64 `toml:"server-memory-limit" json:"server-memory-limit"` // ServerMemoryLimitGCTrigger indicates the gc percentage of the ServerMemoryLimit. ServerMemoryLimitGCTrigger float64 `toml:"server-memory-limit-gc-trigger" json:"server-memory-limit-gc-trigger"` - // EnableGOGCTuner is to enable GOGC tuner. it can tuner GOGC + // EnableGOGCTuner is to enable GOGC tuner. it can tuner GOGC. EnableGOGCTuner bool `toml:"enable-gogc-tuner" json:"enable-gogc-tuner,string"` // GCTunerThreshold is the threshold of GC tuner. GCTunerThreshold float64 `toml:"gc-tuner-threshold" json:"gc-tuner-threshold"` + // BlockSafePointV1 is used to control gc safe point v1 and service safe point v1 can not be updated. + BlockSafePointV1 bool `toml:"block-safe-point-v1" json:"block-safe-point-v1,string"` } func (c *PDServerConfig) adjust(meta *configutil.ConfigMetaData) error { @@ -1215,11 +666,27 @@ func (c LabelPropertyConfig) Clone() LabelPropertyConfig { return m } +// GetLeaderLease returns the leader lease. +func (c *Config) GetLeaderLease() int64 { + return c.LeaderLease +} + // IsLocalTSOEnabled returns if the local TSO is enabled. func (c *Config) IsLocalTSOEnabled() bool { return c.EnableLocalTSO } +// GetMaxConcurrentTSOProxyStreamings returns the max concurrent TSO proxy streamings. +// If the value is negative, there is no limit. +func (c *Config) GetMaxConcurrentTSOProxyStreamings() int { + return c.MaxConcurrentTSOProxyStreamings +} + +// GetTSOProxyRecvFromClientTimeout returns timeout value for TSO proxy receiving from the client. +func (c *Config) GetTSOProxyRecvFromClientTimeout() time.Duration { + return c.TSOProxyRecvFromClientTimeout.Duration +} + // GetTSOUpdatePhysicalInterval returns TSO update physical interval. func (c *Config) GetTSOUpdatePhysicalInterval() time.Duration { return c.TSOUpdatePhysicalInterval.Duration @@ -1383,4 +850,62 @@ func (c *DRAutoSyncReplicationConfig) adjust(meta *configutil.ConfigMetaData) { type KeyspaceConfig struct { // PreAlloc contains the keyspace to be allocated during keyspace manager initialization. PreAlloc []string `toml:"pre-alloc" json:"pre-alloc"` + // WaitRegionSplit indicates whether to wait for the region split to complete + WaitRegionSplit bool `toml:"wait-region-split" json:"wait-region-split"` + // WaitRegionSplitTimeout indicates the max duration to wait region split. + WaitRegionSplitTimeout typeutil.Duration `toml:"wait-region-split-timeout" json:"wait-region-split-timeout"` + // CheckRegionSplitInterval indicates the interval to check whether the region split is complete + CheckRegionSplitInterval typeutil.Duration `toml:"check-region-split-interval" json:"check-region-split-interval"` +} + +// Validate checks if keyspace config falls within acceptable range. +func (c *KeyspaceConfig) Validate() error { + if c.CheckRegionSplitInterval.Duration > maxCheckRegionSplitInterval || c.CheckRegionSplitInterval.Duration < minCheckRegionSplitInterval { + return errors.New(fmt.Sprintf("[keyspace] check-region-split-interval should between %v and %v", + minCheckRegionSplitInterval, maxCheckRegionSplitInterval)) + } + if c.CheckRegionSplitInterval.Duration >= c.WaitRegionSplitTimeout.Duration { + return errors.New("[keyspace] check-region-split-interval should be less than wait-region-split-timeout") + } + return nil +} + +func (c *KeyspaceConfig) adjust(meta *configutil.ConfigMetaData) { + if !meta.IsDefined("wait-region-split") { + c.WaitRegionSplit = true + } + if !meta.IsDefined("wait-region-split-timeout") { + c.WaitRegionSplitTimeout = typeutil.NewDuration(defaultWaitRegionSplitTimeout) + } + if !meta.IsDefined("check-region-split-interval") { + c.CheckRegionSplitInterval = typeutil.NewDuration(defaultCheckRegionSplitInterval) + } +} + +// Clone makes a deep copy of the keyspace config. +func (c *KeyspaceConfig) Clone() *KeyspaceConfig { + preAlloc := append(c.PreAlloc[:0:0], c.PreAlloc...) + cfg := *c + cfg.PreAlloc = preAlloc + return &cfg +} + +// GetPreAlloc returns the keyspace to be allocated during keyspace manager initialization. +func (c *KeyspaceConfig) GetPreAlloc() []string { + return c.PreAlloc +} + +// ToWaitRegionSplit returns whether to wait for the region split to complete. +func (c *KeyspaceConfig) ToWaitRegionSplit() bool { + return c.WaitRegionSplit +} + +// GetWaitRegionSplitTimeout returns the max duration to wait region split. +func (c *KeyspaceConfig) GetWaitRegionSplitTimeout() time.Duration { + return c.WaitRegionSplitTimeout.Duration +} + +// GetCheckRegionSplitInterval returns the interval to check whether the region split is complete. +func (c *KeyspaceConfig) GetCheckRegionSplitInterval() time.Duration { + return c.CheckRegionSplitInterval.Duration } diff --git a/server/config/config_test.go b/server/config/config_test.go index b8025a396b8..75e69c26d5c 100644 --- a/server/config/config_test.go +++ b/server/config/config_test.go @@ -26,6 +26,7 @@ import ( "github.com/BurntSushi/toml" "github.com/spf13/pflag" "github.com/stretchr/testify/require" + sc "github.com/tikv/pd/pkg/schedule/config" "github.com/tikv/pd/pkg/storage" "github.com/tikv/pd/pkg/utils/configutil" ) @@ -78,8 +79,8 @@ func TestReloadUpgrade(t *testing.T) { // Simulate an old configuration that only contains 2 fields. type OldConfig struct { - Schedule ScheduleConfig `toml:"schedule" json:"schedule"` - Replication ReplicationConfig `toml:"replication" json:"replication"` + Schedule sc.ScheduleConfig `toml:"schedule" json:"schedule"` + Replication sc.ReplicationConfig `toml:"replication" json:"replication"` } old := &OldConfig{ Schedule: *opt.GetScheduleConfig(), @@ -101,7 +102,7 @@ func TestReloadUpgrade2(t *testing.T) { // Simulate an old configuration that does not contain ScheduleConfig. type OldConfig struct { - Replication ReplicationConfig `toml:"replication" json:"replication"` + Replication sc.ReplicationConfig `toml:"replication" json:"replication"` } old := &OldConfig{ Replication: *opt.GetReplicationConfig(), @@ -453,12 +454,12 @@ func TestConfigClone(t *testing.T) { emptyConfigMetaData := configutil.NewConfigMetadata(nil) - schedule := &ScheduleConfig{} - schedule.adjust(emptyConfigMetaData, false) + schedule := &sc.ScheduleConfig{} + schedule.Adjust(emptyConfigMetaData, false) re.Equal(schedule, schedule.Clone()) - replication := &ReplicationConfig{} - replication.adjust(emptyConfigMetaData) + replication := &sc.ReplicationConfig{} + replication.Adjust(emptyConfigMetaData) re.Equal(replication, replication.Clone()) pdServer := &PDServerConfig{} diff --git a/server/config/metrics.go b/server/config/metrics.go new file mode 100644 index 00000000000..84f5c00dd61 --- /dev/null +++ b/server/config/metrics.go @@ -0,0 +1,29 @@ +// Copyright 2023 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import "github.com/prometheus/client_golang/prometheus" + +var schedulingAllowanceStatusGauge = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: "pd", + Subsystem: "scheduling", + Name: "allowance_status", + Help: "Status of the scheduling allowance.", + }, []string{"kind"}) + +func init() { + prometheus.MustRegister(schedulingAllowanceStatusGauge) +} diff --git a/server/config/persist_options.go b/server/config/persist_options.go index ef6e4b4ba9a..228351775e4 100644 --- a/server/config/persist_options.go +++ b/server/config/persist_options.go @@ -32,6 +32,7 @@ import ( "github.com/tikv/pd/pkg/cache" "github.com/tikv/pd/pkg/core/constant" "github.com/tikv/pd/pkg/core/storelimit" + sc "github.com/tikv/pd/pkg/schedule/config" "github.com/tikv/pd/pkg/slice" "github.com/tikv/pd/pkg/storage/endpoint" "github.com/tikv/pd/pkg/utils/etcdutil" @@ -50,6 +51,7 @@ type PersistOptions struct { pdServerConfig atomic.Value replicationMode atomic.Value labelProperty atomic.Value + keyspace atomic.Value clusterVersion unsafe.Pointer } @@ -61,28 +63,29 @@ func NewPersistOptions(cfg *Config) *PersistOptions { o.pdServerConfig.Store(&cfg.PDServerCfg) o.replicationMode.Store(&cfg.ReplicationMode) o.labelProperty.Store(cfg.LabelProperty) + o.keyspace.Store(&cfg.Keyspace) o.SetClusterVersion(&cfg.ClusterVersion) o.ttl = nil return o } // GetScheduleConfig returns scheduling configurations. -func (o *PersistOptions) GetScheduleConfig() *ScheduleConfig { - return o.schedule.Load().(*ScheduleConfig) +func (o *PersistOptions) GetScheduleConfig() *sc.ScheduleConfig { + return o.schedule.Load().(*sc.ScheduleConfig) } // SetScheduleConfig sets the PD scheduling configuration. -func (o *PersistOptions) SetScheduleConfig(cfg *ScheduleConfig) { +func (o *PersistOptions) SetScheduleConfig(cfg *sc.ScheduleConfig) { o.schedule.Store(cfg) } // GetReplicationConfig returns replication configurations. -func (o *PersistOptions) GetReplicationConfig() *ReplicationConfig { - return o.replication.Load().(*ReplicationConfig) +func (o *PersistOptions) GetReplicationConfig() *sc.ReplicationConfig { + return o.replication.Load().(*sc.ReplicationConfig) } // SetReplicationConfig sets the PD replication configuration. -func (o *PersistOptions) SetReplicationConfig(cfg *ReplicationConfig) { +func (o *PersistOptions) SetReplicationConfig(cfg *sc.ReplicationConfig) { o.replication.Store(cfg) } @@ -116,6 +119,16 @@ func (o *PersistOptions) SetLabelPropertyConfig(cfg LabelPropertyConfig) { o.labelProperty.Store(cfg) } +// GetKeyspaceConfig returns the keyspace config. +func (o *PersistOptions) GetKeyspaceConfig() *KeyspaceConfig { + return o.keyspace.Load().(*KeyspaceConfig) +} + +// SetKeyspaceConfig sets the keyspace configuration. +func (o *PersistOptions) SetKeyspaceConfig(cfg *KeyspaceConfig) { + o.keyspace.Store(cfg) +} + // GetClusterVersion returns the cluster version. func (o *PersistOptions) GetClusterVersion() *semver.Version { return (*semver.Version)(atomic.LoadPointer(&o.clusterVersion)) @@ -172,13 +185,6 @@ func (o *PersistOptions) SetPlacementRulesCacheEnabled(enabled bool) { o.SetReplicationConfig(v) } -// SetWitnessEnabled set EanbleWitness -func (o *PersistOptions) SetWitnessEnabled(enabled bool) { - v := o.GetScheduleConfig().Clone() - v.EnableWitness = enabled - o.SetScheduleConfig(v) -} - // GetStrictlyMatchLabel returns whether check label strict. func (o *PersistOptions) GetStrictlyMatchLabel() bool { return o.GetReplicationConfig().StrictlyMatchLabel @@ -196,6 +202,9 @@ func (o *PersistOptions) SetMaxReplicas(replicas int) { o.SetReplicationConfig(v) } +// UseRaftV2 set some config for raft store v2 by default temporary. +func (o *PersistOptions) UseRaftV2() {} + const ( maxSnapshotCountKey = "schedule.max-snapshot-count" maxMergeRegionSizeKey = "schedule.max-merge-region-size" @@ -327,25 +336,25 @@ func (o *PersistOptions) SetMaxMergeRegionKeys(maxMergeRegionKeys uint64) { // SetStoreLimit sets a store limit for a given type and rate. func (o *PersistOptions) SetStoreLimit(storeID uint64, typ storelimit.Type, ratePerMin float64) { v := o.GetScheduleConfig().Clone() - var sc StoreLimitConfig + var slc sc.StoreLimitConfig var rate float64 switch typ { case storelimit.AddPeer: if _, ok := v.StoreLimit[storeID]; !ok { - rate = DefaultStoreLimit.GetDefaultStoreLimit(storelimit.RemovePeer) + rate = sc.DefaultStoreLimit.GetDefaultStoreLimit(storelimit.RemovePeer) } else { rate = v.StoreLimit[storeID].RemovePeer } - sc = StoreLimitConfig{AddPeer: ratePerMin, RemovePeer: rate} + slc = sc.StoreLimitConfig{AddPeer: ratePerMin, RemovePeer: rate} case storelimit.RemovePeer: if _, ok := v.StoreLimit[storeID]; !ok { - rate = DefaultStoreLimit.GetDefaultStoreLimit(storelimit.AddPeer) + rate = sc.DefaultStoreLimit.GetDefaultStoreLimit(storelimit.AddPeer) } else { rate = v.StoreLimit[storeID].AddPeer } - sc = StoreLimitConfig{AddPeer: rate, RemovePeer: ratePerMin} + slc = sc.StoreLimitConfig{AddPeer: rate, RemovePeer: ratePerMin} } - v.StoreLimit[storeID] = sc + v.StoreLimit[storeID] = slc o.SetScheduleConfig(v) } @@ -354,15 +363,15 @@ func (o *PersistOptions) SetAllStoresLimit(typ storelimit.Type, ratePerMin float v := o.GetScheduleConfig().Clone() switch typ { case storelimit.AddPeer: - DefaultStoreLimit.SetDefaultStoreLimit(storelimit.AddPeer, ratePerMin) + sc.DefaultStoreLimit.SetDefaultStoreLimit(storelimit.AddPeer, ratePerMin) for storeID := range v.StoreLimit { - sc := StoreLimitConfig{AddPeer: ratePerMin, RemovePeer: v.StoreLimit[storeID].RemovePeer} + sc := sc.StoreLimitConfig{AddPeer: ratePerMin, RemovePeer: v.StoreLimit[storeID].RemovePeer} v.StoreLimit[storeID] = sc } case storelimit.RemovePeer: - DefaultStoreLimit.SetDefaultStoreLimit(storelimit.RemovePeer, ratePerMin) + sc.DefaultStoreLimit.SetDefaultStoreLimit(storelimit.RemovePeer, ratePerMin) for storeID := range v.StoreLimit { - sc := StoreLimitConfig{AddPeer: v.StoreLimit[storeID].AddPeer, RemovePeer: ratePerMin} + sc := sc.StoreLimitConfig{AddPeer: v.StoreLimit[storeID].AddPeer, RemovePeer: ratePerMin} v.StoreLimit[storeID] = sc } } @@ -426,7 +435,7 @@ func (o *PersistOptions) GetHotRegionScheduleLimit() uint64 { } // GetStoreLimit returns the limit of a store. -func (o *PersistOptions) GetStoreLimit(storeID uint64) (returnSC StoreLimitConfig) { +func (o *PersistOptions) GetStoreLimit(storeID uint64) (returnSC sc.StoreLimitConfig) { defer func() { returnSC.RemovePeer = o.getTTLFloatOr(fmt.Sprintf("remove-peer-%v", storeID), returnSC.RemovePeer) returnSC.AddPeer = o.getTTLFloatOr(fmt.Sprintf("add-peer-%v", storeID), returnSC.AddPeer) @@ -435,9 +444,9 @@ func (o *PersistOptions) GetStoreLimit(storeID uint64) (returnSC StoreLimitConfi return limit } cfg := o.GetScheduleConfig().Clone() - sc := StoreLimitConfig{ - AddPeer: DefaultStoreLimit.GetDefaultStoreLimit(storelimit.AddPeer), - RemovePeer: DefaultStoreLimit.GetDefaultStoreLimit(storelimit.RemovePeer), + sc := sc.StoreLimitConfig{ + AddPeer: sc.DefaultStoreLimit.GetDefaultStoreLimit(storelimit.AddPeer), + RemovePeer: sc.DefaultStoreLimit.GetDefaultStoreLimit(storelimit.RemovePeer), } v, ok1, err := o.getTTLFloat("default-add-peer") if err != nil { @@ -489,13 +498,13 @@ func (o *PersistOptions) GetStoreLimitByType(storeID uint64, typ storelimit.Type } // GetAllStoresLimit returns the limit of all stores. -func (o *PersistOptions) GetAllStoresLimit() map[uint64]StoreLimitConfig { +func (o *PersistOptions) GetAllStoresLimit() map[uint64]sc.StoreLimitConfig { return o.GetScheduleConfig().StoreLimit } -// GetStoreLimitMode returns the limit mode of store. -func (o *PersistOptions) GetStoreLimitMode() string { - return o.GetScheduleConfig().StoreLimitMode +// GetStoreLimitVersion returns the limit version of store. +func (o *PersistOptions) GetStoreLimitVersion() string { + return o.GetScheduleConfig().StoreLimitVersion } // GetTolerantSizeRatio gets the tolerant size ratio. @@ -605,11 +614,7 @@ func (o *PersistOptions) IsLocationReplacementEnabled() bool { // GetMaxMovableHotPeerSize returns the max movable hot peer size. func (o *PersistOptions) GetMaxMovableHotPeerSize() int64 { - size := o.GetScheduleConfig().MaxMovableHotPeerSize - if size <= 0 { - size = defaultMaxMovableHotPeerSize - } - return size + return o.GetScheduleConfig().MaxMovableHotPeerSize } // IsDebugMetricsEnabled returns if debug metrics is enabled. @@ -617,7 +622,7 @@ func (o *PersistOptions) IsDebugMetricsEnabled() bool { return o.GetScheduleConfig().EnableDebugMetrics } -// IsUseJointConsensus returns if using joint consensus as a operator step is enabled. +// IsUseJointConsensus returns if using joint consensus as an operator step is enabled. func (o *PersistOptions) IsUseJointConsensus() bool { return o.GetScheduleConfig().EnableJointConsensus } @@ -641,15 +646,26 @@ func (o *PersistOptions) GetHotRegionCacheHitsThreshold() int { } // GetStoresLimit gets the stores' limit. -func (o *PersistOptions) GetStoresLimit() map[uint64]StoreLimitConfig { +func (o *PersistOptions) GetStoresLimit() map[uint64]sc.StoreLimitConfig { return o.GetScheduleConfig().StoreLimit } // GetSchedulers gets the scheduler configurations. -func (o *PersistOptions) GetSchedulers() SchedulerConfigs { +func (o *PersistOptions) GetSchedulers() sc.SchedulerConfigs { return o.GetScheduleConfig().Schedulers } +// IsSchedulerDisabled returns if the scheduler is disabled. +func (o *PersistOptions) IsSchedulerDisabled(t string) bool { + schedulers := o.GetScheduleConfig().Schedulers + for _, s := range schedulers { + if t == s.Type { + return s.Disable + } + } + return false +} + // GetHotRegionsWriteInterval gets interval for PD to store Hot Region information. func (o *PersistOptions) GetHotRegionsWriteInterval() time.Duration { return o.GetScheduleConfig().HotRegionsWriteInterval.Duration @@ -667,21 +683,38 @@ func (o *PersistOptions) AddSchedulerCfg(tp string, args []string) { // comparing args is to cover the case that there are schedulers in same type but not with same name // such as two schedulers of type "evict-leader", // one name is "evict-leader-scheduler-1" and the other is "evict-leader-scheduler-2" - if reflect.DeepEqual(schedulerCfg, SchedulerConfig{Type: tp, Args: args, Disable: false}) { + if reflect.DeepEqual(schedulerCfg, sc.SchedulerConfig{Type: tp, Args: args, Disable: false}) { return } - if reflect.DeepEqual(schedulerCfg, SchedulerConfig{Type: tp, Args: args, Disable: true}) { + if reflect.DeepEqual(schedulerCfg, sc.SchedulerConfig{Type: tp, Args: args, Disable: true}) { schedulerCfg.Disable = false v.Schedulers[i] = schedulerCfg o.SetScheduleConfig(v) return } } - v.Schedulers = append(v.Schedulers, SchedulerConfig{Type: tp, Args: args, Disable: false}) + v.Schedulers = append(v.Schedulers, sc.SchedulerConfig{Type: tp, Args: args, Disable: false}) o.SetScheduleConfig(v) } +// RemoveSchedulerCfg removes the scheduler configurations. +func (o *PersistOptions) RemoveSchedulerCfg(tp string) { + v := o.GetScheduleConfig().Clone() + for i, schedulerCfg := range v.Schedulers { + if tp == schedulerCfg.Type { + if sc.IsDefaultScheduler(tp) { + schedulerCfg.Disable = true + v.Schedulers[i] = schedulerCfg + } else { + v.Schedulers = append(v.Schedulers[:i], v.Schedulers[i+1:]...) + } + o.SetScheduleConfig(v) + return + } + } +} + // SetLabelProperty sets the label property. func (o *PersistOptions) SetLabelProperty(typ, labelKey, labelValue string) { cfg := o.GetLabelPropertyConfig().Clone() @@ -719,6 +752,7 @@ func (o *PersistOptions) Persist(storage endpoint.ConfigStorage) error { PDServerCfg: *o.GetPDServerConfig(), ReplicationMode: *o.GetReplicationModeConfig(), LabelProperty: o.GetLabelPropertyConfig(), + Keyspace: *o.GetKeyspaceConfig(), ClusterVersion: *o.GetClusterVersion(), } err := storage.SaveConfig(cfg) @@ -746,14 +780,15 @@ func (o *PersistOptions) Reload(storage endpoint.ConfigStorage) error { o.pdServerConfig.Store(&cfg.PDServerCfg) o.replicationMode.Store(&cfg.ReplicationMode) o.labelProperty.Store(cfg.LabelProperty) + o.keyspace.Store(&cfg.Keyspace) o.SetClusterVersion(&cfg.ClusterVersion) } return nil } -func (o *PersistOptions) adjustScheduleCfg(scheduleCfg *ScheduleConfig) { +func (o *PersistOptions) adjustScheduleCfg(scheduleCfg *sc.ScheduleConfig) { // In case we add new default schedulers. - for _, ps := range DefaultSchedulers { + for _, ps := range sc.DefaultSchedulers { if slice.NoneOf(scheduleCfg.Schedulers, func(i int) bool { return scheduleCfg.Schedulers[i].Type == ps.Type }) { @@ -898,3 +933,27 @@ func (o *PersistOptions) SetAllStoresLimitTTL(ctx context.Context, client *clien } return err } + +var haltSchedulingStatus = schedulingAllowanceStatusGauge.WithLabelValues("halt-scheduling") + +// SetHaltScheduling set HaltScheduling. +func (o *PersistOptions) SetHaltScheduling(halt bool, source string) { + v := o.GetScheduleConfig().Clone() + v.HaltScheduling = halt + o.SetScheduleConfig(v) + if halt { + haltSchedulingStatus.Set(1) + schedulingAllowanceStatusGauge.WithLabelValues(source).Set(1) + } else { + haltSchedulingStatus.Set(0) + schedulingAllowanceStatusGauge.WithLabelValues(source).Set(0) + } +} + +// IsSchedulingHalted returns if PD scheduling is halted. +func (o *PersistOptions) IsSchedulingHalted() bool { + if o == nil { + return false + } + return o.GetScheduleConfig().HaltScheduling +} diff --git a/server/config/store_config.go b/server/config/store_config.go index a3cd8e54650..a468ebae0ae 100644 --- a/server/config/store_config.go +++ b/server/config/store_config.go @@ -41,18 +41,26 @@ var ( defaultRegionMaxKey = uint64(1440000) // default region split key is 960000 defaultRegionSplitKey = uint64(960000) + + raftStoreV2 = "raft-kv2" ) // StoreConfig is the config of store like TiKV. // generated by https://mholt.github.io/json-to-go/. type StoreConfig struct { Coprocessor `json:"coprocessor"` + Storage `json:"storage"` RegionMaxSizeMB uint64 `json:"-"` RegionSplitSizeMB uint64 `json:"-"` RegionBucketSizeMB uint64 `json:"-"` } +// Storage is the config for the tikv storage. +type Storage struct { + Engine string `json:"engine"` +} + // Coprocessor is the config of coprocessor. type Coprocessor struct { // RegionMaxSize is the max size of a region, if the region size is larger than this value, region will be @@ -68,7 +76,7 @@ type Coprocessor struct { // Equal returns true if the two configs are equal. func (c *StoreConfig) Equal(other *StoreConfig) bool { - return reflect.DeepEqual(c.Coprocessor, other.Coprocessor) + return reflect.DeepEqual(c.Coprocessor, other.Coprocessor) && reflect.DeepEqual(c.Storage, other.Storage) } // String implements fmt.Stringer interface. @@ -120,6 +128,14 @@ func (c *StoreConfig) IsEnableRegionBucket() bool { return c.Coprocessor.EnableRegionBucket } +// IsRaftKV2 returns true if the raft kv is v2. +func (c *StoreConfig) IsRaftKV2() bool { + if c == nil { + return false + } + return c.Storage.Engine == raftStoreV2 +} + // SetRegionBucketEnabled sets if the region bucket is enabled. func (c *StoreConfig) SetRegionBucketEnabled(enabled bool) { if c == nil { @@ -204,24 +220,30 @@ func NewTestStoreConfigManager(whiteList []string) *StoreConfigManager { } // ObserveConfig is used to observe the config change. -func (m *StoreConfigManager) ObserveConfig(address string) error { +// switchRaftV2 is true if the new config's raft engine is v2 and the old is v1. +func (m *StoreConfigManager) ObserveConfig(address string) (switchRaftV2 bool, err error) { cfg, err := m.source.GetConfig(address) if err != nil { - return err + return switchRaftV2, err } old := m.GetStoreConfig() if cfg != nil && !old.Equal(cfg) { - log.Info("sync the store config successful", zap.String("store-address", address), zap.String("store-config", cfg.String())) - m.update(cfg) + log.Info("sync the store config successful", zap.String("store-address", address), zap.String("store-config", cfg.String()), zap.String("old-config", old.String())) + switchRaftV2 = m.update(cfg) } - return nil + return switchRaftV2, nil } -func (m *StoreConfigManager) update(cfg *StoreConfig) { +// update returns true if the new config's raft engine is v2 and the old is v1 +func (m *StoreConfigManager) update(cfg *StoreConfig) (switchRaftV2 bool) { cfg.RegionMaxSizeMB = typeutil.ParseMBFromText(cfg.RegionMaxSize, defaultRegionMaxSize) cfg.RegionSplitSizeMB = typeutil.ParseMBFromText(cfg.RegionSplitSize, defaultRegionSplitSize) cfg.RegionBucketSizeMB = typeutil.ParseMBFromText(cfg.RegionBucketSize, defaultBucketSize) + + config := m.config.Load().(*StoreConfig) + switchRaftV2 = config.Storage.Engine != raftStoreV2 && cfg.Storage.Engine == raftStoreV2 m.config.Store(cfg) + return } // GetStoreConfig returns the current store configuration. @@ -296,5 +318,6 @@ func (f *FakeSource) GetConfig(url string) (*StoreConfig, error) { } config := &StoreConfig{} config.RegionMaxSize = "10MiB" + config.Storage.Engine = raftStoreV2 return config, nil } diff --git a/server/config/store_config_test.go b/server/config/store_config_test.go index 6916fedc929..7cc9119f4d5 100644 --- a/server/config/store_config_test.go +++ b/server/config/store_config_test.go @@ -65,8 +65,10 @@ func TestUpdateConfig(t *testing.T) { manager := NewTestStoreConfigManager([]string{"tidb.com"}) manager.ObserveConfig("tikv.com") re.Equal(uint64(144), manager.GetStoreConfig().GetRegionMaxSize()) + re.NotEqual(raftStoreV2, manager.GetStoreConfig().GetRegionMaxSize()) manager.ObserveConfig("tidb.com") re.Equal(uint64(10), manager.GetStoreConfig().GetRegionMaxSize()) + re.Equal(raftStoreV2, manager.GetStoreConfig().Engine) // case2: the config should not update if config is same expect some ignore field. c, err := manager.source.GetConfig("tidb.com") @@ -100,6 +102,9 @@ func TestParseConfig(t *testing.T) { "region-bucket-size":"96MiB", "region-size-threshold-for-approximate":"384MiB", "region-bucket-merge-size-ratio":0.33 +}, +"storage":{ + "engine":"raft-kv2" } } ` @@ -108,6 +113,8 @@ func TestParseConfig(t *testing.T) { re.NoError(json.Unmarshal([]byte(body), &config)) m.update(&config) re.Equal(uint64(96), config.GetRegionBucketSize()) + re.True(config.IsRaftKV2()) + re.Equal(raftStoreV2, config.Storage.Engine) } func TestMergeCheck(t *testing.T) { @@ -126,21 +133,21 @@ func TestMergeCheck(t *testing.T) { mergeKeys: 200000, pass: true, }, { - // case 2: the smallest region is 68MiB，it can't be merged again. + // case 2: the smallest region is 68MiB, it can't be merged again. size: 144 + 20, mergeSize: 20, keys: 1440000 + 200000, mergeKeys: 200000, pass: true, }, { - // case 3: the smallest region is 50MiB，it can be merged again. + // case 3: the smallest region is 50MiB, it can be merged again. size: 144 + 2, mergeSize: 50, keys: 1440000 + 20000, mergeKeys: 500000, pass: false, }, { - // case4: the smallest region is 51MiB，it can't be merged again. + // case4: the smallest region is 51MiB, it can't be merged again. size: 144 + 3, mergeSize: 50, keys: 1440000 + 30000, diff --git a/server/config/util.go b/server/config/util.go index cb1e947bf8d..2f25e711cc7 100644 --- a/server/config/util.go +++ b/server/config/util.go @@ -16,49 +16,12 @@ package config import ( "net/url" - "regexp" "strings" "github.com/pingcap/errors" - "github.com/pingcap/kvproto/pkg/metapb" "github.com/tikv/pd/pkg/errs" ) -const ( - // Label key consists of alphanumeric characters, '-', '_', '.' or '/', and must start and end with an - // alphanumeric character. If can also contain an extra '$' at the beginning. - keyFormat = "^[$]?[A-Za-z0-9]([-A-Za-z0-9_./]*[A-Za-z0-9])?$" - // Value key can be any combination of alphanumeric characters, '-', '_', '.' or '/'. It can also be empty to - // mark the label as deleted. - valueFormat = "^[-A-Za-z0-9_./]*$" -) - -func validateFormat(s, format string) error { - isValid, _ := regexp.MatchString(format, s) - if !isValid { - return errors.Errorf("%s does not match format %q", s, format) - } - return nil -} - -// ValidateLabels checks the legality of the labels. -func ValidateLabels(labels []*metapb.StoreLabel) error { - for _, label := range labels { - if err := validateFormat(label.Key, keyFormat); err != nil { - return err - } - if err := validateFormat(label.Value, valueFormat); err != nil { - return err - } - } - return nil -} - -// ValidateLabelKey checks the legality of the label key. -func ValidateLabelKey(key string) error { - return validateFormat(key, keyFormat) -} - // ValidateURLWithScheme checks the format of the URL. func ValidateURLWithScheme(rawURL string) error { u, err := url.ParseRequestURI(rawURL) diff --git a/server/config/util_test.go b/server/config/util_test.go index 6327d465bd6..08c21babe20 100644 --- a/server/config/util_test.go +++ b/server/config/util_test.go @@ -17,44 +17,9 @@ package config import ( "testing" - "github.com/pingcap/kvproto/pkg/metapb" "github.com/stretchr/testify/require" ) -func TestValidateLabels(t *testing.T) { - re := require.New(t) - tests := []struct { - label string - hasErr bool - }{ - {"z1", false}, - {"z-1", false}, - {"h1;", true}, - {"z_1", false}, - {"z_1&", true}, - {"cn", false}, - {"Zo^ne", true}, - {"z_", true}, - {"hos&t-15", true}, - {"_test1", true}, - {"-test1", true}, - {"127.0.0.1", false}, - {"www.pingcap.com", false}, - {"h_127.0.0.1", false}, - {"a", false}, - {"a/b", false}, - {"ab/", true}, - {"/ab", true}, - {"$abc", false}, - {"$", true}, - {"a$b", true}, - {"$$", true}, - } - for _, test := range tests { - re.Equal(test.hasErr, ValidateLabels([]*metapb.StoreLabel{{Key: test.label}}) != nil) - } -} - func TestValidateURLWithScheme(t *testing.T) { re := require.New(t) tests := []struct { diff --git a/server/gc_service.go b/server/gc_service.go new file mode 100644 index 00000000000..fcb0550c15f --- /dev/null +++ b/server/gc_service.go @@ -0,0 +1,268 @@ +// Copyright 2023 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "context" + "encoding/json" + "fmt" + "math" + "path" + "strings" + + "github.com/pingcap/kvproto/pkg/pdpb" + "github.com/pingcap/log" + "github.com/tikv/pd/pkg/errs" + "github.com/tikv/pd/pkg/storage/endpoint" + "github.com/tikv/pd/pkg/tso" + "github.com/tikv/pd/pkg/utils/etcdutil" + "github.com/tikv/pd/pkg/utils/tsoutil" + "go.etcd.io/etcd/clientv3" + "go.uber.org/zap" + "google.golang.org/grpc" +) + +// GetGCSafePointV2 return gc safe point for the given keyspace. +func (s *GrpcServer) GetGCSafePointV2(ctx context.Context, request *pdpb.GetGCSafePointV2Request) (*pdpb.GetGCSafePointV2Response, error) { + fn := func(ctx context.Context, client *grpc.ClientConn) (interface{}, error) { + return pdpb.NewPDClient(client).GetGCSafePointV2(ctx, request) + } + if rsp, err := s.unaryMiddleware(ctx, request, fn); err != nil { + return nil, err + } else if rsp != nil { + return rsp.(*pdpb.GetGCSafePointV2Response), err + } + + safePoint, err := s.safePointV2Manager.LoadGCSafePoint(request.GetKeyspaceId()) + + if err != nil { + return &pdpb.GetGCSafePointV2Response{ + Header: s.wrapErrorToHeader(pdpb.ErrorType_UNKNOWN, err.Error()), + }, err + } + + return &pdpb.GetGCSafePointV2Response{ + Header: s.header(), + SafePoint: safePoint.SafePoint, + }, nil +} + +// UpdateGCSafePointV2 update gc safe point for the given keyspace. +func (s *GrpcServer) UpdateGCSafePointV2(ctx context.Context, request *pdpb.UpdateGCSafePointV2Request) (*pdpb.UpdateGCSafePointV2Response, error) { + fn := func(ctx context.Context, client *grpc.ClientConn) (interface{}, error) { + return pdpb.NewPDClient(client).UpdateGCSafePointV2(ctx, request) + } + if rsp, err := s.unaryMiddleware(ctx, request, fn); err != nil { + return nil, err + } else if rsp != nil { + return rsp.(*pdpb.UpdateGCSafePointV2Response), err + } + + newSafePoint := request.GetSafePoint() + oldSafePoint, err := s.safePointV2Manager.UpdateGCSafePoint(&endpoint.GCSafePointV2{ + KeyspaceID: request.KeyspaceId, + SafePoint: request.SafePoint, + }) + if err != nil { + return nil, err + } + if newSafePoint > oldSafePoint.SafePoint { + log.Info("updated gc safe point", + zap.Uint64("safe-point", newSafePoint), + zap.Uint32("keyspace-id", request.GetKeyspaceId())) + } else if newSafePoint < oldSafePoint.SafePoint { + log.Warn("trying to update gc safe point", + zap.Uint64("old-safe-point", oldSafePoint.SafePoint), + zap.Uint64("new-safe-point", newSafePoint), + zap.Uint32("keyspace-id", request.GetKeyspaceId())) + newSafePoint = oldSafePoint.SafePoint + } + + return &pdpb.UpdateGCSafePointV2Response{ + Header: s.header(), + NewSafePoint: newSafePoint, + }, nil +} + +// UpdateServiceSafePointV2 update service safe point for the given keyspace. +func (s *GrpcServer) UpdateServiceSafePointV2(ctx context.Context, request *pdpb.UpdateServiceSafePointV2Request) (*pdpb.UpdateServiceSafePointV2Response, error) { + fn := func(ctx context.Context, client *grpc.ClientConn) (interface{}, error) { + return pdpb.NewPDClient(client).UpdateServiceSafePointV2(ctx, request) + } + if rsp, err := s.unaryMiddleware(ctx, request, fn); err != nil { + return nil, err + } else if rsp != nil { + return rsp.(*pdpb.UpdateServiceSafePointV2Response), err + } + + var ( + nowTSO pdpb.Timestamp + err error + ) + if s.IsAPIServiceMode() { + nowTSO, err = s.getGlobalTSOFromTSOServer(ctx) + } else { + nowTSO, err = s.tsoAllocatorManager.HandleRequest(tso.GlobalDCLocation, 1) + } + if err != nil { + return nil, err + } + now, _ := tsoutil.ParseTimestamp(nowTSO) + + var minServiceSafePoint *endpoint.ServiceSafePointV2 + if request.Ttl < 0 { + minServiceSafePoint, err = s.safePointV2Manager.RemoveServiceSafePoint(request.GetKeyspaceId(), string(request.GetServiceId()), now) + } else { + serviceSafePoint := &endpoint.ServiceSafePointV2{ + KeyspaceID: request.GetKeyspaceId(), + ServiceID: string(request.GetServiceId()), + ExpiredAt: now.Unix() + request.GetTtl(), + SafePoint: request.GetSafePoint(), + } + // Fix possible overflow. + if math.MaxInt64-now.Unix() <= request.GetTtl() { + serviceSafePoint.ExpiredAt = math.MaxInt64 + } + minServiceSafePoint, err = s.safePointV2Manager.UpdateServiceSafePoint(serviceSafePoint, now) + } + if err != nil { + return nil, err + } + return &pdpb.UpdateServiceSafePointV2Response{ + Header: s.header(), + ServiceId: []byte(minServiceSafePoint.ServiceID), + Ttl: minServiceSafePoint.ExpiredAt - now.Unix(), + MinSafePoint: minServiceSafePoint.SafePoint, + }, nil +} + +// WatchGCSafePointV2 watch keyspaces gc safe point changes. +func (s *GrpcServer) WatchGCSafePointV2(request *pdpb.WatchGCSafePointV2Request, stream pdpb.PD_WatchGCSafePointV2Server) error { + ctx, cancel := context.WithCancel(s.Context()) + defer cancel() + revision := request.GetRevision() + // If the revision is compacted, will meet required revision has been compacted error. + // - If required revision < CompactRevision, we need to reload all configs to avoid losing data. + // - If required revision >= CompactRevision, just keep watching. + // Use WithPrevKV() to get the previous key-value pair when get Delete Event. + watchChan := s.client.Watch(ctx, path.Join(s.rootPath, endpoint.GCSafePointV2Prefix()), clientv3.WithRev(revision), clientv3.WithPrefix()) + for { + select { + case <-ctx.Done(): + return nil + case res := <-watchChan: + if res.Err() != nil { + var resp pdpb.WatchGCSafePointV2Response + if revision < res.CompactRevision { + resp.Header = s.wrapErrorToHeader(pdpb.ErrorType_DATA_COMPACTED, + fmt.Sprintf("required watch revision: %d is smaller than current compact/min revision %d.", revision, res.CompactRevision)) + } else { + resp.Header = s.wrapErrorToHeader(pdpb.ErrorType_UNKNOWN, + fmt.Sprintf("watch channel meet other error %s.", res.Err().Error())) + } + if err := stream.Send(&resp); err != nil { + return err + } + // Err() indicates that this WatchResponse holds a channel-closing error. + return res.Err() + } + revision = res.Header.GetRevision() + + safePointEvents := make([]*pdpb.SafePointEvent, 0, len(res.Events)) + for _, event := range res.Events { + gcSafePoint := &endpoint.GCSafePointV2{} + if err := json.Unmarshal(event.Kv.Value, gcSafePoint); err != nil { + return err + } + safePointEvents = append(safePointEvents, &pdpb.SafePointEvent{ + KeyspaceId: gcSafePoint.KeyspaceID, + SafePoint: gcSafePoint.SafePoint, + Type: pdpb.EventType(event.Type), + }) + } + if len(safePointEvents) > 0 { + if err := stream.Send(&pdpb.WatchGCSafePointV2Response{Header: s.header(), Events: safePointEvents, Revision: res.Header.GetRevision()}); err != nil { + return err + } + } + } + } +} + +// GetAllGCSafePointV2 return all gc safe point v2. +func (s *GrpcServer) GetAllGCSafePointV2(ctx context.Context, request *pdpb.GetAllGCSafePointV2Request) (*pdpb.GetAllGCSafePointV2Response, error) { + fn := func(ctx context.Context, client *grpc.ClientConn) (interface{}, error) { + return pdpb.NewPDClient(client).GetAllGCSafePointV2(ctx, request) + } + if rsp, err := s.unaryMiddleware(ctx, request, fn); err != nil { + return nil, err + } else if rsp != nil { + return rsp.(*pdpb.GetAllGCSafePointV2Response), err + } + + startkey := endpoint.GCSafePointV2Prefix() + endkey := clientv3.GetPrefixRangeEnd(startkey) + _, values, revision, err := s.loadRangeFromETCD(startkey, endkey) + + gcSafePoints := make([]*pdpb.GCSafePointV2, 0, len(values)) + for _, value := range values { + jsonGcSafePoint := &endpoint.GCSafePointV2{} + if err = json.Unmarshal([]byte(value), jsonGcSafePoint); err != nil { + return nil, errs.ErrJSONUnmarshal.Wrap(err).GenWithStackByCause() + } + gcSafePoint := &pdpb.GCSafePointV2{ + KeyspaceId: jsonGcSafePoint.KeyspaceID, + GcSafePoint: jsonGcSafePoint.SafePoint, + } + log.Debug("get all gc safe point v2", + zap.Uint32("keyspace-id", jsonGcSafePoint.KeyspaceID), + zap.Uint64("gc-safe-point", jsonGcSafePoint.SafePoint)) + gcSafePoints = append(gcSafePoints, gcSafePoint) + } + + if err != nil { + return &pdpb.GetAllGCSafePointV2Response{ + Header: s.wrapErrorToHeader(pdpb.ErrorType_UNKNOWN, err.Error()), + }, err + } + + return &pdpb.GetAllGCSafePointV2Response{ + Header: s.header(), + GcSafePoints: gcSafePoints, + Revision: revision, + }, nil +} + +func (s *GrpcServer) loadRangeFromETCD(startKey, endKey string) ([]string, []string, int64, error) { + startKey = strings.Join([]string{s.rootPath, startKey}, "/") + var opOption []clientv3.OpOption + if endKey == "\x00" { + opOption = append(opOption, clientv3.WithPrefix()) + } else { + endKey = strings.Join([]string{s.rootPath, endKey}, "/") + opOption = append(opOption, clientv3.WithRange(endKey)) + } + resp, err := etcdutil.EtcdKVGet(s.client, startKey, opOption...) + if err != nil { + return nil, nil, 0, err + } + keys := make([]string, 0, len(resp.Kvs)) + values := make([]string, 0, len(resp.Kvs)) + for _, item := range resp.Kvs { + keys = append(keys, strings.TrimPrefix(strings.TrimPrefix(string(item.Key), s.rootPath), "/")) + values = append(values, string(item.Value)) + } + return keys, values, resp.Header.Revision, nil +} diff --git a/server/grpc_service.go b/server/grpc_service.go index 4531499621e..62a8f8005ce 100644 --- a/server/grpc_service.go +++ b/server/grpc_service.go @@ -20,6 +20,8 @@ import ( "io" "path" "strconv" + "strings" + "sync" "sync/atomic" "time" @@ -27,9 +29,11 @@ import ( "github.com/pingcap/failpoint" "github.com/pingcap/kvproto/pkg/metapb" "github.com/pingcap/kvproto/pkg/pdpb" + "github.com/pingcap/kvproto/pkg/tsopb" "github.com/pingcap/log" "github.com/tikv/pd/pkg/core" "github.com/tikv/pd/pkg/errs" + "github.com/tikv/pd/pkg/mcs/utils" "github.com/tikv/pd/pkg/storage/endpoint" "github.com/tikv/pd/pkg/storage/kv" "github.com/tikv/pd/pkg/tso" @@ -46,25 +50,31 @@ import ( ) const ( - heartbeatSendTimeout = 5 * time.Second - - // tso - maxMergeTSORequests = 10000 - defaultTSOProxyTimeout = 3 * time.Second + heartbeatSendTimeout = 5 * time.Second + maxRetryTimesRequestTSOServer = 3 + retryIntervalRequestTSOServer = 500 * time.Millisecond + getMinTSFromTSOServerTimeout = 1 * time.Second + defaultGRPCDialTimeout = 3 * time.Second ) // gRPC errors var ( // ErrNotLeader is returned when current server is not the leader and not possible to process request. // TODO: work as proxy. - ErrNotLeader = status.Errorf(codes.Unavailable, "not leader") - ErrNotStarted = status.Errorf(codes.Unavailable, "server not started") - ErrSendHeartbeatTimeout = status.Errorf(codes.DeadlineExceeded, "send heartbeat timeout") + ErrNotLeader = status.Errorf(codes.Unavailable, "not leader") + ErrNotStarted = status.Errorf(codes.Unavailable, "server not started") + ErrSendHeartbeatTimeout = status.Errorf(codes.DeadlineExceeded, "send heartbeat timeout") + ErrNotFoundTSOAddr = status.Errorf(codes.NotFound, "not found tso address") + ErrForwardTSOTimeout = status.Errorf(codes.DeadlineExceeded, "forward tso request timeout") + ErrMaxCountTSOProxyRoutinesExceeded = status.Errorf(codes.ResourceExhausted, "max count of concurrent tso proxy routines exceeded") + ErrTSOProxyRecvFromClientTimeout = status.Errorf(codes.DeadlineExceeded, "tso proxy timeout when receiving from client; stream closed by server") + ErrEtcdNotStarted = status.Errorf(codes.Unavailable, "server is started, but etcd not started") ) // GrpcServer wraps Server to provide grpc service. type GrpcServer struct { *Server + concurrentTSOProxyStreamings atomic.Int32 } type request interface { @@ -92,11 +102,177 @@ func (s *GrpcServer) unaryMiddleware(ctx context.Context, req request, fn forwar return nil, nil } -func (s *GrpcServer) wrapErrorToHeader(errorType pdpb.ErrorType, message string) *pdpb.ResponseHeader { - return s.errorHeader(&pdpb.Error{ - Type: errorType, - Message: message, - }) +// GetClusterInfo implements gRPC PDServer. +func (s *GrpcServer) GetClusterInfo(ctx context.Context, _ *pdpb.GetClusterInfoRequest) (*pdpb.GetClusterInfoResponse, error) { + // Here we purposely do not check the cluster ID because the client does not know the correct cluster ID + // at startup and needs to get the cluster ID with the first request (i.e. GetMembers). + if s.IsClosed() { + return &pdpb.GetClusterInfoResponse{ + Header: s.wrapErrorToHeader(pdpb.ErrorType_UNKNOWN, errs.ErrServerNotStarted.FastGenByArgs().Error()), + }, nil + } + + var tsoServiceAddrs []string + svcModes := make([]pdpb.ServiceMode, 0) + if s.IsAPIServiceMode() { + svcModes = append(svcModes, pdpb.ServiceMode_API_SVC_MODE) + tsoServiceAddrs = s.keyspaceGroupManager.GetTSOServiceAddrs() + } else { + svcModes = append(svcModes, pdpb.ServiceMode_PD_SVC_MODE) + } + + return &pdpb.GetClusterInfoResponse{ + Header: s.header(), + ServiceModes: svcModes, + TsoUrls: tsoServiceAddrs, + }, nil +} + +// GetMinTS implements gRPC PDServer. In PD service mode, it simply returns a timestamp. +// In API service mode, it queries all tso servers and gets the minimum timestamp across +// all keyspace groups. +func (s *GrpcServer) GetMinTS( + ctx context.Context, request *pdpb.GetMinTSRequest, +) (*pdpb.GetMinTSResponse, error) { + if err := s.validateRequest(request.GetHeader()); err != nil { + return &pdpb.GetMinTSResponse{ + Header: s.wrapErrorToHeader(pdpb.ErrorType_UNKNOWN, err.Error()), + }, nil + } + + var ( + minTS *pdpb.Timestamp + err error + ) + if s.IsAPIServiceMode() { + minTS, err = s.GetMinTSFromTSOService(tso.GlobalDCLocation) + } else { + start := time.Now() + ts, internalErr := s.tsoAllocatorManager.HandleRequest(tso.GlobalDCLocation, 1) + if internalErr == nil { + tsoHandleDuration.Observe(time.Since(start).Seconds()) + } + minTS = &ts + } + if err != nil { + return &pdpb.GetMinTSResponse{ + Header: s.wrapErrorToHeader(pdpb.ErrorType_UNKNOWN, err.Error()), + Timestamp: minTS, + }, nil + } + + return &pdpb.GetMinTSResponse{ + Header: s.header(), + Timestamp: minTS, + }, nil +} + +// GetMinTSFromTSOService queries all tso servers and gets the minimum timestamp across +// all keyspace groups. +func (s *GrpcServer) GetMinTSFromTSOService(dcLocation string) (*pdpb.Timestamp, error) { + addrs := s.keyspaceGroupManager.GetTSOServiceAddrs() + if len(addrs) == 0 { + return &pdpb.Timestamp{}, errs.ErrGetMinTS.FastGenByArgs("no tso servers/pods discovered") + } + + // Get the minimal timestamp from the TSO servers/pods + var mutex sync.Mutex + resps := make([]*tsopb.GetMinTSResponse, 0) + wg := sync.WaitGroup{} + wg.Add(len(addrs)) + for _, addr := range addrs { + go func(addr string) { + defer wg.Done() + resp, err := s.getMinTSFromSingleServer(s.ctx, dcLocation, addr) + if err != nil || resp == nil { + log.Warn("failed to get min ts from tso server", + zap.String("address", addr), zap.Error(err)) + return + } + mutex.Lock() + defer mutex.Unlock() + resps = append(resps, resp) + }(addr) + } + wg.Wait() + + // Check the results. The returned minimal timestamp is valid if all the conditions are met: + // 1. The number of responses is equal to the number of TSO servers/pods. + // 2. The number of keyspace groups asked is equal to the number of TSO servers/pods. + // 3. The minimal timestamp is not zero. + var ( + minTS *pdpb.Timestamp + keyspaceGroupsAsked uint32 + ) + if len(resps) == 0 { + return &pdpb.Timestamp{}, errs.ErrGetMinTS.FastGenByArgs("none of tso server/pod responded") + } + emptyTS := &pdpb.Timestamp{} + keyspaceGroupsTotal := resps[0].KeyspaceGroupsTotal + for _, resp := range resps { + if resp.KeyspaceGroupsTotal == 0 { + return &pdpb.Timestamp{}, errs.ErrGetMinTS.FastGenByArgs("the tso service has no keyspace group") + } + if resp.KeyspaceGroupsTotal != keyspaceGroupsTotal { + return &pdpb.Timestamp{}, errs.ErrGetMinTS.FastGenByArgs( + "the tso service has inconsistent keyspace group total count") + } + keyspaceGroupsAsked += resp.KeyspaceGroupsServing + if tsoutil.CompareTimestamp(resp.Timestamp, emptyTS) > 0 && + (minTS == nil || tsoutil.CompareTimestamp(resp.Timestamp, minTS) < 0) { + minTS = resp.Timestamp + } + } + + if keyspaceGroupsAsked != keyspaceGroupsTotal { + return &pdpb.Timestamp{}, errs.ErrGetMinTS.FastGenByArgs( + fmt.Sprintf("can't query all the tso keyspace groups. Asked %d, expected %d", + keyspaceGroupsAsked, keyspaceGroupsTotal)) + } + + if minTS == nil { + return &pdpb.Timestamp{}, errs.ErrGetMinTS.FastGenByArgs("the tso service is not ready") + } + + return minTS, nil +} + +func (s *GrpcServer) getMinTSFromSingleServer( + ctx context.Context, dcLocation, tsoSrvAddr string, +) (*tsopb.GetMinTSResponse, error) { + cc, err := s.getDelegateClient(s.ctx, tsoSrvAddr) + if err != nil { + return nil, errs.ErrClientGetMinTSO.FastGenByArgs( + fmt.Sprintf("can't connect to tso server %s", tsoSrvAddr)) + } + + cctx, cancel := context.WithTimeout(ctx, getMinTSFromTSOServerTimeout) + defer cancel() + + resp, err := tsopb.NewTSOClient(cc).GetMinTS( + cctx, &tsopb.GetMinTSRequest{ + Header: &tsopb.RequestHeader{ + ClusterId: s.ClusterID(), + }, + DcLocation: dcLocation, + }) + if err != nil { + attachErr := errors.Errorf("error:%s target:%s status:%s", + err, cc.Target(), cc.GetState().String()) + return nil, errs.ErrClientGetMinTSO.Wrap(attachErr).GenWithStackByCause() + } + if resp == nil { + attachErr := errors.Errorf("error:%s target:%s status:%s", + "no min ts info collected", cc.Target(), cc.GetState().String()) + return nil, errs.ErrClientGetMinTSO.Wrap(attachErr).GenWithStackByCause() + } + if resp.GetHeader().GetError() != nil { + attachErr := errors.Errorf("error:%s target:%s status:%s", + resp.GetHeader().GetError().String(), cc.Target(), cc.GetState().String()) + return nil, errs.ErrClientGetMinTSO.Wrap(attachErr).GenWithStackByCause() + } + + return resp, nil } // GetMembers implements gRPC PDServer. @@ -105,12 +281,7 @@ func (s *GrpcServer) GetMembers(context.Context, *pdpb.GetMembersRequest) (*pdpb // at startup and needs to get the cluster ID with the first request (i.e. GetMembers). if s.IsClosed() { return &pdpb.GetMembersResponse{ - Header: &pdpb.ResponseHeader{ - Error: &pdpb.Error{ - Type: pdpb.ErrorType_UNKNOWN, - Message: errs.ErrServerNotStarted.FastGenByArgs().Error(), - }, - }, + Header: s.wrapErrorToHeader(pdpb.ErrorType_UNKNOWN, errs.ErrServerNotStarted.FastGenByArgs().Error()), }, nil } members, err := cluster.GetMembers(s.GetClient()) @@ -129,8 +300,11 @@ func (s *GrpcServer) GetMembers(context.Context, *pdpb.GetMembersRequest) (*pdpb } } - tsoAllocatorManager := s.GetTSOAllocatorManager() - tsoAllocatorLeaders, err := tsoAllocatorManager.GetLocalAllocatorLeaders() + tsoAllocatorLeaders := make(map[string]*pdpb.Member) + if !s.IsAPIServiceMode() { + tsoAllocatorManager := s.GetTSOAllocatorManager() + tsoAllocatorLeaders, err = tsoAllocatorManager.GetLocalAllocatorLeaders() + } if err != nil { return &pdpb.GetMembersResponse{ Header: s.wrapErrorToHeader(pdpb.ErrorType_UNKNOWN, err.Error()), @@ -156,6 +330,10 @@ func (s *GrpcServer) GetMembers(context.Context, *pdpb.GetMembersRequest) (*pdpb // Tso implements gRPC PDServer. func (s *GrpcServer) Tso(stream pdpb.PD_TsoServer) error { + if s.IsAPIServiceMode() { + return s.forwardTSO(stream) + } + var ( doneCh chan struct{} errCh chan error @@ -179,19 +357,22 @@ func (s *GrpcServer) Tso(stream pdpb.PD_TsoServer) error { return errors.WithStack(err) } - streamCtx := stream.Context() - forwardedHost := grpcutil.GetForwardedHost(streamCtx) - if !s.isLocalRequest(forwardedHost) { + if forwardedHost, err := s.getForwardedHost(ctx, stream.Context()); err != nil { + return err + } else if len(forwardedHost) > 0 { + clientConn, err := s.getDelegateClient(s.ctx, forwardedHost) + if err != nil { + return errors.WithStack(err) + } + if errCh == nil { doneCh = make(chan struct{}) defer close(doneCh) errCh = make(chan error) } - s.dispatchTSORequest(ctx, &tsoRequest{ - forwardedHost, - request, - stream, - }, forwardedHost, doneCh, errCh) + + tsoRequest := tsoutil.NewPDProtoRequest(forwardedHost, clientConn, request, stream) + s.tsoDispatcher.DispatchRequest(ctx, tsoRequest, s.pdProtoFactory, doneCh, errCh, s.tsoPrimaryWatcher) continue } @@ -201,10 +382,11 @@ func (s *GrpcServer) Tso(stream pdpb.PD_TsoServer) error { return status.Errorf(codes.Unknown, "server not started") } if request.GetHeader().GetClusterId() != s.clusterID { - return status.Errorf(codes.FailedPrecondition, "mismatch cluster id, need %d but got %d", s.clusterID, request.GetHeader().GetClusterId()) + return status.Errorf(codes.FailedPrecondition, + "mismatch cluster id, need %d but got %d", s.clusterID, request.GetHeader().GetClusterId()) } count := request.GetCount() - ts, err := s.tsoAllocatorManager.HandleTSORequest(request.GetDcLocation(), count) + ts, err := s.tsoAllocatorManager.HandleRequest(request.GetDcLocation(), count) if err != nil { return status.Errorf(codes.Unknown, err.Error()) } @@ -220,180 +402,266 @@ func (s *GrpcServer) Tso(stream pdpb.PD_TsoServer) error { } } -type tsoRequest struct { - forwardedHost string - request *pdpb.TsoRequest - stream pdpb.PD_TsoServer -} - -func (s *GrpcServer) dispatchTSORequest(ctx context.Context, request *tsoRequest, forwardedHost string, doneCh <-chan struct{}, errCh chan<- error) { - tsoRequestChInterface, loaded := s.tsoDispatcher.LoadOrStore(forwardedHost, make(chan *tsoRequest, maxMergeTSORequests)) - if !loaded { - tsDeadlineCh := make(chan deadline, 1) - go s.handleDispatcher(ctx, forwardedHost, tsoRequestChInterface.(chan *tsoRequest), tsDeadlineCh, doneCh, errCh) - go watchTSDeadline(ctx, tsDeadlineCh) - } - tsoRequestChInterface.(chan *tsoRequest) <- request -} - -func (s *GrpcServer) handleDispatcher(ctx context.Context, forwardedHost string, tsoRequestCh <-chan *tsoRequest, tsDeadlineCh chan<- deadline, doneCh <-chan struct{}, errCh chan<- error) { - dispatcherCtx, ctxCancel := context.WithCancel(ctx) - defer ctxCancel() - defer s.tsoDispatcher.Delete(forwardedHost) - +// forwardTSO forward the TSO requests to the TSO service. +func (s *GrpcServer) forwardTSO(stream pdpb.PD_TsoServer) error { var ( - forwardStream pdpb.PD_TsoClient - cancel context.CancelFunc + server = &tsoServer{stream: stream} + forwardStream tsopb.TSO_TsoClient + forwardCtx context.Context + cancelForward context.CancelFunc + lastForwardedHost string ) - client, err := s.getDelegateClient(ctx, forwardedHost) - if err != nil { - goto errHandling - } - log.Info("create tso forward stream", zap.String("forwarded-host", forwardedHost)) - forwardStream, cancel, err = s.createTsoForwardStream(client) -errHandling: - if err != nil || forwardStream == nil { - log.Error("create tso forwarding stream error", zap.String("forwarded-host", forwardedHost), errs.ZapError(errs.ErrGRPCCreateStream, err)) - select { - case <-dispatcherCtx.Done(): - return - case _, ok := <-doneCh: - if !ok { - return - } - case errCh <- err: - close(errCh) - return + defer func() { + s.concurrentTSOProxyStreamings.Add(-1) + if cancelForward != nil { + cancelForward() + } + }() + + maxConcurrentTSOProxyStreamings := int32(s.GetMaxConcurrentTSOProxyStreamings()) + if maxConcurrentTSOProxyStreamings >= 0 { + if newCount := s.concurrentTSOProxyStreamings.Add(1); newCount > maxConcurrentTSOProxyStreamings { + return errors.WithStack(ErrMaxCountTSOProxyRoutinesExceeded) } } - defer cancel() - requests := make([]*tsoRequest, maxMergeTSORequests+1) + tsDeadlineCh := make(chan *tsoutil.TSDeadline, 1) + go tsoutil.WatchTSDeadline(stream.Context(), tsDeadlineCh) + for { select { - case first := <-tsoRequestCh: - pendingTSOReqCount := len(tsoRequestCh) + 1 - requests[0] = first - for i := 1; i < pendingTSOReqCount; i++ { - requests[i] = <-tsoRequestCh - } - done := make(chan struct{}) - dl := deadline{ - timer: time.After(defaultTSOProxyTimeout), - done: done, - cancel: cancel, + case <-s.ctx.Done(): + return errors.WithStack(s.ctx.Err()) + case <-stream.Context().Done(): + return stream.Context().Err() + default: + } + + request, err := server.Recv(s.GetTSOProxyRecvFromClientTimeout()) + if err == io.EOF { + return nil + } + if err != nil { + return errors.WithStack(err) + } + if request.GetCount() == 0 { + err = errs.ErrGenerateTimestamp.FastGenByArgs("tso count should be positive") + return status.Errorf(codes.Unknown, err.Error()) + } + + forwardedHost, ok := s.GetServicePrimaryAddr(stream.Context(), utils.TSOServiceName) + if !ok || len(forwardedHost) == 0 { + return errors.WithStack(ErrNotFoundTSOAddr) + } + if forwardStream == nil || lastForwardedHost != forwardedHost { + if cancelForward != nil { + cancelForward() } - select { - case tsDeadlineCh <- dl: - case <-dispatcherCtx.Done(): - return + + clientConn, err := s.getDelegateClient(s.ctx, forwardedHost) + if err != nil { + return errors.WithStack(err) } - err = s.processTSORequests(forwardStream, requests[:pendingTSOReqCount]) - close(done) + forwardStream, forwardCtx, cancelForward, err = + s.createTSOForwardStream(stream.Context(), clientConn) if err != nil { - log.Error("proxy forward tso error", zap.String("forwarded-host", forwardedHost), errs.ZapError(errs.ErrGRPCSend, err)) - select { - case <-dispatcherCtx.Done(): - return - case _, ok := <-doneCh: - if !ok { - return - } - case errCh <- err: - close(errCh) - return + return errors.WithStack(err) + } + lastForwardedHost = forwardedHost + } + + tsopbResp, err := s.forwardTSORequestWithDeadLine( + forwardCtx, cancelForward, forwardStream, request, tsDeadlineCh) + if err != nil { + return errors.WithStack(err) + } + + // The error types defined for tsopb and pdpb are different, so we need to convert them. + var pdpbErr *pdpb.Error + tsopbErr := tsopbResp.GetHeader().GetError() + if tsopbErr != nil { + if tsopbErr.Type == tsopb.ErrorType_OK { + pdpbErr = &pdpb.Error{ + Type: pdpb.ErrorType_OK, + Message: tsopbErr.GetMessage(), + } + } else { + // TODO: specify FORWARD FAILURE error type instead of UNKNOWN. + pdpbErr = &pdpb.Error{ + Type: pdpb.ErrorType_UNKNOWN, + Message: tsopbErr.GetMessage(), } } - case <-dispatcherCtx.Done(): - return + } + + response := &pdpb.TsoResponse{ + Header: &pdpb.ResponseHeader{ + ClusterId: tsopbResp.GetHeader().GetClusterId(), + Error: pdpbErr, + }, + Count: tsopbResp.GetCount(), + Timestamp: tsopbResp.GetTimestamp(), + } + if err := server.Send(response); err != nil { + return errors.WithStack(err) } } } -func (s *GrpcServer) processTSORequests(forwardStream pdpb.PD_TsoClient, requests []*tsoRequest) error { - start := time.Now() - // Merge the requests - count := uint32(0) - for _, request := range requests { - count += request.request.GetCount() - } - req := &pdpb.TsoRequest{ - Header: requests[0].request.GetHeader(), - Count: count, - // TODO: support Local TSO proxy forwarding. - DcLocation: requests[0].request.GetDcLocation(), - } - // Send to the leader stream. - if err := forwardStream.Send(req); err != nil { - return err +func (s *GrpcServer) forwardTSORequestWithDeadLine( + forwardCtx context.Context, + cancelForward context.CancelFunc, + forwardStream tsopb.TSO_TsoClient, + request *pdpb.TsoRequest, + tsDeadlineCh chan<- *tsoutil.TSDeadline, +) (*tsopb.TsoResponse, error) { + done := make(chan struct{}) + dl := tsoutil.NewTSDeadline(tsoutil.DefaultTSOProxyTimeout, done, cancelForward) + select { + case tsDeadlineCh <- dl: + case <-forwardCtx.Done(): + return nil, forwardCtx.Err() } - resp, err := forwardStream.Recv() + + start := time.Now() + resp, err := s.forwardTSORequest(forwardCtx, request, forwardStream) + close(done) if err != nil { - return err + if strings.Contains(err.Error(), errs.NotLeaderErr) { + s.tsoPrimaryWatcher.ForceLoad() + } + return nil, err } + tsoProxyBatchSize.Observe(float64(request.GetCount())) tsoProxyHandleDuration.Observe(time.Since(start).Seconds()) - tsoProxyBatchSize.Observe(float64(count)) - // Split the response - physical, logical, suffixBits := resp.GetTimestamp().GetPhysical(), resp.GetTimestamp().GetLogical(), resp.GetTimestamp().GetSuffixBits() - // `logical` is the largest ts's logical part here, we need to do the subtracting before we finish each TSO request. - // This is different from the logic of client batch, for example, if we have a largest ts whose logical part is 10, - // count is 5, then the splitting results should be 5 and 10. - firstLogical := addLogical(logical, -int64(count), suffixBits) - return s.finishTSORequest(requests, physical, firstLogical, suffixBits) -} - -// Because of the suffix, we need to shift the count before we add it to the logical part. -func addLogical(logical, count int64, suffixBits uint32) int64 { - return logical + count< 0 { - percentage, err := scatterRegions(rc, request.GetRegionsId(), request.GetGroup(), int(request.GetRetryLimit())) + percentage, err := scatterRegions(rc, request.GetRegionsId(), request.GetGroup(), int(request.GetRetryLimit()), request.GetSkipStoreLimit()) if err != nil { return nil, err } @@ -1337,7 +1613,7 @@ func (s *GrpcServer) ScatterRegion(ctx context.Context, request *pdpb.ScatterReg region = core.NewRegionInfo(request.GetRegion(), request.GetLeader()) } - op, err := rc.GetRegionScatter().Scatter(region, request.GetGroup()) + op, err := rc.GetRegionScatter().Scatter(region, request.GetGroup(), request.GetSkipStoreLimit()) if err != nil { return nil, err } @@ -1449,8 +1725,15 @@ func (s *GrpcServer) UpdateServiceGCSafePoint(ctx context.Context, request *pdpb return nil, err } } - - nowTSO, err := s.tsoAllocatorManager.HandleTSORequest(tso.GlobalDCLocation, 1) + var ( + nowTSO pdpb.Timestamp + err error + ) + if s.IsAPIServiceMode() { + nowTSO, err = s.getGlobalTSOFromTSOServer(ctx) + } else { + nowTSO, err = s.tsoAllocatorManager.HandleRequest(tso.GlobalDCLocation, 1) + } if err != nil { return nil, err } @@ -1529,6 +1812,13 @@ func (s *GrpcServer) header() *pdpb.ResponseHeader { return &pdpb.ResponseHeader{ClusterId: s.clusterID} } +func (s *GrpcServer) wrapErrorToHeader(errorType pdpb.ErrorType, message string) *pdpb.ResponseHeader { + return s.errorHeader(&pdpb.Error{ + Type: errorType, + Message: message, + }) +} + func (s *GrpcServer) errorHeader(err *pdpb.Error) *pdpb.ResponseHeader { return &pdpb.ResponseHeader{ ClusterId: s.clusterID, @@ -1564,6 +1854,7 @@ var mockLocalAllocatorLeaderChangeFlag = false // SyncMaxTS will check whether MaxTS is the biggest one among all Local TSOs this PD is holding when skipCheck is set, // and write it into all Local TSO Allocators then if it's indeed the biggest one. func (s *GrpcServer) SyncMaxTS(_ context.Context, request *pdpb.SyncMaxTSRequest) (*pdpb.SyncMaxTSResponse, error) { + // TODO: support local tso forward in api service mode in the future. if err := s.validateInternalRequest(request.GetHeader(), true); err != nil { return nil, err } @@ -1696,7 +1987,7 @@ func (s *GrpcServer) SplitAndScatterRegions(ctx context.Context, request *pdpb.S } rc := s.GetRaftCluster() splitFinishedPercentage, newRegionIDs := rc.GetRegionSplitter().SplitRegions(ctx, request.GetSplitKeys(), int(request.GetRetryLimit())) - scatterFinishedPercentage, err := scatterRegions(rc, newRegionIDs, request.GetGroup(), int(request.GetRetryLimit())) + scatterFinishedPercentage, err := scatterRegions(rc, newRegionIDs, request.GetGroup(), int(request.GetRetryLimit()), false) if err != nil { return nil, err } @@ -1709,8 +2000,8 @@ func (s *GrpcServer) SplitAndScatterRegions(ctx context.Context, request *pdpb.S } // scatterRegions add operators to scatter regions and return the processed percentage and error -func scatterRegions(cluster *cluster.RaftCluster, regionsID []uint64, group string, retryLimit int) (int, error) { - opsCount, failures, err := cluster.GetRegionScatter().ScatterRegionsByID(regionsID, group, retryLimit) +func scatterRegions(cluster *cluster.RaftCluster, regionsID []uint64, group string, retryLimit int, skipStoreLimit bool) (int, error) { + opsCount, failures, err := cluster.GetRegionScatter().ScatterRegionsByID(regionsID, group, retryLimit, skipStoreLimit) if err != nil { return 0, err } @@ -1730,6 +2021,7 @@ func scatterRegions(cluster *cluster.RaftCluster, regionsID []uint64, group stri // GetDCLocationInfo gets the dc-location info of the given dc-location from PD leader's TSO allocator manager. func (s *GrpcServer) GetDCLocationInfo(ctx context.Context, request *pdpb.GetDCLocationInfoRequest) (*pdpb.GetDCLocationInfoResponse, error) { + // TODO: support local tso forward in api service mode in the future. var err error if err = s.validateInternalRequest(request.GetHeader(), false); err != nil { return nil, err @@ -1737,7 +2029,7 @@ func (s *GrpcServer) GetDCLocationInfo(ctx context.Context, request *pdpb.GetDCL if !s.member.IsLeader() { return nil, ErrNotLeader } - am := s.tsoAllocatorManager + am := s.GetTSOAllocatorManager() info, ok := am.GetDCLocationInfo(request.GetDcLocation()) if !ok { am.ClusterDCLocationChecker() @@ -1784,19 +2076,30 @@ func (s *GrpcServer) validateInternalRequest(header *pdpb.RequestHeader, onlyAll func (s *GrpcServer) getDelegateClient(ctx context.Context, forwardedHost string) (*grpc.ClientConn, error) { client, ok := s.clientConns.Load(forwardedHost) - if !ok { - tlsConfig, err := s.GetTLSConfig().ToTLSConfig() - if err != nil { - return nil, err - } - cc, err := grpcutil.GetClientConn(ctx, forwardedHost, tlsConfig) - if err != nil { - return nil, err - } - client = cc - s.clientConns.Store(forwardedHost, cc) + if ok { + // Mostly, the connection is already established, and return it directly. + return client.(*grpc.ClientConn), nil } - return client.(*grpc.ClientConn), nil + + tlsConfig, err := s.GetTLSConfig().ToTLSConfig() + if err != nil { + return nil, err + } + ctxTimeout, cancel := context.WithTimeout(ctx, defaultGRPCDialTimeout) + defer cancel() + newConn, err := grpcutil.GetClientConn(ctxTimeout, forwardedHost, tlsConfig) + if err != nil { + return nil, err + } + conn, loaded := s.clientConns.LoadOrStore(forwardedHost, newConn) + if !loaded { + // Successfully stored the connection we created. + return newConn, nil + } + // Loaded a connection created/stored by another goroutine, so close the one we created + // and return the one we loaded. + newConn.Close() + return conn.(*grpc.ClientConn), nil } func (s *GrpcServer) isLocalRequest(forwardedHost string) bool { @@ -1815,15 +2118,6 @@ func (s *GrpcServer) isLocalRequest(forwardedHost string) bool { return false } -func (s *GrpcServer) createTsoForwardStream(client *grpc.ClientConn) (pdpb.PD_TsoClient, context.CancelFunc, error) { - done := make(chan struct{}) - ctx, cancel := context.WithCancel(s.ctx) - go checkStream(ctx, cancel, done) - forwardStream, err := pdpb.NewPDClient(client).Tso(ctx) - done <- struct{}{} - return forwardStream, cancel, err -} - func (s *GrpcServer) createHeartbeatForwardStream(client *grpc.ClientConn) (pdpb.PD_RegionHeartbeatClient, context.CancelFunc, error) { done := make(chan struct{}) ctx, cancel := context.WithCancel(s.ctx) @@ -1834,6 +2128,7 @@ func (s *GrpcServer) createHeartbeatForwardStream(client *grpc.ClientConn) (pdpb } func forwardRegionHeartbeatClientToServer(forwardStream pdpb.PD_RegionHeartbeatClient, server *heartbeatServer, errCh chan error) { + defer logutil.LogPanic() defer close(errCh) for { resp, err := forwardStream.Recv() @@ -1848,6 +2143,17 @@ func forwardRegionHeartbeatClientToServer(forwardStream pdpb.PD_RegionHeartbeatC } } +func (s *GrpcServer) createTSOForwardStream( + ctx context.Context, client *grpc.ClientConn, +) (tsopb.TSO_TsoClient, context.Context, context.CancelFunc, error) { + done := make(chan struct{}) + forwardCtx, cancelForward := context.WithCancel(ctx) + go checkStream(forwardCtx, cancelForward, done) + forwardStream, err := tsopb.NewTSOClient(client).Tso(forwardCtx) + done <- struct{}{} + return forwardStream, forwardCtx, cancelForward, err +} + func (s *GrpcServer) createReportBucketsForwardStream(client *grpc.ClientConn) (pdpb.PD_ReportBucketsClient, context.CancelFunc, error) { done := make(chan struct{}) ctx, cancel := context.WithCancel(s.ctx) @@ -1858,6 +2164,7 @@ func (s *GrpcServer) createReportBucketsForwardStream(client *grpc.ClientConn) ( } func forwardReportBucketClientToServer(forwardStream pdpb.PD_ReportBucketsClient, server *bucketHeartbeatServer, errCh chan error) { + defer logutil.LogPanic() defer close(errCh) for { resp, err := forwardStream.CloseAndRecv() @@ -1874,16 +2181,101 @@ func forwardReportBucketClientToServer(forwardStream pdpb.PD_ReportBucketsClient // TODO: If goroutine here timeout when tso stream created successfully, we need to handle it correctly. func checkStream(streamCtx context.Context, cancel context.CancelFunc, done chan struct{}) { + defer logutil.LogPanic() + timer := time.NewTimer(3 * time.Second) + defer timer.Stop() select { case <-done: return - case <-time.After(3 * time.Second): + case <-timer.C: cancel() case <-streamCtx.Done(): } <-done } +func (s *GrpcServer) getGlobalTSOFromTSOServer(ctx context.Context) (pdpb.Timestamp, error) { + request := &tsopb.TsoRequest{ + Header: &tsopb.RequestHeader{ + ClusterId: s.clusterID, + KeyspaceId: utils.DefaultKeyspaceID, + KeyspaceGroupId: utils.DefaultKeyspaceGroupID, + }, + Count: 1, + } + var ( + forwardedHost string + forwardStream tsopb.TSO_TsoClient + ts *tsopb.TsoResponse + err error + ) + for i := 0; i < maxRetryTimesRequestTSOServer; i++ { + forwardedHost, ok := s.GetServicePrimaryAddr(ctx, utils.TSOServiceName) + if !ok || forwardedHost == "" { + return pdpb.Timestamp{}, ErrNotFoundTSOAddr + } + forwardStream, err = s.getTSOForwardStream(forwardedHost) + if err != nil { + return pdpb.Timestamp{}, err + } + forwardStream.Send(request) + ts, err = forwardStream.Recv() + if err != nil { + if strings.Contains(err.Error(), errs.NotLeaderErr) { + s.tsoPrimaryWatcher.ForceLoad() + time.Sleep(retryIntervalRequestTSOServer) + continue + } + if strings.Contains(err.Error(), codes.Unavailable.String()) { + s.tsoClientPool.Lock() + delete(s.tsoClientPool.clients, forwardedHost) + s.tsoClientPool.Unlock() + continue + } + log.Error("get global tso from tso service primary addr failed", zap.Error(err), zap.String("tso-addr", forwardedHost)) + return pdpb.Timestamp{}, err + } + return *ts.GetTimestamp(), nil + } + log.Error("get global tso from tso service primary addr failed after retry", zap.Error(err), zap.String("tso-addr", forwardedHost)) + return pdpb.Timestamp{}, err +} + +func (s *GrpcServer) getTSOForwardStream(forwardedHost string) (tsopb.TSO_TsoClient, error) { + s.tsoClientPool.RLock() + forwardStream, ok := s.tsoClientPool.clients[forwardedHost] + s.tsoClientPool.RUnlock() + if ok { + // This is the common case to return here + return forwardStream, nil + } + + s.tsoClientPool.Lock() + defer s.tsoClientPool.Unlock() + + // Double check after entering the critical section + forwardStream, ok = s.tsoClientPool.clients[forwardedHost] + if ok { + return forwardStream, nil + } + + // Now let's create the client connection and the forward stream + client, err := s.getDelegateClient(s.ctx, forwardedHost) + if err != nil { + return nil, err + } + done := make(chan struct{}) + ctx, cancel := context.WithCancel(s.ctx) + go checkStream(ctx, cancel, done) + forwardStream, err = tsopb.NewTSOClient(client).Tso(ctx) + done <- struct{}{} + if err != nil { + return nil, err + } + s.tsoClientPool.clients[forwardedHost] = forwardStream + return forwardStream, nil +} + // for CDC compatibility, we need to initialize config path to `globalConfigPath` const globalConfigPath = "/global/config/" @@ -1891,6 +2283,9 @@ const globalConfigPath = "/global/config/" // Since item value needs to support marshal of different struct types, // it should be set to `Payload bytes` instead of `Value string` func (s *GrpcServer) StoreGlobalConfig(_ context.Context, request *pdpb.StoreGlobalConfigRequest) (*pdpb.StoreGlobalConfigResponse, error) { + if s.client == nil { + return nil, ErrEtcdNotStarted + } configPath := request.GetConfigPath() if configPath == "" { configPath = globalConfigPath @@ -1925,6 +2320,9 @@ func (s *GrpcServer) StoreGlobalConfig(_ context.Context, request *pdpb.StoreGlo // - `Names` iteratively get value from `ConfigPath/Name` but not care about revision // - `ConfigPath` if `Names` is nil can get all values and revision of current path func (s *GrpcServer) LoadGlobalConfig(ctx context.Context, request *pdpb.LoadGlobalConfigRequest) (*pdpb.LoadGlobalConfigResponse, error) { + if s.client == nil { + return nil, ErrEtcdNotStarted + } configPath := request.GetConfigPath() if configPath == "" { configPath = globalConfigPath @@ -1961,6 +2359,9 @@ func (s *GrpcServer) LoadGlobalConfig(ctx context.Context, request *pdpb.LoadGlo // by Etcd.Watch() as long as the context has not been canceled or timed out. // Watch on revision which greater than or equal to the required revision. func (s *GrpcServer) WatchGlobalConfig(req *pdpb.WatchGlobalConfigRequest, server pdpb.PD_WatchGlobalConfigServer) error { + if s.client == nil { + return ErrEtcdNotStarted + } ctx, cancel := context.WithCancel(s.Context()) defer cancel() configPath := req.GetConfigPath() @@ -2012,6 +2413,7 @@ func (s *GrpcServer) WatchGlobalConfig(req *pdpb.WatchGlobalConfigRequest, serve } } } + if len(cfgs) > 0 { if err := server.Send(&pdpb.WatchGlobalConfigResponse{Changes: cfgs, Revision: res.Header.GetRevision()}); err != nil { return err @@ -2084,12 +2486,25 @@ func (s *GrpcServer) SetExternalTimestamp(ctx context.Context, request *pdpb.Set return rsp.(*pdpb.SetExternalTimestampResponse), nil } - timestamp := request.GetTimestamp() - if err := s.SetExternalTS(timestamp); err != nil { + var ( + nowTSO pdpb.Timestamp + err error + ) + if s.IsAPIServiceMode() { + nowTSO, err = s.getGlobalTSOFromTSOServer(ctx) + } else { + nowTSO, err = s.tsoAllocatorManager.HandleRequest(tso.GlobalDCLocation, 1) + } + if err != nil { + return nil, err + } + globalTS := tsoutil.GenerateTS(&nowTSO) + externalTS := request.GetTimestamp() + log.Debug("try to set external timestamp", + zap.Uint64("external-ts", externalTS), zap.Uint64("global-ts", globalTS)) + if err := s.SetExternalTS(externalTS, globalTS); err != nil { return &pdpb.SetExternalTimestampResponse{Header: s.invalidValue(err.Error())}, nil } - log.Debug("set external timestamp", - zap.Uint64("timestamp", timestamp)) return &pdpb.SetExternalTimestampResponse{ Header: s.header(), }, nil diff --git a/server/handler.go b/server/handler.go index 35cec65527f..708a94c1bdc 100644 --- a/server/handler.go +++ b/server/handler.go @@ -34,11 +34,13 @@ import ( "github.com/tikv/pd/pkg/encryption" "github.com/tikv/pd/pkg/errs" "github.com/tikv/pd/pkg/schedule" + sc "github.com/tikv/pd/pkg/schedule/config" "github.com/tikv/pd/pkg/schedule/filter" "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/placement" "github.com/tikv/pd/pkg/schedule/schedulers" "github.com/tikv/pd/pkg/statistics" + "github.com/tikv/pd/pkg/statistics/buckets" "github.com/tikv/pd/pkg/storage" "github.com/tikv/pd/pkg/tso" "github.com/tikv/pd/pkg/utils/apiutil" @@ -100,7 +102,7 @@ func (h *Handler) GetRaftCluster() (*cluster.RaftCluster, error) { } // GetOperatorController returns OperatorController. -func (h *Handler) GetOperatorController() (*schedule.OperatorController, error) { +func (h *Handler) GetOperatorController() (*operator.Controller, error) { rc := h.s.GetRaftCluster() if rc == nil { return nil, errs.ErrNotBootstrapped.GenWithStackByArgs() @@ -114,7 +116,7 @@ func (h *Handler) IsSchedulerPaused(name string) (bool, error) { if err != nil { return false, err } - return rc.IsSchedulerPaused(name) + return rc.GetCoordinator().GetSchedulersController().IsSchedulerPaused(name) } // IsSchedulerDisabled returns whether scheduler is disabled. @@ -123,7 +125,7 @@ func (h *Handler) IsSchedulerDisabled(name string) (bool, error) { if err != nil { return false, err } - return rc.IsSchedulerDisabled(name) + return rc.GetCoordinator().GetSchedulersController().IsSchedulerDisabled(name) } // IsSchedulerExisted returns whether scheduler is existed. @@ -132,11 +134,11 @@ func (h *Handler) IsSchedulerExisted(name string) (bool, error) { if err != nil { return false, err } - return rc.IsSchedulerExisted(name) + return rc.GetCoordinator().GetSchedulersController().IsSchedulerExisted(name) } // GetScheduleConfig returns ScheduleConfig. -func (h *Handler) GetScheduleConfig() *config.ScheduleConfig { +func (h *Handler) GetScheduleConfig() *sc.ScheduleConfig { return h.s.GetScheduleConfig() } @@ -155,7 +157,7 @@ func (h *Handler) IsCheckerPaused(name string) (bool, error) { if err != nil { return false, err } - return rc.IsCheckerPaused(name) + return rc.GetCoordinator().IsCheckerPaused(name) } // GetStores returns all stores in the cluster. @@ -186,6 +188,16 @@ func (h *Handler) GetHotWriteRegions() *statistics.StoreHotPeersInfos { return c.GetHotWriteRegions() } +// GetHotBuckets returns all hot buckets stats. +func (h *Handler) GetHotBuckets(regionIDs ...uint64) map[uint64][]*buckets.BucketStat { + c, err := h.GetRaftCluster() + if err != nil { + return nil + } + degree := c.GetOpts().GetHotRegionCacheHitsThreshold() + return c.BucketsStats(degree, regionIDs...) +} + // GetHotReadRegions gets all hot read regions stats. func (h *Handler) GetHotReadRegions() *statistics.StoreHotPeersInfos { c, err := h.GetRaftCluster() @@ -221,7 +233,7 @@ func (h *Handler) AddScheduler(name string, args ...string) error { return err } - s, err := schedule.CreateScheduler(name, c.GetOperatorController(), h.s.storage, schedule.ConfigSliceDecoder(name, args)) + s, err := schedulers.CreateScheduler(name, c.GetOperatorController(), h.s.storage, schedulers.ConfigSliceDecoder(name, args), c.GetCoordinator().GetSchedulersController().RemoveScheduler) if err != nil { return err } @@ -393,7 +405,7 @@ func (h *Handler) GetOperator(regionID uint64) (*operator.Operator, error) { } // GetOperatorStatus returns the status of the region operator. -func (h *Handler) GetOperatorStatus(regionID uint64) (*schedule.OperatorWithStatus, error) { +func (h *Handler) GetOperatorStatus(regionID uint64) (*operator.OpWithStatus, error) { c, err := h.GetOperatorController() if err != nil { return nil, err @@ -419,7 +431,7 @@ func (h *Handler) RemoveOperator(regionID uint64) error { return ErrOperatorNotFound } - _ = c.RemoveOperator(op) + _ = c.RemoveOperator(op, operator.AdminStop) return nil } @@ -532,7 +544,7 @@ func (h *Handler) SetLabelStoresLimit(ratePerMin float64, limitType storelimit.T } // GetAllStoresLimit is used to get limit of all stores. -func (h *Handler) GetAllStoresLimit(limitType storelimit.Type) (map[uint64]config.StoreLimitConfig, error) { +func (h *Handler) GetAllStoresLimit(limitType storelimit.Type) (map[uint64]sc.StoreLimitConfig, error) { c, err := h.GetRaftCluster() if err != nil { return nil, err @@ -843,7 +855,7 @@ func (h *Handler) AddScatterRegionOperator(regionID uint64, group string) error return errors.Errorf("region %d is a hot region", regionID) } - op, err := c.GetRegionScatter().Scatter(region, group) + op, err := c.GetRegionScatter().Scatter(region, group, false) if err != nil { return err } @@ -880,7 +892,7 @@ func (h *Handler) AddScatterRegionsOperators(regionIDs []uint64, startRawKey, en return 0, err } } else { - opsCount, failures, err = c.GetRegionScatter().ScatterRegionsByID(regionIDs, group, retryLimit) + opsCount, failures, err = c.GetRegionScatter().ScatterRegionsByID(regionIDs, group, retryLimit, false) if err != nil { return 0, err } @@ -916,17 +928,8 @@ func (h *Handler) GetSchedulerConfigHandler() (http.Handler, error) { return mux, nil } -// GetOfflinePeer gets the region with offline peer. -func (h *Handler) GetOfflinePeer(typ statistics.RegionStatisticType) ([]*core.RegionInfo, error) { - c := h.s.GetRaftCluster() - if c == nil { - return nil, errs.ErrNotBootstrapped.FastGenByArgs() - } - return c.GetOfflineRegionStatsByType(typ), nil -} - // ResetTS resets the ts with specified tso. -func (h *Handler) ResetTS(ts uint64, ignoreSmaller, skipUpperBoundCheck bool) error { +func (h *Handler) ResetTS(ts uint64, ignoreSmaller, skipUpperBoundCheck bool, _ uint32) error { log.Info("reset-ts", zap.Uint64("new-ts", ts), zap.Bool("ignore-smaller", ignoreSmaller), @@ -983,7 +986,7 @@ func (h *Handler) PluginUnload(pluginPath string) error { h.pluginChMapLock.Lock() defer h.pluginChMapLock.Unlock() if ch, ok := h.pluginChMap[pluginPath]; ok { - ch <- cluster.PluginUnload + ch <- schedule.PluginUnload return nil } return ErrPluginNotFound(pluginPath) diff --git a/server/keyspace/keyspace.go b/server/keyspace/keyspace.go deleted file mode 100644 index bf64f3e680b..00000000000 --- a/server/keyspace/keyspace.go +++ /dev/null @@ -1,486 +0,0 @@ -// Copyright 2022 TiKV Project Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package keyspace - -import ( - "context" - "time" - - "github.com/pingcap/errors" - "github.com/pingcap/failpoint" - "github.com/pingcap/kvproto/pkg/keyspacepb" - "github.com/pingcap/log" - "github.com/tikv/pd/pkg/id" - "github.com/tikv/pd/pkg/slice" - "github.com/tikv/pd/pkg/storage/endpoint" - "github.com/tikv/pd/pkg/storage/kv" - "github.com/tikv/pd/pkg/utils/syncutil" - "github.com/tikv/pd/server/cluster" - "github.com/tikv/pd/server/config" - "go.uber.org/zap" -) - -const ( - // AllocStep set idAllocator's step when write persistent window boundary. - // Use a lower value for denser idAllocation in the event of frequent pd leader change. - AllocStep = uint64(100) - // AllocLabel is used to label keyspace idAllocator's metrics. - AllocLabel = "keyspace-idAlloc" - // DefaultKeyspaceName is the name reserved for default keyspace. - DefaultKeyspaceName = "DEFAULT" - // DefaultKeyspaceID is the id of default keyspace. - DefaultKeyspaceID = uint32(0) - // regionLabelIDPrefix is used to prefix the keyspace region label. - regionLabelIDPrefix = "keyspaces/" - // regionLabelKey is the key for keyspace id in keyspace region label. - regionLabelKey = "id" -) - -// Manager manages keyspace related data. -// It validates requests and provides concurrency control. -type Manager struct { - // metaLock guards keyspace meta. - metaLock *syncutil.LockGroup - // idAllocator allocates keyspace id. - idAllocator id.Allocator - // store is the storage for keyspace related information. - store endpoint.KeyspaceStorage - // rc is the raft cluster of the server. - rc *cluster.RaftCluster - // ctx is the context of the manager, to be used in transaction. - ctx context.Context - // config is the configurations of the manager. - config config.KeyspaceConfig -} - -// CreateKeyspaceRequest represents necessary arguments to create a keyspace. -type CreateKeyspaceRequest struct { - // Name of the keyspace to be created. - // Using an existing name will result in error. - Name string - Config map[string]string - // Now is the timestamp used to record creation time. - Now int64 -} - -// NewKeyspaceManager creates a Manager of keyspace related data. -func NewKeyspaceManager(store endpoint.KeyspaceStorage, - rc *cluster.RaftCluster, - idAllocator id.Allocator, - config config.KeyspaceConfig, -) *Manager { - return &Manager{ - metaLock: syncutil.NewLockGroup(syncutil.WithHash(keyspaceIDHash)), - idAllocator: idAllocator, - store: store, - rc: rc, - ctx: context.TODO(), - config: config, - } -} - -// Bootstrap saves default keyspace info. -func (manager *Manager) Bootstrap() error { - // Split Keyspace Region for default keyspace. - if err := manager.splitKeyspaceRegion(DefaultKeyspaceID); err != nil { - return err - } - now := time.Now().Unix() - defaultKeyspace := &keyspacepb.KeyspaceMeta{ - Id: DefaultKeyspaceID, - Name: DefaultKeyspaceName, - State: keyspacepb.KeyspaceState_ENABLED, - CreatedAt: now, - StateChangedAt: now, - } - err := manager.saveNewKeyspace(defaultKeyspace) - // It's possible that default keyspace already exists in the storage (e.g. PD restart/recover), - // so we ignore the keyspaceExists error. - if err != nil && err != ErrKeyspaceExists { - return err - } - - // Initialize pre-alloc keyspace. - preAlloc := manager.config.PreAlloc - for _, keyspaceName := range preAlloc { - _, err = manager.CreateKeyspace(&CreateKeyspaceRequest{ - Name: keyspaceName, - Now: now, - }) - // Ignore the keyspaceExists error for the same reason as saving default keyspace. - if err != nil && err != ErrKeyspaceExists { - return err - } - } - return nil -} - -// CreateKeyspace create a keyspace meta with given config and save it to storage. -func (manager *Manager) CreateKeyspace(request *CreateKeyspaceRequest) (*keyspacepb.KeyspaceMeta, error) { - // Validate purposed name's legality. - if err := validateName(request.Name); err != nil { - return nil, err - } - // Allocate new keyspaceID. - newID, err := manager.allocID() - if err != nil { - return nil, err - } - // Split keyspace region. - err = manager.splitKeyspaceRegion(newID) - if err != nil { - return nil, err - } - // Create and save keyspace metadata. - keyspace := &keyspacepb.KeyspaceMeta{ - Id: newID, - Name: request.Name, - State: keyspacepb.KeyspaceState_ENABLED, - CreatedAt: request.Now, - StateChangedAt: request.Now, - Config: request.Config, - } - err = manager.saveNewKeyspace(keyspace) - if err != nil { - log.Warn("[keyspace] failed to create keyspace", - zap.Uint32("ID", keyspace.GetId()), - zap.String("name", keyspace.GetName()), - zap.Error(err), - ) - return nil, err - } - log.Info("[keyspace] keyspace created", - zap.Uint32("ID", keyspace.GetId()), - zap.String("name", keyspace.GetName()), - ) - return keyspace, nil -} - -func (manager *Manager) saveNewKeyspace(keyspace *keyspacepb.KeyspaceMeta) error { - manager.metaLock.Lock(keyspace.Id) - defer manager.metaLock.Unlock(keyspace.Id) - - return manager.store.RunInTxn(manager.ctx, func(txn kv.Txn) error { - // Save keyspace ID. - // Check if keyspace with that name already exists. - nameExists, _, err := manager.store.LoadKeyspaceID(txn, keyspace.Name) - if err != nil { - return err - } - if nameExists { - return ErrKeyspaceExists - } - err = manager.store.SaveKeyspaceID(txn, keyspace.Id, keyspace.Name) - if err != nil { - return err - } - // Save keyspace meta. - // Check if keyspace with that id already exists. - loadedMeta, err := manager.store.LoadKeyspaceMeta(txn, keyspace.Id) - if err != nil { - return err - } - if loadedMeta != nil { - return ErrKeyspaceExists - } - return manager.store.SaveKeyspaceMeta(txn, keyspace) - }) -} - -// splitKeyspaceRegion add keyspace's boundaries to region label. The corresponding -// region will then be split by Coordinator's patrolRegion. -func (manager *Manager) splitKeyspaceRegion(id uint32) error { - failpoint.Inject("skipSplitRegion", func() { - failpoint.Return(nil) - }) - - keyspaceRule := makeLabelRule(id) - err := manager.rc.GetRegionLabeler().SetLabelRule(keyspaceRule) - if err != nil { - log.Warn("[keyspace] failed to add region label for keyspace", - zap.Uint32("keyspaceID", id), - zap.Error(err), - ) - } - log.Info("[keyspace] added region label for keyspace", - zap.Uint32("keyspaceID", id), - zap.Any("LabelRule", keyspaceRule), - ) - return nil -} - -// LoadKeyspace returns the keyspace specified by name. -// It returns error if loading or unmarshalling met error or if keyspace does not exist. -func (manager *Manager) LoadKeyspace(name string) (*keyspacepb.KeyspaceMeta, error) { - var meta *keyspacepb.KeyspaceMeta - err := manager.store.RunInTxn(manager.ctx, func(txn kv.Txn) error { - loaded, id, err := manager.store.LoadKeyspaceID(txn, name) - if err != nil { - return err - } - if !loaded { - return ErrKeyspaceNotFound - } - meta, err = manager.store.LoadKeyspaceMeta(txn, id) - if err != nil { - return err - } - if meta == nil { - return ErrKeyspaceNotFound - } - meta.Id = id - return nil - }) - return meta, err -} - -// LoadKeyspaceByID returns the keyspace specified by id. -// It returns error if loading or unmarshalling met error or if keyspace does not exist. -func (manager *Manager) LoadKeyspaceByID(spaceID uint32) (*keyspacepb.KeyspaceMeta, error) { - var ( - meta *keyspacepb.KeyspaceMeta - err error - ) - err = manager.store.RunInTxn(manager.ctx, func(txn kv.Txn) error { - meta, err = manager.store.LoadKeyspaceMeta(txn, spaceID) - if err != nil { - return err - } - if meta == nil { - return ErrKeyspaceNotFound - } - return nil - }) - meta.Id = spaceID - return meta, err -} - -// Mutation represents a single operation to be applied on keyspace config. -type Mutation struct { - Op OpType - Key string - Value string -} - -// OpType defines the type of keyspace config operation. -type OpType int - -const ( - // OpPut denotes a put operation onto the given config. - // If target key exists, it will put a new value, - // otherwise, it creates a new config entry. - OpPut OpType = iota + 1 // Operation type starts at 1. - // OpDel denotes a deletion operation onto the given config. - // Note: OpDel is idempotent, deleting a non-existing key - // will not result in error. - OpDel -) - -// UpdateKeyspaceConfig changes target keyspace's config in the order specified in mutations. -// It returns error if saving failed, operation not allowed, or if keyspace not exists. -func (manager *Manager) UpdateKeyspaceConfig(name string, mutations []*Mutation) (*keyspacepb.KeyspaceMeta, error) { - var meta *keyspacepb.KeyspaceMeta - err := manager.store.RunInTxn(manager.ctx, func(txn kv.Txn) error { - // First get KeyspaceID from Name. - loaded, id, err := manager.store.LoadKeyspaceID(txn, name) - if err != nil { - return err - } - if !loaded { - return ErrKeyspaceNotFound - } - manager.metaLock.Lock(id) - defer manager.metaLock.Unlock(id) - // Load keyspace by id. - meta, err = manager.store.LoadKeyspaceMeta(txn, id) - if err != nil { - return err - } - if meta == nil { - return ErrKeyspaceNotFound - } - // Only keyspace with state listed in allowChangeConfig are allowed to change their config. - if !slice.Contains(allowChangeConfig, meta.GetState()) { - return errors.Errorf("cannot change config for keyspace with state %s", meta.GetState().String()) - } - // Initialize meta's config map if it's nil. - if meta.GetConfig() == nil { - meta.Config = map[string]string{} - } - // Update keyspace config according to mutations. - for _, mutation := range mutations { - switch mutation.Op { - case OpPut: - meta.Config[mutation.Key] = mutation.Value - case OpDel: - delete(meta.Config, mutation.Key) - default: - return errIllegalOperation - } - } - // Save the updated keyspace meta. - return manager.store.SaveKeyspaceMeta(txn, meta) - }) - - if err != nil { - log.Warn("[keyspace] failed to update keyspace config", - zap.Uint32("ID", meta.GetId()), - zap.String("name", meta.GetName()), - zap.Error(err), - ) - return nil, err - } - log.Info("[keyspace] keyspace config updated", - zap.Uint32("ID", meta.GetId()), - zap.String("name", meta.GetName()), - zap.Any("new config", meta.GetConfig()), - ) - return meta, nil -} - -// UpdateKeyspaceState updates target keyspace to the given state if it's not already in that state. -// It returns error if saving failed, operation not allowed, or if keyspace not exists. -func (manager *Manager) UpdateKeyspaceState(name string, newState keyspacepb.KeyspaceState, now int64) (*keyspacepb.KeyspaceMeta, error) { - // Changing the state of default keyspace is not allowed. - if name == DefaultKeyspaceName { - log.Warn("[keyspace] failed to update keyspace config", - zap.Error(errModifyDefault), - ) - return nil, errModifyDefault - } - var meta *keyspacepb.KeyspaceMeta - err := manager.store.RunInTxn(manager.ctx, func(txn kv.Txn) error { - // First get KeyspaceID from Name. - loaded, id, err := manager.store.LoadKeyspaceID(txn, name) - if err != nil { - return err - } - if !loaded { - return ErrKeyspaceNotFound - } - manager.metaLock.Lock(id) - defer manager.metaLock.Unlock(id) - // Load keyspace by id. - meta, err = manager.store.LoadKeyspaceMeta(txn, id) - if err != nil { - return err - } - if meta == nil { - return ErrKeyspaceNotFound - } - // Update keyspace meta. - if err = updateKeyspaceState(meta, newState, now); err != nil { - return err - } - return manager.store.SaveKeyspaceMeta(txn, meta) - }) - if err != nil { - log.Warn("[keyspace] failed to update keyspace config", - zap.Uint32("ID", meta.GetId()), - zap.String("name", meta.GetName()), - zap.Error(err), - ) - return nil, err - } - log.Info("[keyspace] keyspace state updated", - zap.Uint32("ID", meta.GetId()), - zap.String("name", meta.GetName()), - zap.String("new state", newState.String()), - ) - return meta, nil -} - -// UpdateKeyspaceStateByID updates target keyspace to the given state if it's not already in that state. -// It returns error if saving failed, operation not allowed, or if keyspace not exists. -func (manager *Manager) UpdateKeyspaceStateByID(id uint32, newState keyspacepb.KeyspaceState, now int64) (*keyspacepb.KeyspaceMeta, error) { - // Changing the state of default keyspace is not allowed. - if id == DefaultKeyspaceID { - log.Warn("[keyspace] failed to update keyspace config", - zap.Error(errModifyDefault), - ) - return nil, errModifyDefault - } - var meta *keyspacepb.KeyspaceMeta - var err error - err = manager.store.RunInTxn(manager.ctx, func(txn kv.Txn) error { - manager.metaLock.Lock(id) - defer manager.metaLock.Unlock(id) - // Load keyspace by id. - meta, err = manager.store.LoadKeyspaceMeta(txn, id) - if err != nil { - return err - } - if meta == nil { - return ErrKeyspaceNotFound - } - // Update keyspace meta. - if err = updateKeyspaceState(meta, newState, now); err != nil { - return err - } - return manager.store.SaveKeyspaceMeta(txn, meta) - }) - if err != nil { - log.Warn("[keyspace] failed to update keyspace config", - zap.Uint32("ID", meta.GetId()), - zap.String("name", meta.GetName()), - zap.Error(err), - ) - return nil, err - } - log.Info("[keyspace] keyspace state updated", - zap.Uint32("ID", meta.GetId()), - zap.String("name", meta.GetName()), - zap.String("new state", newState.String()), - ) - return meta, nil -} - -// updateKeyspaceState updates keyspace meta and record the update time. -func updateKeyspaceState(meta *keyspacepb.KeyspaceMeta, newState keyspacepb.KeyspaceState, now int64) error { - // If already in the target state, do nothing and return. - if meta.GetState() == newState { - return nil - } - // Consult state transition table to check if the operation is legal. - if !slice.Contains(stateTransitionTable[meta.GetState()], newState) { - return errors.Errorf("cannot change keyspace state from %s to %s", meta.GetState().String(), newState.String()) - } - // If the operation is legal, update keyspace state and change time. - meta.State = newState - meta.StateChangedAt = now - return nil -} - -// LoadRangeKeyspace load up to limit keyspaces starting from keyspace with startID. -func (manager *Manager) LoadRangeKeyspace(startID uint32, limit int) ([]*keyspacepb.KeyspaceMeta, error) { - // Load Start should fall within acceptable ID range. - if startID > spaceIDMax { - return nil, errors.Errorf("startID of the scan %d exceeds spaceID Max %d", startID, spaceIDMax) - } - return manager.store.LoadRangeKeyspace(startID, limit) -} - -// allocID allocate a new keyspace id. -func (manager *Manager) allocID() (uint32, error) { - id64, err := manager.idAllocator.Alloc() - if err != nil { - return 0, err - } - id32 := uint32(id64) - if err = validateID(id32); err != nil { - return 0, err - } - return id32, nil -} diff --git a/server/keyspace/util.go b/server/keyspace/util.go deleted file mode 100644 index a3402846d03..00000000000 --- a/server/keyspace/util.go +++ /dev/null @@ -1,149 +0,0 @@ -// Copyright 2022 TiKV Project Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package keyspace - -import ( - "encoding/binary" - "encoding/hex" - "regexp" - "strconv" - - "github.com/pingcap/errors" - "github.com/pingcap/kvproto/pkg/keyspacepb" - "github.com/tikv/pd/pkg/codec" - "github.com/tikv/pd/pkg/schedule/labeler" - "github.com/tikv/pd/pkg/storage/endpoint" -) - -const ( - spaceIDMax = ^uint32(0) >> 8 // 16777215 (Uint24Max) is the maximum value of spaceID. - // namePattern is a regex that specifies acceptable characters of the keyspace name. - // Name must be non-empty and contains only alphanumerical, `_` and `-`. - namePattern = "^[-A-Za-z0-9_]+$" -) - -var ( - // ErrKeyspaceNotFound is used to indicate target keyspace does not exist. - ErrKeyspaceNotFound = errors.New("keyspace does not exist") - // ErrKeyspaceExists indicates target keyspace already exists. - // Used when creating a new keyspace. - ErrKeyspaceExists = errors.New("keyspace already exists") - errModifyDefault = errors.New("cannot modify default keyspace's state") - errIllegalOperation = errors.New("unknown operation") - - // stateTransitionTable lists all allowed next state for the given current state. - // Note that transit from any state to itself is allowed for idempotence. - stateTransitionTable = map[keyspacepb.KeyspaceState][]keyspacepb.KeyspaceState{ - keyspacepb.KeyspaceState_ENABLED: {keyspacepb.KeyspaceState_ENABLED, keyspacepb.KeyspaceState_DISABLED}, - keyspacepb.KeyspaceState_DISABLED: {keyspacepb.KeyspaceState_DISABLED, keyspacepb.KeyspaceState_ENABLED, keyspacepb.KeyspaceState_ARCHIVED}, - keyspacepb.KeyspaceState_ARCHIVED: {keyspacepb.KeyspaceState_ARCHIVED, keyspacepb.KeyspaceState_TOMBSTONE}, - keyspacepb.KeyspaceState_TOMBSTONE: {keyspacepb.KeyspaceState_TOMBSTONE}, - } - // Only keyspaces in the state specified by allowChangeConfig are allowed to change their config. - allowChangeConfig = []keyspacepb.KeyspaceState{keyspacepb.KeyspaceState_ENABLED, keyspacepb.KeyspaceState_DISABLED} -) - -// validateID check if keyspace falls within the acceptable range. -// It throws errIllegalID when input id is our of range, -// or if it collides with reserved id. -func validateID(id uint32) error { - if id > spaceIDMax { - return errors.Errorf("illegal keyspace id %d, larger than spaceID Max %d", id, spaceIDMax) - } - if id == DefaultKeyspaceID { - return errors.Errorf("illegal keyspace id %d, collides with default keyspace id", id) - } - return nil -} - -// validateName check if user provided name is legal. -// It throws errIllegalName when name contains illegal character, -// or if it collides with reserved name. -func validateName(name string) error { - isValid, err := regexp.MatchString(namePattern, name) - if err != nil { - return err - } - if !isValid { - return errors.Errorf("illegal keyspace name %s, should contain only alphanumerical and underline", name) - } - if name == DefaultKeyspaceName { - return errors.Errorf("illegal keyspace name %s, collides with default keyspace name", name) - } - return nil -} - -// keyspaceIDHash is used to hash the spaceID inside the lockGroup. -// A simple mask is applied to spaceID to use its last byte as map key, -// limiting the maximum map length to 256. -// Since keyspaceID is sequentially allocated, this can also reduce the chance -// of collision when comparing with random hashes. -func keyspaceIDHash(id uint32) uint32 { - return id & 0xFF -} - -// makeKeyRanges encodes keyspace ID to correct LabelRule data. -// For a keyspace with id ['a', 'b', 'c'], it has four boundaries: -// -// Lower bound for raw mode: ['r', 'a', 'b', 'c'] -// Upper bound for raw mode: ['r', 'a', 'b', 'c + 1'] -// Lower bound for txn mode: ['x', 'a', 'b', 'c'] -// Upper bound for txn mode: ['x', 'a', 'b', 'c + 1'] -// -// From which it shares the lower bound with keyspace with id ['a', 'b', 'c-1']. -// And shares upper bound with keyspace with id ['a', 'b', 'c + 1']. -// These repeated bound will not cause any problem, as repetitive bound will be ignored during rangeListBuild, -// but provides guard against hole in keyspace allocations should it occur. -func makeKeyRanges(id uint32) []interface{} { - keyspaceIDBytes := make([]byte, 4) - nextKeyspaceIDBytes := make([]byte, 4) - binary.BigEndian.PutUint32(keyspaceIDBytes, id) - binary.BigEndian.PutUint32(nextKeyspaceIDBytes, id+1) - rawLeftBound := hex.EncodeToString(codec.EncodeBytes(append([]byte{'r'}, keyspaceIDBytes[1:]...))) - rawRightBound := hex.EncodeToString(codec.EncodeBytes(append([]byte{'r'}, nextKeyspaceIDBytes[1:]...))) - txnLeftBound := hex.EncodeToString(codec.EncodeBytes(append([]byte{'x'}, keyspaceIDBytes[1:]...))) - txnRightBound := hex.EncodeToString(codec.EncodeBytes(append([]byte{'x'}, nextKeyspaceIDBytes[1:]...))) - return []interface{}{ - map[string]interface{}{ - "start_key": rawLeftBound, - "end_key": rawRightBound, - }, - map[string]interface{}{ - "start_key": txnLeftBound, - "end_key": txnRightBound, - }, - } -} - -// getRegionLabelID returns the region label id of the target keyspace. -func getRegionLabelID(id uint32) string { - return regionLabelIDPrefix + strconv.FormatUint(uint64(id), endpoint.SpaceIDBase) -} - -// makeLabelRule makes the label rule for the given keyspace id. -func makeLabelRule(id uint32) *labeler.LabelRule { - return &labeler.LabelRule{ - ID: getRegionLabelID(id), - Index: 0, - Labels: []labeler.RegionLabel{ - { - Key: regionLabelKey, - Value: strconv.FormatUint(uint64(id), endpoint.SpaceIDBase), - }, - }, - RuleType: labeler.KeyRange, - Data: makeKeyRanges(id), - } -} diff --git a/server/keyspace_service.go b/server/keyspace_service.go index 5fb06c38bdb..b17239ba0a4 100644 --- a/server/keyspace_service.go +++ b/server/keyspace_service.go @@ -22,9 +22,11 @@ import ( "github.com/gogo/protobuf/proto" "github.com/pingcap/kvproto/pkg/keyspacepb" "github.com/pingcap/kvproto/pkg/pdpb" + "github.com/tikv/pd/pkg/keyspace" "github.com/tikv/pd/pkg/storage/endpoint" - "github.com/tikv/pd/server/keyspace" + "github.com/tikv/pd/pkg/utils/etcdutil" "go.etcd.io/etcd/clientv3" + "go.etcd.io/etcd/mvcc/mvccpb" ) // KeyspaceServer wraps GrpcServer to provide keyspace service. @@ -52,10 +54,6 @@ func (s *KeyspaceServer) LoadKeyspace(_ context.Context, request *keyspacepb.Loa if err := s.validateRequest(request.GetHeader()); err != nil { return nil, err } - rc := s.GetRaftCluster() - if rc == nil { - return &keyspacepb.LoadKeyspaceResponse{Header: s.notBootstrappedHeader()}, nil - } manager := s.GetKeyspaceManager() meta, err := manager.LoadKeyspace(request.GetName()) @@ -74,58 +72,56 @@ func (s *KeyspaceServer) WatchKeyspaces(request *keyspacepb.WatchKeyspacesReques if err := s.validateRequest(request.GetHeader()); err != nil { return err } - rc := s.GetRaftCluster() - if rc == nil { - return stream.Send(&keyspacepb.WatchKeyspacesResponse{Header: s.notBootstrappedHeader()}) - } - ctx, cancel := context.WithCancel(s.Context()) defer cancel() + startKey := path.Join(s.rootPath, endpoint.KeyspaceMetaPrefix()) + "/" - err := s.sendAllKeyspaceMeta(ctx, stream) - if err != nil { - return err - } - watchChan := s.client.Watch(ctx, path.Join(s.rootPath, endpoint.KeyspaceMetaPrefix()), clientv3.WithPrefix()) - for { - select { - case <-ctx.Done(): - return nil - case res := <-watchChan: - keyspaces := make([]*keyspacepb.KeyspaceMeta, 0, len(res.Events)) - for _, event := range res.Events { - if event.Type != clientv3.EventTypePut { - continue - } - meta := &keyspacepb.KeyspaceMeta{} - if err = proto.Unmarshal(event.Kv.Value, meta); err != nil { - return err - } - keyspaces = append(keyspaces, meta) - } - if len(keyspaces) > 0 { - if err = stream.Send(&keyspacepb.WatchKeyspacesResponse{Header: s.header(), Keyspaces: keyspaces}); err != nil { - return err - } - } + keyspaces := make([]*keyspacepb.KeyspaceMeta, 0) + putFn := func(kv *mvccpb.KeyValue) error { + meta := &keyspacepb.KeyspaceMeta{} + if err := proto.Unmarshal(kv.Value, meta); err != nil { + defer cancel() // cancel context to stop watcher + return err } + keyspaces = append(keyspaces, meta) + return nil } -} - -func (s *KeyspaceServer) sendAllKeyspaceMeta(ctx context.Context, stream keyspacepb.Keyspace_WatchKeyspacesServer) error { - getResp, err := s.client.Get(ctx, path.Join(s.rootPath, endpoint.KeyspaceMetaPrefix()), clientv3.WithPrefix()) - if err != nil { - return err + deleteFn := func(kv *mvccpb.KeyValue) error { + return nil } - metas := make([]*keyspacepb.KeyspaceMeta, getResp.Count) - for i, kv := range getResp.Kvs { - meta := &keyspacepb.KeyspaceMeta{} - if err = proto.Unmarshal(kv.Value, meta); err != nil { + postEventFn := func() error { + defer func() { + keyspaces = keyspaces[:0] + }() + err := stream.Send(&keyspacepb.WatchKeyspacesResponse{ + Header: s.header(), + Keyspaces: keyspaces}) + if err != nil { + defer cancel() // cancel context to stop watcher return err } - metas[i] = meta + return nil + } + + watcher := etcdutil.NewLoopWatcher( + ctx, + &s.serverLoopWg, + s.client, + "keyspace-server-watcher", + startKey, + putFn, + deleteFn, + postEventFn, + clientv3.WithRange(clientv3.GetPrefixRangeEnd(startKey)), + ) + watcher.StartWatchLoop() + if err := watcher.WaitLoad(); err != nil { + cancel() // cancel context to stop watcher + return err } - return stream.Send(&keyspacepb.WatchKeyspacesResponse{Header: s.header(), Keyspaces: metas}) + + <-ctx.Done() // wait for context done + return nil } // UpdateKeyspaceState updates the state of keyspace specified in the request. @@ -133,10 +129,7 @@ func (s *KeyspaceServer) UpdateKeyspaceState(_ context.Context, request *keyspac if err := s.validateRequest(request.GetHeader()); err != nil { return nil, err } - rc := s.GetRaftCluster() - if rc == nil { - return &keyspacepb.UpdateKeyspaceStateResponse{Header: s.notBootstrappedHeader()}, nil - } + manager := s.GetKeyspaceManager() meta, err := manager.UpdateKeyspaceStateByID(request.GetId(), request.GetState(), time.Now().Unix()) if err != nil { @@ -147,3 +140,21 @@ func (s *KeyspaceServer) UpdateKeyspaceState(_ context.Context, request *keyspac Keyspace: meta, }, nil } + +// GetAllKeyspaces get all keyspace's metadata. +func (s *KeyspaceServer) GetAllKeyspaces(_ context.Context, request *keyspacepb.GetAllKeyspacesRequest) (*keyspacepb.GetAllKeyspacesResponse, error) { + if err := s.validateRequest(request.GetHeader()); err != nil { + return nil, err + } + + manager := s.GetKeyspaceManager() + keyspaces, err := manager.LoadRangeKeyspace(request.StartId, int(request.Limit)) + if err != nil { + return &keyspacepb.GetAllKeyspacesResponse{Header: s.getErrorHeader(err)}, nil + } + + return &keyspacepb.GetAllKeyspacesResponse{ + Header: s.header(), + Keyspaces: keyspaces, + }, nil +} diff --git a/server/metrics.go b/server/metrics.go index 7eed1020186..94eb9bf19a2 100644 --- a/server/metrics.go +++ b/server/metrics.go @@ -82,6 +82,14 @@ var ( Buckets: prometheus.ExponentialBuckets(1, 2, 13), }) + tsoProxyForwardTimeoutCounter = prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: "pd", + Subsystem: "server", + Name: "tso_proxy_forward_timeout_total", + Help: "Counter of timeouts when tso proxy forwarding tso requests to tso service.", + }) + tsoHandleDuration = prometheus.NewHistogram( prometheus.HistogramOpts{ Namespace: "pd", @@ -161,6 +169,7 @@ func init() { prometheus.MustRegister(etcdStateGauge) prometheus.MustRegister(tsoProxyHandleDuration) prometheus.MustRegister(tsoProxyBatchSize) + prometheus.MustRegister(tsoProxyForwardTimeoutCounter) prometheus.MustRegister(tsoHandleDuration) prometheus.MustRegister(regionHeartbeatHandleDuration) prometheus.MustRegister(storeHeartbeatHandleDuration) diff --git a/server/server.go b/server/server.go index 6c4205d3420..1a0a80450d6 100644 --- a/server/server.go +++ b/server/server.go @@ -21,7 +21,6 @@ import ( "math/rand" "net/http" "os" - "path" "path/filepath" "runtime" "strconv" @@ -31,6 +30,7 @@ import ( "time" "github.com/coreos/go-semver/semver" + "github.com/gogo/protobuf/proto" "github.com/gorilla/mux" "github.com/pingcap/errors" "github.com/pingcap/failpoint" @@ -38,27 +38,32 @@ import ( "github.com/pingcap/kvproto/pkg/keyspacepb" "github.com/pingcap/kvproto/pkg/metapb" "github.com/pingcap/kvproto/pkg/pdpb" + "github.com/pingcap/kvproto/pkg/tsopb" "github.com/pingcap/log" "github.com/pingcap/sysutil" "github.com/tikv/pd/pkg/audit" - bs "github.com/tikv/pd/pkg/basicserver" "github.com/tikv/pd/pkg/core" "github.com/tikv/pd/pkg/encryption" "github.com/tikv/pd/pkg/errs" + "github.com/tikv/pd/pkg/gc" "github.com/tikv/pd/pkg/id" - ms_server "github.com/tikv/pd/pkg/mcs/meta_storage/server" + "github.com/tikv/pd/pkg/keyspace" + ms_server "github.com/tikv/pd/pkg/mcs/metastorage/server" "github.com/tikv/pd/pkg/mcs/registry" - rm_server "github.com/tikv/pd/pkg/mcs/resource_manager/server" - _ "github.com/tikv/pd/pkg/mcs/resource_manager/server/apis/v1" // init API group - _ "github.com/tikv/pd/pkg/mcs/tso/server/apis/v1" // init tso API group + rm_server "github.com/tikv/pd/pkg/mcs/resourcemanager/server" + _ "github.com/tikv/pd/pkg/mcs/resourcemanager/server/apis/v1" // init API group + _ "github.com/tikv/pd/pkg/mcs/tso/server/apis/v1" // init tso API group + mcs "github.com/tikv/pd/pkg/mcs/utils" "github.com/tikv/pd/pkg/member" "github.com/tikv/pd/pkg/ratelimit" - "github.com/tikv/pd/pkg/schedule" + sc "github.com/tikv/pd/pkg/schedule/config" "github.com/tikv/pd/pkg/schedule/hbstream" "github.com/tikv/pd/pkg/schedule/placement" + "github.com/tikv/pd/pkg/schedule/schedulers" "github.com/tikv/pd/pkg/storage" "github.com/tikv/pd/pkg/storage/endpoint" "github.com/tikv/pd/pkg/storage/kv" + "github.com/tikv/pd/pkg/syncer" "github.com/tikv/pd/pkg/systimemon" "github.com/tikv/pd/pkg/tso" "github.com/tikv/pd/pkg/utils/apiutil" @@ -71,11 +76,9 @@ import ( "github.com/tikv/pd/pkg/versioninfo" "github.com/tikv/pd/server/cluster" "github.com/tikv/pd/server/config" - "github.com/tikv/pd/server/gc" - "github.com/tikv/pd/server/keyspace" - syncer "github.com/tikv/pd/server/region_syncer" "go.etcd.io/etcd/clientv3" "go.etcd.io/etcd/embed" + "go.etcd.io/etcd/mvcc/mvccpb" "go.etcd.io/etcd/pkg/types" "go.uber.org/zap" "google.golang.org/grpc" @@ -83,7 +86,6 @@ import ( const ( serverMetricsInterval = time.Minute - leaderTickInterval = 50 * time.Millisecond // pdRootPath for all pd servers. pdRootPath = "/pd" pdAPIPrefix = "/pd/" @@ -98,6 +100,15 @@ const ( PDMode = "PD" // APIServiceMode represents that server is in API service mode. APIServiceMode = "API service" + + // maxRetryTimesGetServicePrimary is the max retry times for getting primary addr. + // Note: it need to be less than client.defaultPDTimeout + maxRetryTimesGetServicePrimary = 25 + // retryIntervalGetServicePrimary is the retry interval for getting primary addr. + retryIntervalGetServicePrimary = 100 * time.Millisecond + + lostPDLeaderMaxTimeoutSecs = 10 + lostPDLeaderReElectionFactor = 10 ) // EtcdStartTimeout the timeout of the startup etcd. @@ -115,8 +126,8 @@ var ( type Server struct { diagnosticspb.DiagnosticsServer - // Server state. - isServing int64 + // Server state. 0 is not running, 1 is running. + isRunning int64 // Server start timestamp startTimestamp int64 @@ -138,6 +149,8 @@ type Server struct { member *member.EmbeddedEtcdMember // etcd client client *clientv3.Client + // electionClient is used for leader election. + electionClient *clientv3.Client // http client httpClient *http.Client clusterID uint64 // pd cluster id. @@ -156,6 +169,10 @@ type Server struct { gcSafePointManager *gc.SafePointManager // keyspace manager keyspaceManager *keyspace.Manager + // safe point V2 manager + safePointV2Manager *gc.SafePointV2Manager + // keyspace group manager + keyspaceGroupManager *keyspace.GroupManager // for basicCluster operation. basicCluster *core.BasicCluster // for tso. @@ -180,9 +197,21 @@ type Server struct { hotRegionStorage *storage.HotRegionStorage // Store as map[string]*grpc.ClientConn clientConns sync.Map + + tsoClientPool struct { + sync.RWMutex + clients map[string]tsopb.TSO_TsoClient + } + // tsoDispatcher is used to dispatch different TSO requests to // the corresponding forwarding TSO channel. - tsoDispatcher sync.Map /* Store as map[string]chan *tsoRequest */ + tsoDispatcher *tsoutil.TSODispatcher + // tsoProtoFactory is the abstract factory for creating tso + // related data structures defined in the TSO grpc service + tsoProtoFactory *tsoutil.TSOProtoFactory + // pdProtoFactory is the abstract factory for creating tso + // related data structures defined in the PD grpc service + pdProtoFactory *tsoutil.PDProtoFactory serviceRateLimiter *ratelimit.Limiter serviceLabels map[string][]apiutil.AccessPath @@ -192,8 +221,10 @@ type Server struct { auditBackends []audit.Backend - registry *registry.ServiceRegistry - mode string + registry *registry.ServiceRegistry + mode string + servicePrimaryMap sync.Map /* Store as map[string]string */ + tsoPrimaryWatcher *etcdutil.LoopWatcher } // HandlerBuilder builds a server HTTP handler. @@ -208,7 +239,6 @@ func CreateServer(ctx context.Context, cfg *config.Config, services []string, le mode = APIServiceMode } log.Info(fmt.Sprintf("%s config", mode), zap.Reflect("config", cfg)) - rand.New(rand.NewSource(time.Now().UnixNano())) serviceMiddlewareCfg := config.NewServiceMiddlewareConfig() s := &Server{ @@ -221,6 +251,12 @@ func CreateServer(ctx context.Context, cfg *config.Config, services []string, le startTimestamp: time.Now().Unix(), DiagnosticsServer: sysutil.NewDiagnosticsServer(cfg.Log.File.Filename), mode: mode, + tsoClientPool: struct { + sync.RWMutex + clients map[string]tsopb.TSO_TsoClient + }{ + clients: make(map[string]tsopb.TSO_TsoClient), + }, } s.handler = newHandler(s) @@ -302,7 +338,12 @@ func (s *Server) startEtcd(ctx context.Context) error { } // start client - s.client, s.httpClient, err = startClient(s.cfg) + s.client, s.httpClient, err = s.startClient() + if err != nil { + return err + } + + s.electionClient, err = s.startElectionClient() if err != nil { return err } @@ -325,22 +366,35 @@ func (s *Server) startEtcd(ctx context.Context) error { failpoint.Inject("memberNil", func() { time.Sleep(1500 * time.Millisecond) }) - s.member = member.NewMember(etcd, s.client, etcdServerID) + s.member = member.NewMember(etcd, s.electionClient, etcdServerID) return nil } -func startClient(cfg *config.Config) (*clientv3.Client, *http.Client, error) { - tlsConfig, err := cfg.Security.ToTLSConfig() +func (s *Server) startClient() (*clientv3.Client, *http.Client, error) { + tlsConfig, err := s.cfg.Security.ToTLSConfig() if err != nil { return nil, nil, err } - etcdCfg, err := cfg.GenEmbedEtcdConfig() + etcdCfg, err := s.cfg.GenEmbedEtcdConfig() if err != nil { return nil, nil, err } return etcdutil.CreateClients(tlsConfig, etcdCfg.ACUrls) } +func (s *Server) startElectionClient() (*clientv3.Client, error) { + tlsConfig, err := s.cfg.Security.ToTLSConfig() + if err != nil { + return nil, err + } + etcdCfg, err := s.cfg.GenEmbedEtcdConfig() + if err != nil { + return nil, err + } + + return etcdutil.CreateEtcdClient(tlsConfig, etcdCfg.ACUrls) +} + // AddStartCallback adds a callback in the startServer phase. func (s *Server) AddStartCallback(callbacks ...func()) { s.startCallbacks = append(s.startCallbacks, callbacks...) @@ -357,7 +411,7 @@ func (s *Server) startServer(ctx context.Context) error { metadataGauge.WithLabelValues(fmt.Sprintf("cluster%d", s.clusterID)).Set(0) serverInfo.WithLabelValues(versioninfo.PDReleaseVersion, versioninfo.PDGitHash).Set(float64(time.Now().Unix())) - s.rootPath = path.Join(pdRootPath, strconv.FormatUint(s.clusterID, 10)) + s.rootPath = endpoint.PDRootPath(s.clusterID) s.member.InitMemberInfo(s.cfg.AdvertiseClientUrls, s.cfg.AdvertisePeerUrls, s.Name(), s.rootPath) s.member.SetMemberDeployPath(s.member.ID()) s.member.SetMemberBinaryVersion(s.member.ID(), versioninfo.PDReleaseVersion) @@ -375,29 +429,30 @@ func (s *Server) startServer(ctx context.Context) error { } defaultStorage := storage.NewStorageWithEtcdBackend(s.client, s.rootPath) s.storage = storage.NewCoreStorage(defaultStorage, regionStorage) - - s.tsoAllocatorManager = tso.NewAllocatorManager( - s.member, s.rootPath, s.storage, s.cfg.IsLocalTSOEnabled(), s.cfg.GetTSOSaveInterval(), s.cfg.GetTSOUpdatePhysicalInterval(), s.cfg.GetTLSConfig(), - func() time.Duration { return s.persistOptions.GetMaxResetTSGap() }) - // Set up the Global TSO Allocator here, it will be initialized once the PD campaigns leader successfully. - s.tsoAllocatorManager.SetUpAllocator(ctx, tso.GlobalDCLocation, s.member.GetLeadership()) - // When disabled the Local TSO, we should clean up the Local TSO Allocator's meta info written in etcd if it exists. - if !s.cfg.EnableLocalTSO { - if err = s.tsoAllocatorManager.CleanUpDCLocation(); err != nil { - return err + s.tsoDispatcher = tsoutil.NewTSODispatcher(tsoProxyHandleDuration, tsoProxyBatchSize) + s.tsoProtoFactory = &tsoutil.TSOProtoFactory{} + s.pdProtoFactory = &tsoutil.PDProtoFactory{} + if !s.IsAPIServiceMode() { + s.tsoAllocatorManager = tso.NewAllocatorManager(s.ctx, mcs.DefaultKeyspaceGroupID, s.member, s.rootPath, s.storage, s, false) + // When disabled the Local TSO, we should clean up the Local TSO Allocator's meta info written in etcd if it exists. + if !s.cfg.EnableLocalTSO { + if err = s.tsoAllocatorManager.CleanUpDCLocation(); err != nil { + return err + } } - } - if zone, exist := s.cfg.Labels[config.ZoneLabel]; exist && zone != "" && s.cfg.EnableLocalTSO { - if err = s.tsoAllocatorManager.SetLocalTSOConfig(zone); err != nil { - return err + if zone, exist := s.cfg.Labels[config.ZoneLabel]; exist && zone != "" && s.cfg.EnableLocalTSO { + if err = s.tsoAllocatorManager.SetLocalTSOConfig(zone); err != nil { + return err + } } } + s.encryptionKeyManager, err = encryption.NewManager(s.client, &s.cfg.Security.Encryption) if err != nil { return err } - s.gcSafePointManager = gc.NewSafePointManager(s.storage) + s.gcSafePointManager = gc.NewSafePointManager(s.storage, s.cfg.PDServerCfg) s.basicCluster = core.NewBasicCluster() s.cluster = cluster.NewRaftCluster(ctx, s.clusterID, syncer.NewRegionSyncer(s), s.client, s.httpClient) keyspaceIDAllocator := id.NewAllocator(&id.AllocatorParams{ @@ -408,7 +463,11 @@ func (s *Server) startServer(ctx context.Context) error { Member: s.member.MemberValue(), Step: keyspace.AllocStep, }) - s.keyspaceManager = keyspace.NewKeyspaceManager(s.storage, s.cluster, keyspaceIDAllocator, s.cfg.Keyspace) + if s.IsAPIServiceMode() { + s.keyspaceGroupManager = keyspace.NewKeyspaceGroupManager(s.ctx, s.storage, s.client, s.clusterID) + } + s.keyspaceManager = keyspace.NewKeyspaceManager(s.ctx, s.storage, s.cluster, keyspaceIDAllocator, &s.cfg.Keyspace, s.keyspaceGroupManager) + s.safePointV2Manager = gc.NewSafePointManagerV2(s.ctx, s.storage, s.storage, s.storage) s.hbStreams = hbstream.NewHeartbeatStreams(ctx, s.clusterID, s.cluster) // initial hot_region_storage in here. s.hotRegionStorage, err = storage.NewHotRegionsStorage( @@ -423,7 +482,7 @@ func (s *Server) startServer(ctx context.Context) error { } // Server has started. - atomic.StoreInt64(&s.isServing, 1) + atomic.StoreInt64(&s.isRunning, 1) serverMaxProcs.Set(float64(runtime.GOMAXPROCS(0))) return nil } @@ -435,7 +494,7 @@ func (s *Server) AddCloseCallback(callbacks ...func()) { // Close closes the server. func (s *Server) Close() { - if !atomic.CompareAndSwapInt64(&s.isServing, 1, 0) { + if !atomic.CompareAndSwapInt64(&s.isRunning, 1, 0) { // server is already closed return } @@ -443,12 +502,20 @@ func (s *Server) Close() { log.Info("closing server") s.stopServerLoop() + if s.IsAPIServiceMode() { + s.keyspaceGroupManager.Close() + } if s.client != nil { if err := s.client.Close(); err != nil { log.Error("close etcd client meet error", errs.ZapError(errs.ErrCloseEtcdClient, err)) } } + if s.electionClient != nil { + if err := s.electionClient.Close(); err != nil { + log.Error("close election client meet error", errs.ZapError(errs.ErrCloseEtcdClient, err)) + } + } if s.httpClient != nil { s.httpClient.CloseIdleConnections() @@ -480,7 +547,7 @@ func (s *Server) Close() { // IsClosed checks whether server is closed or not. func (s *Server) IsClosed() bool { - return atomic.LoadInt64(&s.isServing) == 0 + return atomic.LoadInt64(&s.isRunning) == 0 } // Run runs the pd server. @@ -492,10 +559,14 @@ func (s *Server) Run() error { if err := s.startEtcd(s.ctx); err != nil { return err } + if err := s.startServer(s.ctx); err != nil { return err } + failpoint.Inject("delayStartServerLoop", func() { + time.Sleep(2 * time.Second) + }) s.startServerLoop(s.ctx) return nil @@ -523,12 +594,15 @@ func (s *Server) LoopContext() context.Context { func (s *Server) startServerLoop(ctx context.Context) { s.serverLoopCtx, s.serverLoopCancel = context.WithCancel(ctx) - s.serverLoopWg.Add(5) + s.serverLoopWg.Add(4) go s.leaderLoop() go s.etcdLeaderLoop() go s.serverMetricsLoop() - go s.tsoAllocatorLoop() go s.encryptionKeyManagerLoop() + if s.IsAPIServiceMode() { + s.initTSOPrimaryWatcher() + s.tsoPrimaryWatcher.StartWatchLoop() + } } func (s *Server) stopServerLoop() { @@ -542,9 +616,11 @@ func (s *Server) serverMetricsLoop() { ctx, cancel := context.WithCancel(s.serverLoopCtx) defer cancel() + ticker := time.NewTicker(serverMetricsInterval) + defer ticker.Stop() for { select { - case <-time.After(serverMetricsInterval): + case <-ticker.C: s.collectEtcdStateMetrics() case <-ctx.Done(): log.Info("server is closed, exit metrics loop") @@ -553,17 +629,6 @@ func (s *Server) serverMetricsLoop() { } } -// tsoAllocatorLoop is used to run the TSO Allocator updating daemon. -func (s *Server) tsoAllocatorLoop() { - defer logutil.LogPanic() - defer s.serverLoopWg.Done() - - ctx, cancel := context.WithCancel(s.serverLoopCtx) - defer cancel() - s.tsoAllocatorManager.AllocatorDaemon(ctx) - log.Info("server is closed, exit allocator loop") -} - // encryptionKeyManagerLoop is used to start monitor encryption key changes. func (s *Server) encryptionKeyManagerLoop() { defer logutil.LogPanic() @@ -660,7 +725,7 @@ func (s *Server) bootstrapCluster(req *pdpb.BootstrapRequest) (*pdpb.BootstrapRe } if err = s.GetKeyspaceManager().Bootstrap(); err != nil { - log.Warn("bootstrap keyspace manager failed", errs.ZapError(err)) + log.Warn("bootstrapping keyspace manager failed", errs.ZapError(err)) } return &pdpb.BootstrapResponse{ @@ -729,10 +794,9 @@ func (s *Server) GetLeader() *pdpb.Member { return s.member.GetLeader() } -// GetPrimary returns the primary member provider of the api server. -// api service's leader is equal to the primary member. -func (s *Server) GetPrimary() bs.MemberProvider { - return s.member.GetLeader() +// GetLeaderListenUrls gets service endpoints from the leader in election group. +func (s *Server) GetLeaderListenUrls() []string { + return s.member.GetLeaderListenUrls() } // GetMember returns the member of server. @@ -791,6 +855,16 @@ func (s *Server) GetKeyspaceManager() *keyspace.Manager { return s.keyspaceManager } +// GetSafePointV2Manager returns the safe point v2 manager of server. +func (s *Server) GetSafePointV2Manager() *gc.SafePointV2Manager { + return s.safePointV2Manager +} + +// GetKeyspaceGroupManager returns the keyspace group manager of server. +func (s *Server) GetKeyspaceGroupManager() *keyspace.GroupManager { + return s.keyspaceGroupManager +} + // Name returns the unique etcd Name for this server in etcd cluster. func (s *Server) Name() string { return s.cfg.Name @@ -834,6 +908,7 @@ func (s *Server) GetConfig() *config.Config { cfg.Replication = *s.persistOptions.GetReplicationConfig().Clone() cfg.PDServerCfg = *s.persistOptions.GetPDServerConfig().Clone() cfg.ReplicationMode = *s.persistOptions.GetReplicationModeConfig() + cfg.Keyspace = *s.persistOptions.GetKeyspaceConfig().Clone() cfg.LabelProperty = s.persistOptions.GetLabelPropertyConfig().Clone() cfg.ClusterVersion = *s.persistOptions.GetClusterVersion() if s.storage == nil { @@ -846,7 +921,7 @@ func (s *Server) GetConfig() *config.Config { payload := make(map[string]interface{}) for i, sche := range sches { var config interface{} - err := schedule.DecodeConfig([]byte(configs[i]), &config) + err := schedulers.DecodeConfig([]byte(configs[i]), &config) if err != nil { log.Error("failed to decode scheduler config", zap.String("config", configs[i]), @@ -860,13 +935,38 @@ func (s *Server) GetConfig() *config.Config { return cfg } +// GetKeyspaceConfig gets the keyspace config information. +func (s *Server) GetKeyspaceConfig() *config.KeyspaceConfig { + return s.persistOptions.GetKeyspaceConfig().Clone() +} + +// SetKeyspaceConfig sets the keyspace config information. +func (s *Server) SetKeyspaceConfig(cfg config.KeyspaceConfig) error { + if err := cfg.Validate(); err != nil { + return err + } + old := s.persistOptions.GetKeyspaceConfig() + s.persistOptions.SetKeyspaceConfig(&cfg) + if err := s.persistOptions.Persist(s.storage); err != nil { + s.persistOptions.SetKeyspaceConfig(old) + log.Error("failed to update keyspace config", + zap.Reflect("new", cfg), + zap.Reflect("old", old), + errs.ZapError(err)) + return err + } + s.keyspaceManager.UpdateConfig(&cfg) + log.Info("keyspace config is updated", zap.Reflect("new", cfg), zap.Reflect("old", old)) + return nil +} + // GetScheduleConfig gets the balance config information. -func (s *Server) GetScheduleConfig() *config.ScheduleConfig { +func (s *Server) GetScheduleConfig() *sc.ScheduleConfig { return s.persistOptions.GetScheduleConfig().Clone() } // SetScheduleConfig sets the balance config information. -func (s *Server) SetScheduleConfig(cfg config.ScheduleConfig) error { +func (s *Server) SetScheduleConfig(cfg sc.ScheduleConfig) error { if err := cfg.Validate(); err != nil { return err } @@ -889,12 +989,12 @@ func (s *Server) SetScheduleConfig(cfg config.ScheduleConfig) error { } // GetReplicationConfig get the replication config. -func (s *Server) GetReplicationConfig() *config.ReplicationConfig { +func (s *Server) GetReplicationConfig() *sc.ReplicationConfig { return s.persistOptions.GetReplicationConfig().Clone() } // SetReplicationConfig sets the replication config. -func (s *Server) SetReplicationConfig(cfg config.ReplicationConfig) error { +func (s *Server) SetReplicationConfig(cfg sc.ReplicationConfig) error { if err := cfg.Validate(); err != nil { return err } @@ -930,14 +1030,14 @@ func (s *Server) SetReplicationConfig(cfg config.ReplicationConfig) error { len(defaultRule.StartKey) == 0 && len(defaultRule.EndKey) == 0) { return errors.New("cannot update MaxReplicas or LocationLabels when placement rules feature is enabled and not only default rule exists, please update rule instead") } - if !(defaultRule.Count == int(old.MaxReplicas) && typeutil.StringsEqual(defaultRule.LocationLabels, []string(old.LocationLabels))) { + if !(defaultRule.Count == int(old.MaxReplicas) && typeutil.AreStringSlicesEqual(defaultRule.LocationLabels, []string(old.LocationLabels))) { return errors.New("cannot to update replication config, the default rules do not consistent with replication config, please update rule instead") } return nil } - if !(cfg.MaxReplicas == old.MaxReplicas && typeutil.StringsEqual(cfg.LocationLabels, old.LocationLabels)) { + if !(cfg.MaxReplicas == old.MaxReplicas && typeutil.AreStringSlicesEqual(cfg.LocationLabels, old.LocationLabels)) { if err := CheckInDefaultRule(); err != nil { return err } @@ -1170,9 +1270,9 @@ func (s *Server) GetTLSConfig() *grpcutil.TLSConfig { return &s.cfg.Security.TLSConfig } -// GetRequestUnitConfig gets the RU config. -func (s *Server) GetRequestUnitConfig() *rm_server.RequestUnitConfig { - return &s.cfg.RequestUnit +// GetControllerConfig gets the resource manager controller config. +func (s *Server) GetControllerConfig() *rm_server.ControllerConfig { + return &s.cfg.Controller } // GetRaftCluster gets Raft cluster. @@ -1271,7 +1371,7 @@ func (s *Server) GetServiceRateLimiter() *ratelimit.Limiter { return s.serviceRateLimiter } -// IsInRateLimitAllowList returns whethis given service label is in allow lost +// IsInRateLimitAllowList returns whether given service label is in allow lost func (s *Server) IsInRateLimitAllowList(serviceLabel string) bool { return s.serviceRateLimiter.IsInAllowList(serviceLabel) } @@ -1373,7 +1473,15 @@ func (s *Server) leaderLoop() { return } - leader, rev, checkAgain := s.member.CheckLeader() + leader, checkAgain := s.member.CheckLeader() + // add failpoint to test leader check go to stuck. + failpoint.Inject("leaderLoopCheckAgain", func(val failpoint.Value) { + memberString := val.(string) + memberID, _ := strconv.ParseUint(memberString, 10, 64) + if s.member.ID() == memberID { + checkAgain = true + } + }) if checkAgain { continue } @@ -1383,15 +1491,17 @@ func (s *Server) leaderLoop() { log.Error("reload config failed", errs.ZapError(err)) continue } - // Check the cluster dc-location after the PD leader is elected - go s.tsoAllocatorManager.ClusterDCLocationChecker() + if !s.IsAPIServiceMode() { + // Check the cluster dc-location after the PD leader is elected + go s.tsoAllocatorManager.ClusterDCLocationChecker() + } syncer := s.cluster.GetRegionSyncer() if s.persistOptions.IsUseRegionStorage() { - syncer.StartSyncWithLeader(leader.GetClientUrls()[0]) + syncer.StartSyncWithLeader(leader.GetListenUrls()[0]) } log.Info("start to watch pd leader", zap.Stringer("pd-leader", leader)) // WatchLeader will keep looping and never return unless the PD leader has changed. - s.member.WatchLeader(s.serverLoopCtx, leader, rev) + leader.Watch(s.serverLoopCtx) syncer.StopSyncWithLeader() log.Info("pd leader has changed, try to re-campaign a pd leader") } @@ -1399,6 +1509,25 @@ func (s *Server) leaderLoop() { // To make sure the etcd leader and PD leader are on the same server. etcdLeader := s.member.GetEtcdLeader() if etcdLeader != s.member.ID() { + if s.member.GetLeader() == nil { + lastUpdated := s.member.GetLastLeaderUpdatedTime() + // use random timeout to avoid leader campaigning storm. + randomTimeout := time.Duration(rand.Intn(int(lostPDLeaderMaxTimeoutSecs)))*time.Second + lostPDLeaderMaxTimeoutSecs*time.Second + lostPDLeaderReElectionFactor*s.cfg.ElectionInterval.Duration + // add failpoint to test the campaign leader logic. + failpoint.Inject("timeoutWaitPDLeader", func() { + log.Info("timeoutWaitPDLeader is injected, skip wait other etcd leader be etcd leader") + randomTimeout = time.Duration(rand.Intn(10))*time.Millisecond + 100*time.Millisecond + }) + if lastUpdated.Add(randomTimeout).Before(time.Now()) && !lastUpdated.IsZero() && etcdLeader != 0 { + log.Info("the pd leader is lost for a long time, try to re-campaign a pd leader with resign etcd leader", + zap.Duration("timeout", randomTimeout), + zap.Time("last-updated", lastUpdated), + zap.String("current-leader-member-id", types.ID(etcdLeader).String()), + zap.String("transferee-member-id", types.ID(s.member.ID()).String()), + ) + s.member.MoveEtcdLeader(s.ctx, etcdLeader, s.member.ID()) + } + } log.Info("skip campaigning of pd leader and check later", zap.String("server-name", s.Name()), zap.Uint64("etcd-leader-id", etcdLeader), @@ -1439,25 +1568,26 @@ func (s *Server) campaignLeader() { s.member.KeepLeader(ctx) log.Info(fmt.Sprintf("campaign %s leader ok", s.mode), zap.String("campaign-leader-name", s.Name())) - allocator, err := s.tsoAllocatorManager.GetAllocator(tso.GlobalDCLocation) - if err != nil { - log.Error("failed to get the global TSO allocator", errs.ZapError(err)) - return - } - log.Info("initializing the global TSO allocator") - if err := allocator.Initialize(0); err != nil { - log.Error("failed to initialize the global TSO allocator", errs.ZapError(err)) - return + if !s.IsAPIServiceMode() { + allocator, err := s.tsoAllocatorManager.GetAllocator(tso.GlobalDCLocation) + if err != nil { + log.Error("failed to get the global TSO allocator", errs.ZapError(err)) + return + } + log.Info("initializing the global TSO allocator") + if err := allocator.Initialize(0); err != nil { + log.Error("failed to initialize the global TSO allocator", errs.ZapError(err)) + return + } + defer func() { + s.tsoAllocatorManager.ResetAllocatorGroup(tso.GlobalDCLocation) + failpoint.Inject("updateAfterResetTSO", func() { + if err = allocator.UpdateTSO(); err != nil { + panic(err) + } + }) + }() } - defer func() { - s.tsoAllocatorManager.ResetAllocatorGroup(tso.GlobalDCLocation) - failpoint.Inject("updateAfterResetTSO", func() { - if err = allocator.UpdateTSO(); err != nil { - panic(err) - } - }) - }() - if err := s.reloadConfigFromKV(); err != nil { log.Error("failed to reload configuration", errs.ZapError(err)) return @@ -1490,8 +1620,10 @@ func (s *Server) campaignLeader() { } // EnableLeader to accept the remaining service, such as GetStore, GetRegion. s.member.EnableLeader() - // Check the cluster dc-location after the PD leader is elected. - go s.tsoAllocatorManager.ClusterDCLocationChecker() + if !s.IsAPIServiceMode() { + // Check the cluster dc-location after the PD leader is elected. + go s.tsoAllocatorManager.ClusterDCLocationChecker() + } defer resetLeaderOnce.Do(func() { // as soon as cancel the leadership keepalive, then other member have chance // to be new leader. @@ -1502,7 +1634,7 @@ func (s *Server) campaignLeader() { CheckPDVersion(s.persistOptions) log.Info(fmt.Sprintf("%s leader is ready to serve", s.mode), zap.String("leader-name", s.Name())) - leaderTicker := time.NewTicker(leaderTickInterval) + leaderTicker := time.NewTicker(mcs.LeaderTickInterval) defer leaderTicker.Stop() for { @@ -1512,6 +1644,16 @@ func (s *Server) campaignLeader() { log.Info("no longer a leader because lease has expired, pd leader will step down") return } + // add failpoint to test exit leader, failpoint judge the member is the give value, then break + failpoint.Inject("exitCampaignLeader", func(val failpoint.Value) { + memberString := val.(string) + memberID, _ := strconv.ParseUint(memberString, 10, 64) + if s.member.ID() == memberID { + log.Info("exit PD leader") + failpoint.Return() + } + }) + etcdLeader := s.member.GetEtcdLeader() if etcdLeader != s.member.ID() { log.Info("etcd leader changed, resigns pd leadership", zap.String("old-pd-leader-name", s.Name())) @@ -1531,10 +1673,14 @@ func (s *Server) etcdLeaderLoop() { ctx, cancel := context.WithCancel(s.serverLoopCtx) defer cancel() + ticker := time.NewTicker(s.cfg.LeaderPriorityCheckInterval.Duration) + defer ticker.Stop() for { select { - case <-time.After(s.cfg.LeaderPriorityCheckInterval.Duration): + case <-ticker.C: s.member.CheckPriority(ctx) + // Note: we reset the ticker here to support updating configuration dynamically. + ticker.Reset(s.cfg.LeaderPriorityCheckInterval.Duration) case <-ctx.Done(): log.Info("server is closed, exit etcd leader loop") return @@ -1552,6 +1698,7 @@ func (s *Server) reloadConfigFromKV() error { return err } s.loadRateLimitConfig() + s.loadKeyspaceConfig() useRegionStorage := s.persistOptions.IsUseRegionStorage() regionStorage := storage.TrySwitchRegionStorage(s.storage, useRegionStorage) if regionStorage != nil { @@ -1564,6 +1711,11 @@ func (s *Server) reloadConfigFromKV() error { return nil } +func (s *Server) loadKeyspaceConfig() { + cfg := s.persistOptions.GetKeyspaceConfig() + s.keyspaceManager.UpdateConfig(cfg) +} + func (s *Server) loadRateLimitConfig() { cfg := s.serviceMiddlewarePersistOptions.GetRateLimitConfig().LimiterConfig for key := range cfg { @@ -1666,30 +1818,67 @@ func (s *Server) UnmarkSnapshotRecovering(ctx context.Context) error { } // GetServicePrimaryAddr returns the primary address for a given service. -func (s *Server) GetServicePrimaryAddr(ctx context.Context, serviceName string) (bool, string, error) { - // TODO: replace default group name after we make a decision. - key := path.Join("/ms/0", serviceName, fmt.Sprintf("%05d", 0), "primary") - leader := &pdpb.Member{} - ok, _, err := etcdutil.GetProtoMsgWithModRev(s.client, key, leader) - if err != nil || !ok { - return false, "", err +// Note: This function will only return primary address without judging if it's alive. +func (s *Server) GetServicePrimaryAddr(ctx context.Context, serviceName string) (string, bool) { + ticker := time.NewTicker(retryIntervalGetServicePrimary) + defer ticker.Stop() + for i := 0; i < maxRetryTimesGetServicePrimary; i++ { + if v, ok := s.servicePrimaryMap.Load(serviceName); ok { + return v.(string), true + } + select { + case <-s.ctx.Done(): + return "", false + case <-ctx.Done(): + return "", false + case <-ticker.C: + } } - // TODO: need to refactor after we redefine the member - return true, leader.GetName(), nil + return "", false } -// RecoverAllocID recover alloc id. set current base id to input id -func (s *Server) RecoverAllocID(ctx context.Context, id uint64) error { - return s.idAllocator.SetBase(id) +// SetServicePrimaryAddr sets the primary address directly. +// Note: This function is only used for test. +func (s *Server) SetServicePrimaryAddr(serviceName, addr string) { + s.servicePrimaryMap.Store(serviceName, addr) } -// GetGlobalTS returns global tso. -func (s *Server) GetGlobalTS() (uint64, error) { - ts, err := s.tsoAllocatorManager.GetGlobalTSO() - if err != nil { - return 0, err +func (s *Server) initTSOPrimaryWatcher() { + serviceName := mcs.TSOServiceName + tsoRootPath := endpoint.TSOSvcRootPath(s.clusterID) + tsoServicePrimaryKey := endpoint.KeyspaceGroupPrimaryPath(tsoRootPath, mcs.DefaultKeyspaceGroupID) + putFn := func(kv *mvccpb.KeyValue) error { + primary := &tsopb.Participant{} // TODO: use Generics + if err := proto.Unmarshal(kv.Value, primary); err != nil { + return err + } + listenUrls := primary.GetListenUrls() + if len(listenUrls) > 0 { + // listenUrls[0] is the primary service endpoint of the keyspace group + s.servicePrimaryMap.Store(serviceName, listenUrls[0]) + log.Info("update tso primary", zap.String("primary", listenUrls[0])) + } + return nil + } + deleteFn := func(kv *mvccpb.KeyValue) error { + s.servicePrimaryMap.Delete(serviceName) + return nil } - return tsoutil.GenerateTS(ts), nil + s.tsoPrimaryWatcher = etcdutil.NewLoopWatcher( + s.serverLoopCtx, + &s.serverLoopWg, + s.client, + "tso-primary-watcher", + tsoServicePrimaryKey, + putFn, + deleteFn, + func() error { return nil }, + ) +} + +// RecoverAllocID recover alloc id. set current base id to input id +func (s *Server) RecoverAllocID(ctx context.Context, id uint64) error { + return s.idAllocator.SetBase(id) } // GetExternalTS returns external timestamp. @@ -1698,11 +1887,7 @@ func (s *Server) GetExternalTS() uint64 { } // SetExternalTS returns external timestamp. -func (s *Server) SetExternalTS(externalTS uint64) error { - globalTS, err := s.GetGlobalTS() - if err != nil { - return err - } +func (s *Server) SetExternalTS(externalTS, globalTS uint64) error { if tsoutil.CompareTimestampUint64(externalTS, globalTS) == 1 { desc := "the external timestamp should not be larger than global ts" log.Error(desc, zap.Uint64("request timestamp", externalTS), zap.Uint64("global ts", globalTS)) @@ -1710,10 +1895,52 @@ func (s *Server) SetExternalTS(externalTS uint64) error { } currentExternalTS := s.GetRaftCluster().GetExternalTS() if tsoutil.CompareTimestampUint64(externalTS, currentExternalTS) != 1 { - desc := "the external timestamp should be larger than now" - log.Error(desc, zap.Uint64("request timestamp", externalTS), zap.Uint64("current external timestamp", currentExternalTS)) + desc := "the external timestamp should be larger than current external timestamp" + log.Error(desc, zap.Uint64("request", externalTS), zap.Uint64("current", currentExternalTS)) return errors.New(desc) } s.GetRaftCluster().SetExternalTS(externalTS) return nil } + +// IsLocalTSOEnabled returns if the local TSO is enabled. +func (s *Server) IsLocalTSOEnabled() bool { + return s.cfg.IsLocalTSOEnabled() +} + +// GetMaxConcurrentTSOProxyStreamings returns the max concurrent TSO proxy streamings. +// If the value is negative, there is no limit. +func (s *Server) GetMaxConcurrentTSOProxyStreamings() int { + return s.cfg.GetMaxConcurrentTSOProxyStreamings() +} + +// GetTSOProxyRecvFromClientTimeout returns timeout value for TSO proxy receiving from the client. +func (s *Server) GetTSOProxyRecvFromClientTimeout() time.Duration { + return s.cfg.GetTSOProxyRecvFromClientTimeout() +} + +// GetLeaderLease returns the leader lease. +func (s *Server) GetLeaderLease() int64 { + return s.cfg.GetLeaderLease() +} + +// GetTSOSaveInterval returns TSO save interval. +func (s *Server) GetTSOSaveInterval() time.Duration { + return s.cfg.GetTSOSaveInterval() +} + +// GetTSOUpdatePhysicalInterval returns TSO update physical interval. +func (s *Server) GetTSOUpdatePhysicalInterval() time.Duration { + return s.cfg.GetTSOUpdatePhysicalInterval() +} + +// GetMaxResetTSGap gets the max gap to reset the tso. +func (s *Server) GetMaxResetTSGap() time.Duration { + return s.persistOptions.GetMaxResetTSGap() +} + +// SetClient sets the etcd client. +// Notes: it is only used for test. +func (s *Server) SetClient(client *clientv3.Client) { + s.client = client +} diff --git a/server/server_test.go b/server/server_test.go index a284372cdbc..47ec2dd735c 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -23,6 +23,7 @@ import ( "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" + "github.com/tikv/pd/pkg/mcs/utils" "github.com/tikv/pd/pkg/utils/assertutil" "github.com/tikv/pd/pkg/utils/etcdutil" "github.com/tikv/pd/pkg/utils/testutil" @@ -84,7 +85,7 @@ func (suite *leaderServerTestSuite) TearDownSuite() { } } -func (suite *leaderServerTestSuite) newTestServersWithCfgs(ctx context.Context, cfgs []*config.Config) ([]*Server, CleanupFunc) { +func (suite *leaderServerTestSuite) newTestServersWithCfgs(ctx context.Context, cfgs []*config.Config) ([]*Server, testutil.CleanupFunc) { svrs := make([]*Server, 0, len(cfgs)) ch := make(chan *Server) @@ -297,7 +298,7 @@ func TestAPIService(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() mockHandler := CreateMockHandler(re, "127.0.0.1") - svr, err := CreateServer(ctx, cfg, []string{"api"}, mockHandler) + svr, err := CreateServer(ctx, cfg, []string{utils.APIServiceName}, mockHandler) re.NoError(err) defer svr.Close() err = svr.Run() diff --git a/server/testutil.go b/server/testutil.go index b0fe6ed7a1d..506139e20f1 100644 --- a/server/testutil.go +++ b/server/testutil.go @@ -36,11 +36,8 @@ import ( "go.etcd.io/etcd/embed" ) -// CleanupFunc closes test pd server(s) and deletes any files left behind. -type CleanupFunc func() - // NewTestServer creates a pd server for testing. -func NewTestServer(re *require.Assertions, c *assertutil.Checker) (*Server, CleanupFunc, error) { +func NewTestServer(re *require.Assertions, c *assertutil.Checker) (*Server, testutil.CleanupFunc, error) { ctx, cancel := context.WithCancel(context.Background()) cfg := NewTestSingleConfig(c) mockHandler := CreateMockHandler(re, "127.0.0.1") diff --git a/tests/cluster.go b/tests/cluster.go index 43887636a33..28506858f0c 100644 --- a/tests/cluster.go +++ b/tests/cluster.go @@ -32,6 +32,8 @@ import ( "github.com/tikv/pd/pkg/dashboard" "github.com/tikv/pd/pkg/errs" "github.com/tikv/pd/pkg/id" + "github.com/tikv/pd/pkg/keyspace" + "github.com/tikv/pd/pkg/mcs/utils" "github.com/tikv/pd/pkg/schedule/schedulers" "github.com/tikv/pd/pkg/swaggerserver" "github.com/tikv/pd/pkg/tso" @@ -43,7 +45,6 @@ import ( "github.com/tikv/pd/server/cluster" "github.com/tikv/pd/server/config" "github.com/tikv/pd/server/join" - "github.com/tikv/pd/server/keyspace" "go.etcd.io/etcd/clientv3" ) @@ -60,6 +61,8 @@ var ( WaitLeaderReturnDelay = 20 * time.Millisecond // WaitLeaderCheckInterval represents the time interval of WaitLeader running check. WaitLeaderCheckInterval = 500 * time.Millisecond + // WaitLeaderRetryTimes represents the maximum number of loops of WaitLeader. + WaitLeaderRetryTimes = 100 ) // TestServer is only for test. @@ -74,6 +77,15 @@ var zapLogOnce sync.Once // NewTestServer creates a new TestServer. func NewTestServer(ctx context.Context, cfg *config.Config) (*TestServer, error) { + return createTestServer(ctx, cfg, nil) +} + +// NewTestAPIServer creates a new TestServer. +func NewTestAPIServer(ctx context.Context, cfg *config.Config) (*TestServer, error) { + return createTestServer(ctx, cfg, []string{utils.APIServiceName}) +} + +func createTestServer(ctx context.Context, cfg *config.Config, services []string) (*TestServer, error) { err := logutil.SetupLogger(cfg.Log, &cfg.Logger, &cfg.LogProps, cfg.Security.RedactInfoLog) if err != nil { return nil, err @@ -85,9 +97,12 @@ func NewTestServer(ctx context.Context, cfg *config.Config) (*TestServer, error) if err != nil { return nil, err } - serviceBuilders := []server.HandlerBuilder{api.NewHandler, apiv2.NewV2Handler, swaggerserver.NewHandler, autoscaling.NewHandler} + serviceBuilders := []server.HandlerBuilder{api.NewHandler, apiv2.NewV2Handler, autoscaling.NewHandler} + if swaggerserver.Enabled() { + serviceBuilders = append(serviceBuilders, swaggerserver.NewHandler) + } serviceBuilders = append(serviceBuilders, dashboard.GetServiceBuilders()...) - svr, err := server.CreateServer(ctx, cfg, nil, serviceBuilders...) + svr, err := server.CreateServer(ctx, cfg, services, serviceBuilders...) if err != nil { return nil, err } @@ -397,7 +412,7 @@ func (s *TestServer) BootstrapCluster() error { // make a test know the PD leader has been elected as soon as possible. // If it exceeds the maximum number of loops, it will return nil. func (s *TestServer) WaitLeader() bool { - for i := 0; i < 100; i++ { + for i := 0; i < WaitLeaderRetryTimes; i++ { if s.server.GetMember().IsLeader() { return true } @@ -430,19 +445,33 @@ type ConfigOption func(conf *config.Config, serverName string) // NewTestCluster creates a new TestCluster. func NewTestCluster(ctx context.Context, initialServerCount int, opts ...ConfigOption) (*TestCluster, error) { + return createTestCluster(ctx, initialServerCount, false, opts...) +} + +// NewTestAPICluster creates a new TestCluster with API service. +func NewTestAPICluster(ctx context.Context, initialServerCount int, opts ...ConfigOption) (*TestCluster, error) { + return createTestCluster(ctx, initialServerCount, true, opts...) +} + +func createTestCluster(ctx context.Context, initialServerCount int, isAPIServiceMode bool, opts ...ConfigOption) (*TestCluster, error) { schedulers.Register() config := newClusterConfig(initialServerCount) servers := make(map[string]*TestServer) - for _, conf := range config.InitialServers { - serverConf, err := conf.Generate(opts...) + for _, cfg := range config.InitialServers { + serverConf, err := cfg.Generate(opts...) if err != nil { return nil, err } - s, err := NewTestServer(ctx, serverConf) + var s *TestServer + if isAPIServiceMode { + s, err = NewTestAPIServer(ctx, serverConf) + } else { + s, err = NewTestServer(ctx, serverConf) + } if err != nil { return nil, err } - servers[conf.Name] = s + servers[cfg.Name] = s } return &TestCluster{ config: config, @@ -456,6 +485,68 @@ func NewTestCluster(ctx context.Context, initialServerCount int, opts ...ConfigO }, nil } +// RestartTestAPICluster restarts the API test cluster. +func RestartTestAPICluster(ctx context.Context, cluster *TestCluster) (*TestCluster, error) { + return restartTestCluster(ctx, cluster, true) +} + +func restartTestCluster( + ctx context.Context, cluster *TestCluster, isAPIServiceMode bool, +) (newTestCluster *TestCluster, err error) { + schedulers.Register() + newTestCluster = &TestCluster{ + config: cluster.config, + servers: make(map[string]*TestServer, len(cluster.servers)), + tsPool: struct { + sync.Mutex + pool map[uint64]struct{} + }{ + pool: make(map[uint64]struct{}), + }, + } + + var serverMap sync.Map + var errorMap sync.Map + wg := sync.WaitGroup{} + for serverName, server := range newTestCluster.servers { + serverCfg := server.GetConfig() + wg.Add(1) + go func(serverName string, server *TestServer) { + defer wg.Done() + server.Destroy() + var ( + newServer *TestServer + serverErr error + ) + if isAPIServiceMode { + newServer, serverErr = NewTestAPIServer(ctx, serverCfg) + } else { + newServer, serverErr = NewTestServer(ctx, serverCfg) + } + serverMap.Store(serverName, newServer) + errorMap.Store(serverName, serverErr) + }(serverName, server) + } + wg.Wait() + + errorMap.Range(func(key, value interface{}) bool { + if value != nil { + err = value.(error) + return false + } + serverName := key.(string) + newServer, _ := serverMap.Load(serverName) + newTestCluster.servers[serverName] = newServer.(*TestServer) + return true + }) + + if err != nil { + return nil, errors.New("failed to restart cluster. " + err.Error()) + } + + return newTestCluster, nil +} + // RunServer starts to run TestServer. func (c *TestCluster) RunServer(server *TestServer) <-chan error { resC := make(chan error) @@ -530,7 +621,7 @@ func (c *TestCluster) GetFollower() string { // If it exceeds the maximum number of loops, it will return an empty string. func (c *TestCluster) WaitLeader(ops ...WaitOption) string { option := &WaitOp{ - retryTimes: 100, + retryTimes: WaitLeaderRetryTimes, waitInterval: WaitLeaderCheckInterval, } for _, op := range ops { @@ -540,9 +631,11 @@ func (c *TestCluster) WaitLeader(ops ...WaitOption) string { counter := make(map[string]int) running := 0 for _, s := range c.servers { + s.RLock() if s.state == Running { running++ } + s.RUnlock() n := s.GetLeader().GetName() if n != "" { counter[n]++ @@ -597,7 +690,7 @@ func (c *TestCluster) ResignLeader() error { // If it exceeds the maximum number of loops, it will return an empty string. func (c *TestCluster) WaitAllocatorLeader(dcLocation string, ops ...WaitOption) string { option := &WaitOp{ - retryTimes: 100, + retryTimes: WaitLeaderRetryTimes, waitInterval: WaitLeaderCheckInterval, } for _, op := range ops { @@ -679,6 +772,13 @@ func (c *TestCluster) HandleRegionHeartbeat(region *core.RegionInfo) error { return cluster.HandleRegionHeartbeat(region) } +// HandleReportBuckets processes BucketInfo reports from the client. +func (c *TestCluster) HandleReportBuckets(b *metapb.Buckets) error { + leader := c.GetLeader() + cluster := c.servers[leader].GetRaftCluster() + return cluster.HandleReportBuckets(b) +} + // Join is used to add a new TestServer into the cluster. func (c *TestCluster) Join(ctx context.Context, opts ...ConfigOption) (*TestServer, error) { conf, err := c.config.Join().Generate(opts...) @@ -693,6 +793,20 @@ func (c *TestCluster) Join(ctx context.Context, opts ...ConfigOption) (*TestServ return s, nil } +// JoinAPIServer is used to add a new TestAPIServer into the cluster. +func (c *TestCluster) JoinAPIServer(ctx context.Context, opts ...ConfigOption) (*TestServer, error) { + conf, err := c.config.Join().Generate(opts...) + if err != nil { + return nil, err + } + s, err := NewTestAPIServer(ctx, conf) + if err != nil { + return nil, err + } + c.servers[conf.Name] = s + return s, nil +} + // Destroy is used to destroy a TestCluster. func (c *TestCluster) Destroy() { for _, s := range c.servers { diff --git a/tests/client/Makefile b/tests/integrations/client/Makefile similarity index 57% rename from tests/client/Makefile rename to tests/integrations/client/Makefile index 5663a2f3b45..4b6f3336151 100644 --- a/tests/client/Makefile +++ b/tests/integrations/client/Makefile @@ -12,37 +12,38 @@ # See the License for the specific language governing permissions and # limitations under the License. -GO_TOOLS_BIN_PATH := $(shell pwd)/../../.tools/bin +ROOT_PATH := ../../.. +GO_TOOLS_BIN_PATH := $(ROOT_PATH)/.tools/bin PATH := $(GO_TOOLS_BIN_PATH):$(PATH) SHELL := env PATH='$(PATH)' GOBIN='$(GO_TOOLS_BIN_PATH)' $(shell which bash) static: install-tools + @ echo "gofmt ..." @ gofmt -s -l -d . 2>&1 | awk '{ print } END { if (NR > 0) { exit 1 } }' - @ golangci-lint run ./... - @ revive -formatter friendly -config ../../revive.toml . + @ echo "golangci-lint ..." + @ golangci-lint run -c $(ROOT_PATH)/.golangci.yml --verbose ./... --allow-parallel-runners + @ echo "revive ..." + @ revive -formatter friendly -config $(ROOT_PATH)/revive.toml ./... tidy: @ go mod tidy git diff go.mod go.sum | cat git diff --quiet go.mod go.sum -test: enable-codegen - CGO_ENABLED=1 go test -v -tags deadlock -race -cover || { $(MAKE) disable-codegen && exit 1; } - $(MAKE) disable-codegen +test: failpoint-enable + CGO_ENABLED=1 go test -v -tags deadlock -race -cover || { $(MAKE) failpoint-disable && exit 1; } + $(MAKE) failpoint-disable -basic-test: - # skip - -ci-test-job: enable-codegen - CGO_ENABLED=1 go test -tags deadlock -race -covermode=atomic -coverprofile=covprofile -coverpkg=../../... github.com/tikv/pd/tests/client +ci-test-job: + CGO_ENABLED=1 go test -v -tags deadlock -race -covermode=atomic -coverprofile=covprofile -coverpkg=$(ROOT_PATH)/... github.com/tikv/pd/tests/integrations/client install-tools: - cd ../../ && $(MAKE) install-tools + cd $(ROOT_PATH) && $(MAKE) install-tools -enable-codegen: - cd ../../ && $(MAKE) failpoint-enable +failpoint-enable: + cd $(ROOT_PATH) && $(MAKE) failpoint-enable go mod tidy -disable-codegen: - cd ../../ && $(MAKE) failpoint-disable +failpoint-disable: + cd $(ROOT_PATH) && $(MAKE) failpoint-disable go mod tidy diff --git a/tests/client/cert-expired/ca-config.json b/tests/integrations/client/cert-expired/ca-config.json similarity index 100% rename from tests/client/cert-expired/ca-config.json rename to tests/integrations/client/cert-expired/ca-config.json diff --git a/tests/client/cert-expired/ca-csr.json b/tests/integrations/client/cert-expired/ca-csr.json similarity index 100% rename from tests/client/cert-expired/ca-csr.json rename to tests/integrations/client/cert-expired/ca-csr.json diff --git a/tests/client/cert-expired/ca-key.pem b/tests/integrations/client/cert-expired/ca-key.pem similarity index 100% rename from tests/client/cert-expired/ca-key.pem rename to tests/integrations/client/cert-expired/ca-key.pem diff --git a/tests/client/cert-expired/ca.csr b/tests/integrations/client/cert-expired/ca.csr similarity index 100% rename from tests/client/cert-expired/ca.csr rename to tests/integrations/client/cert-expired/ca.csr diff --git a/tests/client/cert-expired/ca.pem b/tests/integrations/client/cert-expired/ca.pem similarity index 100% rename from tests/client/cert-expired/ca.pem rename to tests/integrations/client/cert-expired/ca.pem diff --git a/tests/client/cert-expired/client-key.pem b/tests/integrations/client/cert-expired/client-key.pem similarity index 100% rename from tests/client/cert-expired/client-key.pem rename to tests/integrations/client/cert-expired/client-key.pem diff --git a/tests/client/cert-expired/client.csr b/tests/integrations/client/cert-expired/client.csr similarity index 100% rename from tests/client/cert-expired/client.csr rename to tests/integrations/client/cert-expired/client.csr diff --git a/tests/client/cert-expired/client.pem b/tests/integrations/client/cert-expired/client.pem similarity index 100% rename from tests/client/cert-expired/client.pem rename to tests/integrations/client/cert-expired/client.pem diff --git a/tests/client/cert-expired/gencerts.sh b/tests/integrations/client/cert-expired/gencerts.sh similarity index 100% rename from tests/client/cert-expired/gencerts.sh rename to tests/integrations/client/cert-expired/gencerts.sh diff --git a/tests/client/cert-expired/pd-server-key.pem b/tests/integrations/client/cert-expired/pd-server-key.pem similarity index 100% rename from tests/client/cert-expired/pd-server-key.pem rename to tests/integrations/client/cert-expired/pd-server-key.pem diff --git a/tests/client/cert-expired/pd-server.csr b/tests/integrations/client/cert-expired/pd-server.csr similarity index 100% rename from tests/client/cert-expired/pd-server.csr rename to tests/integrations/client/cert-expired/pd-server.csr diff --git a/tests/client/cert-expired/pd-server.pem b/tests/integrations/client/cert-expired/pd-server.pem similarity index 100% rename from tests/client/cert-expired/pd-server.pem rename to tests/integrations/client/cert-expired/pd-server.pem diff --git a/tests/client/cert/ca-config.json b/tests/integrations/client/cert/ca-config.json similarity index 100% rename from tests/client/cert/ca-config.json rename to tests/integrations/client/cert/ca-config.json diff --git a/tests/client/cert/ca-csr.json b/tests/integrations/client/cert/ca-csr.json similarity index 100% rename from tests/client/cert/ca-csr.json rename to tests/integrations/client/cert/ca-csr.json diff --git a/tests/client/cert/ca-key.pem b/tests/integrations/client/cert/ca-key.pem similarity index 100% rename from tests/client/cert/ca-key.pem rename to tests/integrations/client/cert/ca-key.pem diff --git a/tests/client/cert/ca.csr b/tests/integrations/client/cert/ca.csr similarity index 100% rename from tests/client/cert/ca.csr rename to tests/integrations/client/cert/ca.csr diff --git a/tests/client/cert/ca.pem b/tests/integrations/client/cert/ca.pem similarity index 100% rename from tests/client/cert/ca.pem rename to tests/integrations/client/cert/ca.pem diff --git a/tests/client/cert/client-key.pem b/tests/integrations/client/cert/client-key.pem similarity index 100% rename from tests/client/cert/client-key.pem rename to tests/integrations/client/cert/client-key.pem diff --git a/tests/client/cert/client.csr b/tests/integrations/client/cert/client.csr similarity index 100% rename from tests/client/cert/client.csr rename to tests/integrations/client/cert/client.csr diff --git a/tests/client/cert/client.pem b/tests/integrations/client/cert/client.pem similarity index 100% rename from tests/client/cert/client.pem rename to tests/integrations/client/cert/client.pem diff --git a/tests/client/cert/gencerts.sh b/tests/integrations/client/cert/gencerts.sh similarity index 100% rename from tests/client/cert/gencerts.sh rename to tests/integrations/client/cert/gencerts.sh diff --git a/tests/client/cert/pd-server-key.pem b/tests/integrations/client/cert/pd-server-key.pem similarity index 100% rename from tests/client/cert/pd-server-key.pem rename to tests/integrations/client/cert/pd-server-key.pem diff --git a/tests/client/cert/pd-server.csr b/tests/integrations/client/cert/pd-server.csr similarity index 100% rename from tests/client/cert/pd-server.csr rename to tests/integrations/client/cert/pd-server.csr diff --git a/tests/client/cert/pd-server.pem b/tests/integrations/client/cert/pd-server.pem similarity index 100% rename from tests/client/cert/pd-server.pem rename to tests/integrations/client/cert/pd-server.pem diff --git a/tests/client/client_test.go b/tests/integrations/client/client_test.go similarity index 93% rename from tests/client/client_test.go rename to tests/integrations/client/client_test.go index 89994a8fb1b..8ada9f9d519 100644 --- a/tests/client/client_test.go +++ b/tests/integrations/client/client_test.go @@ -28,6 +28,7 @@ import ( "testing" "time" + "github.com/docker/go-units" "github.com/pingcap/failpoint" "github.com/pingcap/kvproto/pkg/meta_storagepb" "github.com/pingcap/kvproto/pkg/metapb" @@ -36,6 +37,7 @@ import ( "github.com/stretchr/testify/suite" pd "github.com/tikv/pd/client" "github.com/tikv/pd/pkg/core" + "github.com/tikv/pd/pkg/errs" "github.com/tikv/pd/pkg/mock/mockid" "github.com/tikv/pd/pkg/storage/endpoint" "github.com/tikv/pd/pkg/tso" @@ -142,7 +144,7 @@ func TestClientLeaderChange(t *testing.T) { // Check URL list. cli.Close() - urls := innerCli.GetServiceDiscovery().GetURLs() + urls := innerCli.GetServiceDiscovery().GetServiceURLs() sort.Strings(urls) sort.Strings(endpoints) re.Equal(endpoints, urls) @@ -207,47 +209,6 @@ func TestLeaderTransfer(t *testing.T) { wg.Wait() } -// More details can be found in this issue: https://github.com/tikv/pd/issues/4884 -func TestUpdateAfterResetTSO(t *testing.T) { - re := require.New(t) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - cluster, err := tests.NewTestCluster(ctx, 2) - re.NoError(err) - defer cluster.Destroy() - - endpoints := runServer(re, cluster) - cli := setupCli(re, ctx, endpoints) - - testutil.Eventually(re, func() bool { - _, _, err := cli.GetTS(context.TODO()) - return err == nil - }) - // Transfer leader to trigger the TSO resetting. - re.NoError(failpoint.Enable("github.com/tikv/pd/server/updateAfterResetTSO", "return(true)")) - oldLeaderName := cluster.WaitLeader() - err = cluster.GetServer(oldLeaderName).ResignLeader() - re.NoError(err) - re.NoError(failpoint.Disable("github.com/tikv/pd/server/updateAfterResetTSO")) - newLeaderName := cluster.WaitLeader() - re.NotEqual(oldLeaderName, newLeaderName) - // Request a new TSO. - testutil.Eventually(re, func() bool { - _, _, err := cli.GetTS(context.TODO()) - return err == nil - }) - // Transfer leader back. - re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/tso/delaySyncTimestamp", `return(true)`)) - err = cluster.GetServer(newLeaderName).ResignLeader() - re.NoError(err) - // Should NOT panic here. - testutil.Eventually(re, func() bool { - _, _, err := cli.GetTS(context.TODO()) - return err == nil - }) - re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/tso/delaySyncTimestamp")) -} - func TestTSOAllocatorLeader(t *testing.T) { re := require.New(t) ctx, cancel := context.WithCancel(context.Background()) @@ -293,9 +254,9 @@ func TestTSOAllocatorLeader(t *testing.T) { // Check allocator leaders URL map. cli.Close() - for dcLocation, url := range getTSOAllocatorServingEndpointURLs(cli.(pd.TSOClient)) { + for dcLocation, url := range getTSOAllocatorServingEndpointURLs(cli.(TSOAllocatorsGetter)) { if dcLocation == tso.GlobalDCLocation { - urls := innerCli.GetServiceDiscovery().GetURLs() + urls := innerCli.GetServiceDiscovery().GetServiceURLs() sort.Strings(urls) sort.Strings(endpoints) re.Equal(endpoints, urls) @@ -387,7 +348,7 @@ func TestUnavailableTimeAfterLeaderIsReady(t *testing.T) { defer wg.Done() leader := cluster.GetServer(cluster.GetLeader()) leader.Stop() - cluster.WaitLeader() + re.NotEmpty(cluster.WaitLeader()) leaderReadyTime = time.Now() cluster.RunServers([]*tests.TestServer{leader}) }() @@ -403,7 +364,7 @@ func TestUnavailableTimeAfterLeaderIsReady(t *testing.T) { leader := cluster.GetServer(cluster.GetLeader()) re.NoError(failpoint.Enable("github.com/tikv/pd/client/unreachableNetwork", "return(true)")) leader.Stop() - cluster.WaitLeader() + re.NotEmpty(cluster.WaitLeader()) re.NoError(failpoint.Disable("github.com/tikv/pd/client/unreachableNetwork")) leaderReadyTime = time.Now() }() @@ -411,6 +372,7 @@ func TestUnavailableTimeAfterLeaderIsReady(t *testing.T) { re.Less(maxUnavailableTime.UnixMilli(), leaderReadyTime.Add(1*time.Second).UnixMilli()) } +// TODO: migrate the Local/Global TSO tests to TSO integration test folder. func TestGlobalAndLocalTSO(t *testing.T) { re := require.New(t) ctx, cancel := context.WithCancel(context.Background()) @@ -460,7 +422,7 @@ func TestGlobalAndLocalTSO(t *testing.T) { re.NoError(failpoint.Enable("github.com/tikv/pd/client/skipUpdateMember", `return(true)`)) err = cluster.ResignLeader() re.NoError(err) - cluster.WaitLeader() + re.NotEmpty(cluster.WaitLeader()) _, _, err = cli.GetTS(ctx) re.Error(err) re.True(pd.IsLeaderChange(err)) @@ -492,13 +454,20 @@ func requestGlobalAndLocalTSO( var lastTS uint64 for i := 0; i < tsoRequestRound; i++ { globalPhysical1, globalLogical1, err := cli.GetTS(context.TODO()) - re.NoError(err) + // The allocator leader may be changed due to the environment issue. + if err != nil { + re.ErrorContains(err, errs.NotLeaderErr) + } globalTS1 := tsoutil.ComposeTS(globalPhysical1, globalLogical1) localPhysical, localLogical, err := cli.GetLocalTS(context.TODO(), dc) - re.NoError(err) + if err != nil { + re.ErrorContains(err, errs.NotLeaderErr) + } localTS := tsoutil.ComposeTS(localPhysical, localLogical) globalPhysical2, globalLogical2, err := cli.GetTS(context.TODO()) - re.NoError(err) + if err != nil { + re.ErrorContains(err, errs.NotLeaderErr) + } globalTS2 := tsoutil.ComposeTS(globalPhysical2, globalLogical2) re.Less(lastTS, globalTS1) re.Less(globalTS1, localTS) @@ -512,7 +481,10 @@ func requestGlobalAndLocalTSO( wg.Wait() } -func getTSOAllocatorServingEndpointURLs(c pd.TSOClient) map[string]string { +// GetTSOAllocators defines the TSO allocators getter. +type TSOAllocatorsGetter interface{ GetTSOAllocators() *sync.Map } + +func getTSOAllocatorServingEndpointURLs(c TSOAllocatorsGetter) map[string]string { allocatorLeaders := make(map[string]string) c.GetTSOAllocators().Range(func(dcLocation, url interface{}) bool { allocatorLeaders[dcLocation.(string)] = url.(string) @@ -624,7 +596,7 @@ func TestGetTsoFromFollowerClient2(t *testing.T) { lastTS = checkTS(re, cli, lastTS) re.NoError(cluster.GetServer(cluster.GetLeader()).ResignLeader()) - cluster.WaitLeader() + re.NotEmpty(cluster.WaitLeader()) lastTS = checkTS(re, cli, lastTS) re.NoError(failpoint.Disable("github.com/tikv/pd/client/unreachableNetwork")) @@ -648,7 +620,7 @@ func checkTS(re *require.Assertions, cli pd.Client, lastTS uint64) uint64 { func runServer(re *require.Assertions, cluster *tests.TestCluster) []string { err := cluster.RunInitialServers() re.NoError(err) - cluster.WaitLeader() + re.NotEmpty(cluster.WaitLeader()) leaderServer := cluster.GetServer(cluster.GetLeader()) re.NoError(leaderServer.BootstrapCluster()) @@ -778,7 +750,7 @@ var ( type clientTestSuite struct { suite.Suite - cleanup server.CleanupFunc + cleanup testutil.CleanupFunc ctx context.Context clean context.CancelFunc srv *server.Server @@ -823,6 +795,14 @@ func (suite *clientTestSuite) SetupSuite() { LastHeartbeat: now, }, }) + + storeInfo := suite.grpcSvr.GetRaftCluster().GetStore(store.GetId()) + newStore := storeInfo.Clone(core.SetStoreStats(&pdpb.StoreStats{ + Capacity: uint64(10 * units.GiB), + UsedSize: uint64(9 * units.GiB), + Available: uint64(1 * units.GiB), + })) + suite.grpcSvr.GetRaftCluster().GetBasicCluster().PutStore(newStore) } cluster.GetStoreConfig().SetRegionBucketEnabled(true) } @@ -859,48 +839,6 @@ func (suite *clientTestSuite) bootstrapServer(header *pdpb.RequestHeader, client suite.Equal(pdpb.ErrorType_OK, resp.GetHeader().GetError().GetType()) } -func (suite *clientTestSuite) TestNormalTSO() { - var wg sync.WaitGroup - wg.Add(tsoRequestConcurrencyNumber) - for i := 0; i < tsoRequestConcurrencyNumber; i++ { - go func() { - defer wg.Done() - var lastTS uint64 - for i := 0; i < tsoRequestRound; i++ { - physical, logical, err := suite.client.GetTS(context.Background()) - suite.NoError(err) - ts := tsoutil.ComposeTS(physical, logical) - suite.Less(lastTS, ts) - lastTS = ts - } - }() - } - wg.Wait() -} - -func (suite *clientTestSuite) TestGetTSAsync() { - var wg sync.WaitGroup - wg.Add(tsoRequestConcurrencyNumber) - for i := 0; i < tsoRequestConcurrencyNumber; i++ { - go func() { - defer wg.Done() - tsFutures := make([]pd.TSFuture, tsoRequestRound) - for i := range tsFutures { - tsFutures[i] = suite.client.GetTSAsync(context.Background()) - } - var lastTS uint64 = math.MaxUint64 - for i := len(tsFutures) - 1; i >= 0; i-- { - physical, logical, err := tsFutures[i].Wait() - suite.NoError(err) - ts := tsoutil.ComposeTS(physical, logical) - suite.Greater(lastTS, ts) - lastTS = ts - } - }() - } - wg.Wait() -} - func (suite *clientTestSuite) TestGetRegion() { regionID := regionIDAllocator.alloc() region := &metapb.Region{ @@ -1437,7 +1375,10 @@ func TestWatch(t *testing.T) { defer client.Close() key := "test" - ch, err := client.Watch(ctx, []byte(key)) + resp, err := client.Get(ctx, []byte(key)) + re.NoError(err) + rev := resp.GetHeader().GetRevision() + ch, err := client.Watch(ctx, []byte(key), pd.WithRev(rev)) re.NoError(err) exit := make(chan struct{}) go func() { diff --git a/tests/client/client_tls_test.go b/tests/integrations/client/client_tls_test.go similarity index 100% rename from tests/client/client_tls_test.go rename to tests/integrations/client/client_tls_test.go diff --git a/tests/integrations/client/gc_client_test.go b/tests/integrations/client/gc_client_test.go new file mode 100644 index 00000000000..acb1c458812 --- /dev/null +++ b/tests/integrations/client/gc_client_test.go @@ -0,0 +1,208 @@ +// Copyright 2023 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package client_test + +import ( + "path" + "strconv" + "testing" + "time" + + "github.com/pingcap/failpoint" + "github.com/pingcap/kvproto/pkg/pdpb" + "github.com/pingcap/log" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + pd "github.com/tikv/pd/client" + "github.com/tikv/pd/pkg/storage/endpoint" + "github.com/tikv/pd/pkg/utils/assertutil" + "github.com/tikv/pd/pkg/utils/testutil" + "github.com/tikv/pd/server" + "go.etcd.io/etcd/clientv3" + "go.uber.org/zap" + "google.golang.org/grpc" +) + +// gcClientTestReceiver is the pdpb.PD_WatchGCSafePointV2Server mock for testing. +type gcClientTestReceiver struct { + re *require.Assertions + grpc.ServerStream +} + +// Send is the mock implementation for pdpb.PD_WatchGCSafePointV2Server's Send. +// Instead of sending the response to the client, it will check the response. +// In testing, we will set all keyspace's safe point to be equal to its id, +// and this mock verifies that the response is correct. +func (s gcClientTestReceiver) Send(m *pdpb.WatchGCSafePointV2Response) error { + log.Info("received", zap.Any("received", m.GetEvents())) + for _, change := range m.GetEvents() { + s.re.Equal(change.SafePoint, uint64(change.KeyspaceId)) + } + return nil +} + +type gcClientTestSuite struct { + suite.Suite + server *server.GrpcServer + client pd.Client + cleanup testutil.CleanupFunc + gcSafePointV2Prefix string +} + +func TestGcClientTestSuite(t *testing.T) { + suite.Run(t, new(gcClientTestSuite)) +} + +func (suite *gcClientTestSuite) SetupSuite() { + var err error + var gsi *server.Server + checker := assertutil.NewChecker() + checker.FailNow = func() {} + gsi, suite.cleanup, err = server.NewTestServer(suite.Require(), checker) + suite.server = &server.GrpcServer{Server: gsi} + suite.NoError(err) + addr := suite.server.GetAddr() + suite.client, err = pd.NewClientWithContext(suite.server.Context(), []string{addr}, pd.SecurityOption{}) + suite.NoError(err) + rootPath := path.Join("/pd", strconv.FormatUint(suite.server.ClusterID(), 10)) + suite.gcSafePointV2Prefix = path.Join(rootPath, endpoint.GCSafePointV2Prefix()) + // Enable the fail-point to skip checking keyspace validity. + suite.NoError(failpoint.Enable("github.com/tikv/pd/pkg/gc/checkKeyspace", "return(true)")) +} + +func (suite *gcClientTestSuite) TearDownSuite() { + suite.NoError(failpoint.Disable("github.com/tikv/pd/pkg/gc/checkKeyspace")) + suite.cleanup() +} + +func (suite *gcClientTestSuite) TearDownTest() { + suite.CleanupEtcdGCPath() +} + +func (suite *gcClientTestSuite) CleanupEtcdGCPath() { + _, err := suite.server.GetClient().Delete(suite.server.Context(), suite.gcSafePointV2Prefix, clientv3.WithPrefix()) + suite.NoError(err) +} + +func (suite *gcClientTestSuite) TestWatch1() { + receiver := gcClientTestReceiver{re: suite.Require()} + go suite.server.WatchGCSafePointV2(&pdpb.WatchGCSafePointV2Request{ + Revision: 0, + }, receiver) + + // Init gc safe points as index value of keyspace 0 ~ 5. + for i := 0; i < 6; i++ { + suite.mustUpdateSafePoint(uint32(i), uint64(i)) + } + + // delete gc safe points of keyspace 3 ~ 5. + for i := 3; i < 6; i++ { + suite.mustDeleteSafePoint(uint32(i)) + } + + // check gc safe point equal to keyspace id for keyspace 0 ~ 2 . + for i := 0; i < 3; i++ { + suite.Equal(uint64(i), suite.mustLoadSafePoint(uint32(i))) + } + + // check gc safe point is 0 for keyspace 3 ~ 5 after delete. + for i := 3; i < 6; i++ { + suite.Equal(uint64(0), suite.mustLoadSafePoint(uint32(i))) + } +} + +func (suite *gcClientTestSuite) TestClientWatchWithRevision() { + suite.testClientWatchWithRevision(false) + suite.testClientWatchWithRevision(true) +} + +func (suite *gcClientTestSuite) testClientWatchWithRevision(fromNewRevision bool) { + testKeyspaceID := uint32(100) + initGCSafePoint := uint64(50) + updatedGCSafePoint := uint64(100) + + // Init gc safe point. + suite.mustUpdateSafePoint(testKeyspaceID, initGCSafePoint) + + // Get the initial revision. + initRevision := suite.mustGetRevision(testKeyspaceID) + + // Update the gc safe point. + suite.mustUpdateSafePoint(testKeyspaceID, updatedGCSafePoint) + + // Get the revision of the updated gc safe point. + updatedRevision := suite.mustGetRevision(testKeyspaceID) + + // Set the start revision of the watch request based on fromNewRevision. + startRevision := initRevision + if fromNewRevision { + startRevision = updatedRevision + } + watchChan, err := suite.client.WatchGCSafePointV2(suite.server.Context(), startRevision) + suite.NoError(err) + + timeout := time.After(time.Second) + + isFirstUpdate := true + for { + select { + case <-timeout: + return + case res := <-watchChan: + for _, r := range res { + suite.Equal(r.GetKeyspaceId(), testKeyspaceID) + if fromNewRevision { + // If fromNewRevision, first response should be the updated gc safe point. + suite.Equal(r.GetSafePoint(), updatedGCSafePoint) + } else if isFirstUpdate { + isFirstUpdate = false + suite.Equal(r.GetSafePoint(), initGCSafePoint) + } else { + suite.Equal(r.GetSafePoint(), updatedGCSafePoint) + continue + } + } + } + } +} + +// mustUpdateSafePoint updates the gc safe point of the given keyspace id. +func (suite *gcClientTestSuite) mustUpdateSafePoint(keyspaceID uint32, safePoint uint64) { + _, err := suite.client.UpdateGCSafePointV2(suite.server.Context(), keyspaceID, safePoint) + suite.NoError(err) +} + +// mustLoadSafePoint loads the gc safe point of the given keyspace id. +func (suite *gcClientTestSuite) mustLoadSafePoint(keyspaceID uint32) uint64 { + res, err := suite.server.GetSafePointV2Manager().LoadGCSafePoint(keyspaceID) + suite.NoError(err) + return res.SafePoint +} + +// mustDeleteSafePoint deletes the gc safe point of the given keyspace id. +func (suite *gcClientTestSuite) mustDeleteSafePoint(keyspaceID uint32) { + safePointPath := path.Join(suite.gcSafePointV2Prefix, endpoint.EncodeKeyspaceID(keyspaceID)) + log.Info("test etcd path", zap.Any("path", safePointPath)) // TODO: Delete + _, err := suite.server.GetClient().Delete(suite.server.Context(), safePointPath) + suite.NoError(err) +} + +// mustGetRevision gets the revision of the given keyspace's gc safe point. +func (suite *gcClientTestSuite) mustGetRevision(keyspaceID uint32) int64 { + safePointPath := path.Join(suite.gcSafePointV2Prefix, endpoint.EncodeKeyspaceID(keyspaceID)) + res, err := suite.server.GetClient().Get(suite.server.Context(), safePointPath) + suite.NoError(err) + return res.Header.GetRevision() +} diff --git a/tests/client/global_config_test.go b/tests/integrations/client/global_config_test.go similarity index 92% rename from tests/client/global_config_test.go rename to tests/integrations/client/global_config_test.go index 5b99a570121..15034d035a6 100644 --- a/tests/client/global_config_test.go +++ b/tests/integrations/client/global_config_test.go @@ -17,6 +17,7 @@ package client_test import ( "path" "strconv" + "sync" "testing" "time" @@ -27,6 +28,7 @@ import ( "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" "github.com/tikv/pd/pkg/utils/assertutil" + "github.com/tikv/pd/pkg/utils/testutil" "github.com/tikv/pd/server" "go.uber.org/zap" "google.golang.org/grpc" @@ -51,7 +53,8 @@ type globalConfigTestSuite struct { suite.Suite server *server.GrpcServer client pd.Client - cleanup server.CleanupFunc + cleanup testutil.CleanupFunc + mu sync.Mutex } func TestGlobalConfigTestSuite(t *testing.T) { @@ -321,3 +324,31 @@ func (suite *globalConfigTestSuite) TestClientWatchWithRevision() { } } } + +func (suite *globalConfigTestSuite) TestEtcdNotStart() { + cli := suite.server.GetClient() + defer func() { + suite.mu.Lock() + suite.server.SetClient(cli) + suite.mu.Unlock() + }() + suite.mu.Lock() + suite.server.SetClient(nil) + suite.mu.Unlock() + err := suite.server.WatchGlobalConfig(&pdpb.WatchGlobalConfigRequest{ + ConfigPath: globalConfigPath, + Revision: 0, + }, nil) + suite.Error(err) + + _, err = suite.server.StoreGlobalConfig(suite.server.Context(), &pdpb.StoreGlobalConfigRequest{ + ConfigPath: globalConfigPath, + Changes: []*pdpb.GlobalConfigItem{{Kind: pdpb.EventType_PUT, Name: "0", Payload: []byte("0")}}, + }) + suite.Error(err) + + _, err = suite.server.LoadGlobalConfig(suite.server.Context(), &pdpb.LoadGlobalConfigRequest{ + Names: []string{"test_etcd"}, + }) + suite.Error(err) +} diff --git a/tests/client/go.mod b/tests/integrations/client/go.mod similarity index 74% rename from tests/client/go.mod rename to tests/integrations/client/go.mod index b724d2ec995..a308c176eb6 100644 --- a/tests/client/go.mod +++ b/tests/integrations/client/go.mod @@ -1,18 +1,27 @@ -module github.com/tikv/pd/tests/client +module github.com/tikv/pd/tests/integrations/client go 1.20 +replace ( + github.com/tikv/pd => ../../../ + github.com/tikv/pd/client => ../../../client +) + +// reset grpc and protobuf deps in order to import client and server at the same time +replace google.golang.org/grpc v1.54.0 => google.golang.org/grpc v1.26.0 + require ( + github.com/docker/go-units v0.4.0 github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00 - github.com/pingcap/kvproto v0.0.0-20230228041042-1e9aca94bab6 + github.com/pingcap/kvproto v0.0.0-20230727073445-53e1f8730c30 github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 - github.com/stretchr/testify v1.8.1 + github.com/stretchr/testify v1.8.2 github.com/tikv/pd v0.0.0-00010101000000-000000000000 github.com/tikv/pd/client v0.0.0-00010101000000-000000000000 go.etcd.io/etcd v0.5.0-alpha.5.0.20220915004622-85b640cee793 go.uber.org/goleak v1.1.12 - go.uber.org/zap v1.20.0 - google.golang.org/grpc v1.51.0 + go.uber.org/zap v1.24.0 + google.golang.org/grpc v1.54.0 ) require ( @@ -22,27 +31,38 @@ require ( github.com/PuerkitoBio/purell v1.1.1 // indirect github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect github.com/ReneKroon/ttlcache/v2 v2.3.0 // indirect - github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect github.com/VividCortex/mysqlerr v1.0.0 // indirect github.com/Xeoncross/go-aesctr-with-hmac v0.0.0-20200623134604-12b17a7ff502 // indirect - github.com/aws/aws-sdk-go v1.35.3 // indirect - github.com/benbjohnson/clock v1.1.0 // indirect + github.com/aws/aws-sdk-go-v2 v1.17.7 // indirect + github.com/aws/aws-sdk-go-v2/config v1.18.19 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.13.18 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.1 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.31 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.25 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.3.32 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.25 // indirect + github.com/aws/aws-sdk-go-v2/service/kms v1.20.8 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.12.6 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.6 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.18.7 // indirect + github.com/aws/smithy-go v1.13.5 // indirect + github.com/benbjohnson/clock v1.3.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bitly/go-simplejson v0.5.0 // indirect github.com/breeswish/gin-jwt/v2 v2.6.4-jwt-patch // indirect github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 // indirect github.com/cenkalti/backoff/v4 v4.0.2 // indirect - github.com/cespare/xxhash/v2 v2.1.2 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/coreos/go-semver v0.3.0 // indirect github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f // indirect github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/docker/go-units v0.4.0 // indirect github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4 // indirect github.com/elliotchance/pie/v2 v2.1.0 // indirect github.com/fogleman/gg v1.3.0 // indirect github.com/gin-contrib/cors v1.4.0 // indirect github.com/gin-contrib/gzip v0.0.1 // indirect + github.com/gin-contrib/pprof v1.4.0 // indirect github.com/gin-contrib/sse v0.1.0 // indirect github.com/gin-gonic/gin v1.8.1 // indirect github.com/go-ole/go-ole v1.2.6 // indirect @@ -60,7 +80,7 @@ require ( github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt v3.2.1+incompatible // indirect github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 // indirect - github.com/golang/protobuf v1.5.2 // indirect + github.com/golang/protobuf v1.5.3 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/btree v1.1.2 // indirect github.com/google/pprof v0.0.0-20211122183932-1daafda22083 // indirect @@ -75,7 +95,6 @@ require ( github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect - github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/joho/godotenv v1.4.0 // indirect github.com/jonboulle/clockwork v0.2.2 // indirect github.com/joomcode/errorx v1.0.1 // indirect @@ -83,7 +102,7 @@ require ( github.com/json-iterator/go v1.1.12 // indirect github.com/konsorten/go-windows-terminal-sequences v1.0.3 // indirect github.com/leodido/go-urn v1.2.1 // indirect - github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect + github.com/lufia/plan9stats v0.0.0-20230326075908-cb1d2100619a // indirect github.com/mailru/easyjson v0.7.6 // indirect github.com/mattn/go-isatty v0.0.14 // indirect github.com/mattn/go-sqlite3 v1.14.15 // indirect @@ -99,12 +118,12 @@ require ( github.com/pingcap/check v0.0.0-20211026125417-57bd13f7b5f0 // indirect github.com/pingcap/errcode v0.3.0 // indirect github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c // indirect - github.com/pingcap/sysutil v0.0.0-20211208032423-041a72e5860d // indirect - github.com/pingcap/tidb-dashboard v0.0.0-20230209052558-a58fc2a7e924 // indirect + github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21 // indirect + github.com/pingcap/tidb-dashboard v0.0.0-20230705095454-5e220f970f27 // indirect github.com/pingcap/tipb v0.0.0-20220718022156-3e2483c20a9e // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect + github.com/power-devops/perfstat v0.0.0-20221212215047-62379fc7944b // indirect github.com/prometheus/client_golang v1.11.1 // indirect github.com/prometheus/client_model v0.2.0 // indirect github.com/prometheus/common v0.26.0 // indirect @@ -112,8 +131,8 @@ require ( github.com/rs/cors v1.7.0 // indirect github.com/samber/lo v1.37.0 // indirect github.com/sasha-s/go-deadlock v0.2.0 // indirect - github.com/shirou/gopsutil v3.21.3+incompatible // indirect - github.com/shirou/gopsutil/v3 v3.22.12 // indirect + github.com/shirou/gopsutil/v3 v3.23.3 // indirect + github.com/shoenig/go-m1cpu v0.1.5 // indirect github.com/shurcooL/httpgzip v0.0.0-20190720172056-320755c1c1b0 // indirect github.com/sirupsen/logrus v1.6.0 // indirect github.com/smallnest/chanx v0.0.0-20221229104322-eb4c998d2072 // indirect @@ -121,8 +140,8 @@ require ( github.com/spf13/cobra v1.0.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/stretchr/objx v0.5.0 // indirect - github.com/swaggo/files v0.0.0-20190704085106-630677cd5c14 // indirect - github.com/swaggo/http-swagger v0.0.0-20200308142732-58ac5e232fba // indirect + github.com/swaggo/files v0.0.0-20210815190702-a29dd2bc99b2 // indirect + github.com/swaggo/http-swagger v1.2.6 // indirect github.com/swaggo/swag v1.8.3 // indirect github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965 // indirect github.com/tklauser/go-sysconf v0.3.11 // indirect @@ -139,21 +158,21 @@ require ( go.uber.org/atomic v1.10.0 // indirect go.uber.org/dig v1.9.0 // indirect go.uber.org/fx v1.12.0 // indirect - go.uber.org/multierr v1.7.0 // indirect + go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.1.0 // indirect - golang.org/x/exp v0.0.0-20230108222341-4b8118a2686a // indirect + golang.org/x/exp v0.0.0-20230711005742-c3f37128e5a4 // indirect golang.org/x/image v0.5.0 // indirect - golang.org/x/net v0.2.0 // indirect - golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783 // indirect + golang.org/x/net v0.10.0 // indirect + golang.org/x/oauth2 v0.4.0 // indirect golang.org/x/sync v0.1.0 // indirect - golang.org/x/sys v0.3.0 // indirect - golang.org/x/text v0.7.0 // indirect + golang.org/x/sys v0.8.0 // indirect + golang.org/x/text v0.9.0 // indirect golang.org/x/time v0.1.0 // indirect - golang.org/x/tools v0.2.0 // indirect + golang.org/x/tools v0.6.0 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20221202195650-67e5cbc046fd // indirect - google.golang.org/protobuf v1.28.1 // indirect - gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect + google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect + google.golang.org/protobuf v1.30.0 // indirect + gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect gorm.io/datatypes v1.1.0 // indirect @@ -163,11 +182,3 @@ require ( moul.io/zapgorm2 v1.1.0 // indirect sigs.k8s.io/yaml v1.2.0 // indirect ) - -replace ( - github.com/tikv/pd => ../../ - github.com/tikv/pd/client => ../../client -) - -// reset grpc and protobuf deps in order to import client and server at the same time -replace google.golang.org/grpc v1.51.0 => google.golang.org/grpc v1.26.0 diff --git a/tests/client/go.sum b/tests/integrations/client/go.sum similarity index 87% rename from tests/client/go.sum rename to tests/integrations/client/go.sum index 260b905907a..3f4906e6848 100644 --- a/tests/client/go.sum +++ b/tests/integrations/client/go.sum @@ -7,19 +7,17 @@ github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6Xge github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww= github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/ReneKroon/ttlcache/v2 v2.3.0 h1:qZnUjRKIrbKHH6vF5T7Y9Izn5ObfTZfyYpGhvz2BKPo= github.com/ReneKroon/ttlcache/v2 v2.3.0/go.mod h1:zbo6Pv/28e21Z8CzzqgYRArQYGYtjONRxaAKGxzQvG4= -github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUWq3EgK3CesDbo8upS2Vm9/P3FtgI+Jk= -github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/VividCortex/mysqlerr v1.0.0 h1:5pZ2TZA+YnzPgzBfiUWGqWmKDVNBdrkf9g+DNe1Tiq8= github.com/VividCortex/mysqlerr v1.0.0/go.mod h1:xERx8E4tBhLvpjzdUyQiSfUxeMcATEQrflDAfXsqcAE= github.com/Xeoncross/go-aesctr-with-hmac v0.0.0-20200623134604-12b17a7ff502 h1:L8IbaI/W6h5Cwgh0n4zGeZpVK78r/jBf9ASurHo9+/o= github.com/Xeoncross/go-aesctr-with-hmac v0.0.0-20200623134604-12b17a7ff502/go.mod h1:pmnBM9bxWSiHvC/gSWunUIyDvGn33EkP2CUjxFKtTTM= +github.com/agiledragon/gomonkey/v2 v2.3.1/go.mod h1:ap1AmDzcVOAz1YpeJ3TCzIgstoaWLA6jbbgxfB4w2iY= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -30,10 +28,35 @@ github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kd github.com/appleboy/gofight/v2 v2.1.2 h1:VOy3jow4vIK8BRQJoC/I9muxyYlJ2yb9ht2hZoS3rf4= github.com/appleboy/gofight/v2 v2.1.2/go.mod h1:frW+U1QZEdDgixycTj4CygQ48yLTUhplt43+Wczp3rw= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= -github.com/aws/aws-sdk-go v1.35.3 h1:r0puXncSaAfRt7Btml2swUo74Kao+vKhO3VLjwDjK54= -github.com/aws/aws-sdk-go v1.35.3/go.mod h1:H7NKnBqNVzoTJpGfLrQkkD+ytBA93eiDYi/+8rV9s48= -github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= +github.com/aws/aws-sdk-go-v2 v1.17.7 h1:CLSjnhJSTSogvqUGhIC6LqFKATMRexcxLZ0i/Nzk9Eg= +github.com/aws/aws-sdk-go-v2 v1.17.7/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= +github.com/aws/aws-sdk-go-v2/config v1.18.19 h1:AqFK6zFNtq4i1EYu+eC7lcKHYnZagMn6SW171la0bGw= +github.com/aws/aws-sdk-go-v2/config v1.18.19/go.mod h1:XvTmGMY8d52ougvakOv1RpiTLPz9dlG/OQHsKU/cMmY= +github.com/aws/aws-sdk-go-v2/credentials v1.13.18 h1:EQMdtHwz0ILTW1hoP+EwuWhwCG1hD6l3+RWFQABET4c= +github.com/aws/aws-sdk-go-v2/credentials v1.13.18/go.mod h1:vnwlwjIe+3XJPBYKu1et30ZPABG3VaXJYr8ryohpIyM= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.1 h1:gt57MN3liKiyGopcqgNzJb2+d9MJaKT/q1OksHNXVE4= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.1/go.mod h1:lfUx8puBRdM5lVVMQlwt2v+ofiG/X6Ms+dy0UkG/kXw= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.31 h1:sJLYcS+eZn5EeNINGHSCRAwUJMFVqklwkH36Vbyai7M= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.31/go.mod h1:QT0BqUvX1Bh2ABdTGnjqEjvjzrCfIniM9Sc8zn9Yndo= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.25 h1:1mnRASEKnkqsntcxHaysxwgVoUUp5dkiB+l3llKnqyg= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.25/go.mod h1:zBHOPwhBc3FlQjQJE/D3IfPWiWaQmT06Vq9aNukDo0k= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.32 h1:p5luUImdIqywn6JpQsW3tq5GNOxKmOnEpybzPx+d1lk= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.32/go.mod h1:XGhIBZDEgfqmFIugclZ6FU7v75nHhBDtzuB4xB/tEi4= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.25 h1:5LHn8JQ0qvjD9L9JhMtylnkcw7j05GDZqM9Oin6hpr0= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.25/go.mod h1:/95IA+0lMnzW6XzqYJRpjjsAbKEORVeO0anQqjd2CNU= +github.com/aws/aws-sdk-go-v2/service/kms v1.20.8 h1:R5f4VOFi3ScTe7TtePyxLqEhNqTJIAxL57MzrXFNs6I= +github.com/aws/aws-sdk-go-v2/service/kms v1.20.8/go.mod h1:OtP3pBOgmJM+acQyQcQXtQHets3yJoVuanCx2T5M7v4= +github.com/aws/aws-sdk-go-v2/service/sso v1.12.6 h1:5V7DWLBd7wTELVz5bPpwzYy/sikk0gsgZfj40X+l5OI= +github.com/aws/aws-sdk-go-v2/service/sso v1.12.6/go.mod h1:Y1VOmit/Fn6Tz1uFAeCO6Q7M2fmfXSCLeL5INVYsLuY= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.6 h1:B8cauxOH1W1v7rd8RdI/MWnoR4Ze0wIHWrb90qczxj4= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.6/go.mod h1:Lh/bc9XUf8CfOY6Jp5aIkQtN+j1mc+nExc+KXj9jx2s= +github.com/aws/aws-sdk-go-v2/service/sts v1.18.7 h1:bWNgNdRko2x6gqa0blfATqAZKZokPIeM1vfmQt2pnvM= +github.com/aws/aws-sdk-go-v2/service/sts v1.18.7/go.mod h1:JuTnSoeePXmMVe9G8NcjjwgOKEfZ4cOjMuT2IBT/2eI= +github.com/aws/smithy-go v1.13.5 h1:hgz0X/DX0dGqTYpGALqXJoRKRj5oQ7150i5FdTePzO8= +github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= +github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= +github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -51,8 +74,8 @@ github.com/cenkalti/backoff/v4 v4.0.2/go.mod h1:eEew/i+1Q6OrCDZh3WiXYv3+nJwBASZ8 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= -github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= @@ -74,6 +97,7 @@ github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbp github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/corona10/goimagehash v1.0.2 h1:pUfB0LnsJASMPGEZLj7tGY251vF+qLGqOgEP4rUs6kA= github.com/corona10/goimagehash v1.0.2/go.mod h1:/l9umBhvcHQXVtQO1V6Gp1yD20STawkhRnnX0D1bvVI= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= @@ -103,38 +127,30 @@ github.com/gin-contrib/cors v1.4.0 h1:oJ6gwtUl3lqV0WEIwM/LxPF1QZ5qe2lGWdY2+bz7y0 github.com/gin-contrib/cors v1.4.0/go.mod h1:bs9pNM0x/UsmHPBWT2xZz9ROh8xYjYkiURUfmBoMlcs= github.com/gin-contrib/gzip v0.0.1 h1:ezvKOL6jH+jlzdHNE4h9h8q8uMpDQjyl0NN0Jd7jozc= github.com/gin-contrib/gzip v0.0.1/go.mod h1:fGBJBCdt6qCZuCAOwWuFhBB4OOq9EFqlo5dEaFhhu5w= +github.com/gin-contrib/pprof v1.4.0 h1:XxiBSf5jWZ5i16lNOPbMTVdgHBdhfGRD5PZ1LWazzvg= +github.com/gin-contrib/pprof v1.4.0/go.mod h1:RrehPJasUVBPK6yTUwOl8/NP6i0vbUgmxtis+Z5KE90= github.com/gin-contrib/sse v0.0.0-20170109093832-22d885f9ecc7/go.mod h1:VJ0WA2NBN22VlZ2dKZQPAPnyWw5XTlK1KymzLKsr59s= -github.com/gin-contrib/sse v0.0.0-20190301062529-5545eab6dad3/go.mod h1:VJ0WA2NBN22VlZ2dKZQPAPnyWw5XTlK1KymzLKsr59s= github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= github.com/gin-gonic/gin v1.3.0/go.mod h1:7cKuhb5qV2ggCFctp2fJQ+ErvciLZrIeoOSOm6mUr7Y= -github.com/gin-gonic/gin v1.4.0/go.mod h1:OW2EZn3DO8Ln9oIKOvM++LBO+5UPHJJDH72/q/3rZdM= github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M= github.com/gin-gonic/gin v1.8.1 h1:4+fr/el88TOO3ewCmQr8cx/CtZ/umlIRIs5M4NTNjf8= github.com/gin-gonic/gin v1.8.1/go.mod h1:ji8BvRH1azfM+SYow9zQ6SZMvR8qOMZHmsCuWR9tTTk= -github.com/go-chi/chi v4.0.2+incompatible h1:maB6vn6FqCxrpz4FqWdh4+lwpyZIQS7YEAUcHlgXVRs= -github.com/go-chi/chi v4.0.2+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= -github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= -github.com/go-openapi/jsonpointer v0.17.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonreference v0.17.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= -github.com/go-openapi/jsonreference v0.19.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= github.com/go-openapi/jsonreference v0.19.6 h1:UBIxjkht+AWIgYzCDSv2GN+E/togfwXUJFRTWhl2Jjs= github.com/go-openapi/jsonreference v0.19.6/go.mod h1:diGHMEHg2IqXZGKxqyvWdfWU/aim5Dprw5bqpKkTvns= -github.com/go-openapi/spec v0.19.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= github.com/go-openapi/spec v0.20.4 h1:O8hJrt0UMnhHcluhIdUgCLRWyM2x7QkBXRvOs7m+O1M= github.com/go-openapi/spec v0.20.4/go.mod h1:faYFR1CvsJZ0mNsmsphTMSoRrNV3TEDoAM7FOEWeq8I= -github.com/go-openapi/swag v0.17.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.19.15 h1:D2NRCBzS9/pEY3gP9Nl8aDqGUcPFrwG2p+CNFrLyrCM= github.com/go-openapi/swag v0.19.15/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= @@ -151,7 +167,6 @@ github.com/go-playground/validator/v10 v10.10.0 h1:I7mrTYv78z8k8VXa/qJlOlEXn/nBh github.com/go-playground/validator/v10 v10.10.0/go.mod h1:74x4gJWsvQexRdW8Pn3dXSGrTK4nAUsbPlLADvpJkos= github.com/go-resty/resty/v2 v2.6.0 h1:joIR5PNLM2EFqqESUjCMGXrWmXNHEU9CEiK813oKYS4= github.com/go-resty/resty/v2 v2.6.0/go.mod h1:PwvJS6hvaPkjtjNg9ph+VrSD92bi5Zq73w/BIH7cC3Q= -github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-sql-driver/mysql v1.7.0 h1:ueSltNNllEqE3qcWBTD0iQd3IpL/6U+mJxLkazJ7YPc= github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= @@ -171,8 +186,8 @@ github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 h1:au07oEsX2xN0kt github.com/golang-sql/sqlexp v0.1.0 h1:ZCD6MBpcuOVfGVqsEmY5/4FtYiKz6tSyUv9LPEDei6A= github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 h1:DACJavvAHhabrF08vX0COfcOBJRhZ8lUbR+ZWIs0Y5g= github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/glog v1.1.0 h1:/d3pCKDPWNnvIWe0vVUpNP32qc8U3PDVxySP/y360qE= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef h1:veQD95Isof8w9/WXiA+pa3tz3fJXkt5B7QaRBrM62gk= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -191,8 +206,8 @@ github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvq github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= -github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= @@ -206,6 +221,7 @@ github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -216,7 +232,7 @@ github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/mux v1.7.4 h1:VuZ8uybHlWmqV03+zRzdwKL4tUnIp1MAQtp1mIFE1bc= github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= @@ -256,9 +272,7 @@ github.com/jinzhu/now v1.1.2/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/ github.com/jinzhu/now v1.1.4/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= -github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= -github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/joho/godotenv v1.4.0 h1:3l4+N6zfMWnkbPEXKng2o2/MR5mSwTrBih4ZEkkz1lg= github.com/joho/godotenv v1.4.0/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= @@ -279,6 +293,7 @@ github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/ github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= @@ -299,18 +314,16 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w= github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= -github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= +github.com/lufia/plan9stats v0.0.0-20230326075908-cb1d2100619a h1:N9zuLhTvBSRt0gWSiJswwQ2HqDmtX/ZCDJURnKUt1Ik= +github.com/lufia/plan9stats v0.0.0-20230326075908-cb1d2100619a/go.mod h1:JKx41uQRwqlTZabZc+kILPrO/3jlKnQ2Z8b7YiVw5cE= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA= github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= @@ -348,6 +361,11 @@ github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1Cpa github.com/onsi/gomega v1.20.1 h1:PA/3qinGoukvymdIDV8pii6tiZgC8kbmJO6Z5+b002Q= github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= +github.com/otiai10/copy v1.7.0/go.mod h1:rmRl6QPdJj6EiUqXQ/4Nn2lLXoNQjFCQbbNrxgc/t3U= +github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95/go.mod h1:9qAhocn7zKJG+0mI8eUu6xqkFDYS2kb2saOteoSB3cE= +github.com/otiai10/curr v1.0.0/go.mod h1:LskTG5wDwr8Rs+nNQ+1LlxRjAtTZZjtJW4rMXl6j4vs= +github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT91xUo= +github.com/otiai10/mint v1.3.3/go.mod h1:/yxELlJQ0ufhjUwhshSj+wFjZ78CnZ48/1wtmBH1OTc= github.com/pascaldekloe/name v0.0.0-20180628100202-0fd16699aae1/go.mod h1:eD5JxqMiuNYyFNmyY9rkJ/slN8y59oEu4Ei7F8OoKWQ= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml/v2 v2.0.1 h1:8e3L2cCQzLFi2CR4g7vGFuFxX7Jl1kKX8gW+iV0GUKU= @@ -357,7 +375,6 @@ github.com/petermattis/goid v0.0.0-20211229010228-4d14c490ee36/go.mod h1:pxMtw7c github.com/phf/go-queue v0.0.0-20170504031614-9abe38d0371d h1:U+PMnTlV2tu7RuMK5etusZG3Cf+rpow5hqQByeCzJ2g= github.com/phf/go-queue v0.0.0-20170504031614-9abe38d0371d/go.mod h1:lXfE4PvvTW5xOjO6Mba8zDPyw8M93B6AQ7frTGnMlA8= github.com/pingcap/check v0.0.0-20190102082844-67f458068fc8/go.mod h1:B1+S9LNcuMyLH/4HMTViQOJevkGiik3wW2AN9zb2fNQ= -github.com/pingcap/check v0.0.0-20191107115940-caf2b9e6ccf4/go.mod h1:PYMCGwN0JHjoqGr3HrZoD+b8Tgx8bKnArhSq8YVzUMc= github.com/pingcap/check v0.0.0-20211026125417-57bd13f7b5f0 h1:HVl5539r48eA+uDuX/ziBmQCxzT1pGrzWbKuXT46Bq0= github.com/pingcap/check v0.0.0-20211026125417-57bd13f7b5f0/go.mod h1:PYMCGwN0JHjoqGr3HrZoD+b8Tgx8bKnArhSq8YVzUMc= github.com/pingcap/errcode v0.3.0 h1:IF6LC/4+b1KNwrMlr2rBTUrojFPMexXBcDWZSpNwxjg= @@ -370,16 +387,16 @@ github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c/go.mod h1:X2r9ue github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00 h1:C3N3itkduZXDZFh4N3vQ5HEtld3S+Y+StULhWVvumU0= github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00/go.mod h1:4qGtCB0QK0wBzKtFEGDhxXnSnbQApw1gc9siScUl8ew= github.com/pingcap/kvproto v0.0.0-20191211054548-3c6b38ea5107/go.mod h1:WWLmULLO7l8IOcQG+t+ItJ3fEcrL5FxF0Wu+HrMy26w= -github.com/pingcap/kvproto v0.0.0-20230228041042-1e9aca94bab6 h1:bgLRG7gPJCq6aduA65ZV7xWQBThTcuarBB9VdfAzV4g= -github.com/pingcap/kvproto v0.0.0-20230228041042-1e9aca94bab6/go.mod h1:KUrW1FGoznGMMTssYBu0czfAhn6vQcIrHyZoSC6T990= +github.com/pingcap/kvproto v0.0.0-20230727073445-53e1f8730c30 h1:EvqKcDT7ceGLW0mXqM8Cp5Z8DfgQRnwj2YTnlCLj2QI= +github.com/pingcap/kvproto v0.0.0-20230727073445-53e1f8730c30/go.mod h1:r0q/CFcwvyeRhKtoqzmWMBebrtpIziQQ9vR+JKh1knc= github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8= github.com/pingcap/log v0.0.0-20210625125904-98ed8e2eb1c7/go.mod h1:8AanEdAHATuRurdGxZXBz0At+9avep+ub7U1AGYLIMM= github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 h1:HR/ylkkLmGdSSDaD8IDP+SZrdhV1Kibl9KrHxJ9eciw= github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4= -github.com/pingcap/sysutil v0.0.0-20211208032423-041a72e5860d h1:k3/APKZjXOyJrFy8VyYwRlZhMelpD3qBLJNsw3bPl/g= -github.com/pingcap/sysutil v0.0.0-20211208032423-041a72e5860d/go.mod h1:7j18ezaWTao2LHOyMlsc2Dg1vW+mDY9dEbPzVyOlaeM= -github.com/pingcap/tidb-dashboard v0.0.0-20230209052558-a58fc2a7e924 h1:49x3JR5zEYqjVqONKV9r/nrv0Rh5QU8ivIhktoLvP4g= -github.com/pingcap/tidb-dashboard v0.0.0-20230209052558-a58fc2a7e924/go.mod h1:OUzFMMVjR1GKlf4LWLqza9QNKjCrYJ7stVn/3PN0djM= +github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21 h1:QV6jqlfOkh8hqvEAgwBZa+4bSgO0EeKC7s5c6Luam2I= +github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21/go.mod h1:QYnjfA95ZaMefyl1NO8oPtKeb8pYUdnDVhQgf+qdpjM= +github.com/pingcap/tidb-dashboard v0.0.0-20230705095454-5e220f970f27 h1:K9lZMYuDuAiR5kOjFESwJ8KfSb4ui5zX6vZGbUp58uk= +github.com/pingcap/tidb-dashboard v0.0.0-20230705095454-5e220f970f27/go.mod h1:OUzFMMVjR1GKlf4LWLqza9QNKjCrYJ7stVn/3PN0djM= github.com/pingcap/tipb v0.0.0-20220718022156-3e2483c20a9e h1:FBaTXU8C3xgt/drM58VHxojHo/QoG1oPsgWTGvaSpO4= github.com/pingcap/tipb v0.0.0-20220718022156-3e2483c20a9e/go.mod h1:A7mrd7WHBl1o63LE2bIBGEJMTNWXqhgmYiOvMLxozfs= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= @@ -389,8 +406,9 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw= github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= +github.com/power-devops/perfstat v0.0.0-20221212215047-62379fc7944b h1:0LFwY6Q3gMACTjAbMZBjXAqTOzOwFaj2Ld6cjeQ7Rig= +github.com/power-devops/perfstat v0.0.0-20221212215047-62379fc7944b/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= @@ -429,10 +447,14 @@ github.com/samber/lo v1.37.0/go.mod h1:9vaz2O4o8oOnK23pd2TrXufcbdbJIa3b6cstBWKpo github.com/sasha-s/go-deadlock v0.2.0 h1:lMqc+fUb7RrFS3gQLtoQsJ7/6TV/pAIFvBsqX73DK8Y= github.com/sasha-s/go-deadlock v0.2.0/go.mod h1:StQn567HiB1fF2yJ44N9au7wOhrPS3iZqiDbRupzT10= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= -github.com/shirou/gopsutil v3.21.3+incompatible h1:uenXGGa8ESCQq+dbgtl916dmg6PSAz2cXov0uORQ9v8= -github.com/shirou/gopsutil v3.21.3+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= -github.com/shirou/gopsutil/v3 v3.22.12 h1:oG0ns6poeUSxf78JtOsfygNWuEHYYz8hnnNg7P04TJs= -github.com/shirou/gopsutil/v3 v3.22.12/go.mod h1:Xd7P1kwZcp5VW52+9XsirIKd/BROzbb2wdX3Kqlz9uI= +github.com/shirou/gopsutil/v3 v3.21.12/go.mod h1:BToYZVTlSVlfazpDDYFnsVZLaoRG+g8ufT6fPQLdJzA= +github.com/shirou/gopsutil/v3 v3.23.3 h1:Syt5vVZXUDXPEXpIBt5ziWsJ4LdSAAxF4l/xZeQgSEE= +github.com/shirou/gopsutil/v3 v3.23.3/go.mod h1:lSBNN6t3+D6W5e5nXTxc8KIMMVxAcS+6IJlffjRRlMU= +github.com/shoenig/go-m1cpu v0.1.4/go.mod h1:Wwvst4LR89UxjeFtLRMrpgRiyY4xPsejnVZym39dbAQ= +github.com/shoenig/go-m1cpu v0.1.5 h1:LF57Z/Fpb/WdGLjt2HZilNnmZOxg/q2bSKTQhgbrLrQ= +github.com/shoenig/go-m1cpu v0.1.5/go.mod h1:Wwvst4LR89UxjeFtLRMrpgRiyY4xPsejnVZym39dbAQ= +github.com/shoenig/test v0.6.3 h1:GVXWJFk9PiOjN0KoJ7VrJGH6uLPnqxR7/fe3HUPfE0c= +github.com/shoenig/test v0.6.3/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k= github.com/shurcooL/httpgzip v0.0.0-20190720172056-320755c1c1b0 h1:mj/nMDAwTBiaCqMEs4cYCqF7pO6Np7vhy1D1wcQGz+E= github.com/shurcooL/httpgzip v0.0.0-20190720172056-320755c1c1b0/go.mod h1:919LwcH0M7/W4fcZ0/jy0qGght1GIhqyS/EgWGH2j5Q= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= @@ -442,6 +464,8 @@ github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/smallnest/chanx v0.0.0-20221229104322-eb4c998d2072 h1:Txo4SXVJq/OgEjwgkWoxkMoTjGlcrgsQE/XSghjmu0w= github.com/smallnest/chanx v0.0.0-20221229104322-eb4c998d2072/go.mod h1:+4nWMF0+CqEcU74SnX2NxaGqZ8zX4pcQ8Jcs77DbX5A= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.4 h1:0HKaf1o97UwFjHH9o5XsHUOF+tqmdA7KEzXLpiyaw0E= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= @@ -469,15 +493,13 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= -github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/swaggo/files v0.0.0-20190704085106-630677cd5c14 h1:PyYN9JH5jY9j6av01SpfRMb+1DWg/i3MbGOKPxJ2wjM= -github.com/swaggo/files v0.0.0-20190704085106-630677cd5c14/go.mod h1:gxQT6pBGRuIGunNf/+tSOB5OHvguWi8Tbt82WOkf35E= -github.com/swaggo/gin-swagger v1.2.0/go.mod h1:qlH2+W7zXGZkczuL+r2nEBR2JTT+/lX05Nn6vPhc7OI= -github.com/swaggo/http-swagger v0.0.0-20200308142732-58ac5e232fba h1:lUPlXKqgbqT2SVg2Y+eT9mu5wbqMnG+i/+Q9nK7C0Rs= -github.com/swaggo/http-swagger v0.0.0-20200308142732-58ac5e232fba/go.mod h1:O1lAbCgAAX/KZ80LM/OXwtWFI/5TvZlwxSg8Cq08PV0= -github.com/swaggo/swag v1.5.1/go.mod h1:1Bl9F/ZBpVWh22nY0zmYyASPO1lI/zIwRDrpZU+tv8Y= -github.com/swaggo/swag v1.6.3/go.mod h1:wcc83tB4Mb2aNiL/HP4MFeQdpHUrca+Rp/DRNgWAUio= +github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= +github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/swaggo/files v0.0.0-20210815190702-a29dd2bc99b2 h1:+iNTcqQJy0OZ5jk6a5NLib47eqXK8uYcPX+O4+cBpEM= +github.com/swaggo/files v0.0.0-20210815190702-a29dd2bc99b2/go.mod h1:lKJPbtWzJ9JhsTN1k1gZgleJWY/cqq0psdoMmaThG3w= +github.com/swaggo/http-swagger v1.2.6 h1:ihTjChUoSRMpFMjWw+0AkL1Ti4r6v8pCgVYLmQVRlRw= +github.com/swaggo/http-swagger v1.2.6/go.mod h1:CcoICgY3yVDk2u1LQUCMHbAj0fjlxIX+873psXlIKNA= +github.com/swaggo/swag v1.7.9/go.mod h1:gZ+TJ2w/Ve1RwQsA2IRoSOTidHz6DX+PIG8GWvbnoLU= github.com/swaggo/swag v1.8.3 h1:3pZSSCQ//gAH88lfmxM3Cd1+JCsxV8Md6f36b9hrZ5s= github.com/swaggo/swag v1.8.3/go.mod h1:jMLeXOOmYyjk8PvHTsXBdrubsNd9gUJTTCzL5iBnseg= github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965 h1:1oFLiOyVl+W7bnBzGhf7BbIv9loSFQcieWWYIjLqcAw= @@ -488,27 +510,26 @@ github.com/tidwall/match v1.0.1 h1:PnKP62LPNxHKTwvHHZZzdOAOCtsJTjo6dZLCwpKm5xc= github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E= github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= -github.com/tklauser/go-sysconf v0.3.4/go.mod h1:Cl2c8ZRWfHD5IrfHo9VN+FX9kCFjIOyVklgXycLB6ek= +github.com/tklauser/go-sysconf v0.3.9/go.mod h1:11DU/5sG7UexIrp/O6g35hrWzu0JxlwQ3LSFUzyeuhs= github.com/tklauser/go-sysconf v0.3.11 h1:89WgdJhk5SNwJfu+GKyYveZ4IaJ7xAkecBo+KdJV0CM= github.com/tklauser/go-sysconf v0.3.11/go.mod h1:GqXfhXY3kiPa0nAXPDIQIWzJbMCB7AmcWpGR8lSZfqI= -github.com/tklauser/numcpus v0.2.1/go.mod h1:9aU+wOc6WjUIZEwWMP62PL/41d65P+iks1gBkr4QyP8= +github.com/tklauser/numcpus v0.3.0/go.mod h1:yFGUr7TUHQRAhyqBcEg0Ge34zDBAsIvJJcyE6boqnA8= github.com/tklauser/numcpus v0.6.0 h1:kebhY2Qt+3U6RNK7UqpYNA+tJ23IBEGKkB7JQBfDYms= github.com/tklauser/numcpus v0.6.0/go.mod h1:FEZLMke0lhOUG6w2JadTzp0a+Nl8PF/GFkQ5UVIcaL4= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20200427203606-3cfed13b9966 h1:j6JEOq5QWFker+d7mFQYOhjTZonQ7YkLTHm56dbn+yM= github.com/tmc/grpc-websocket-proxy v0.0.0-20200427203606-3cfed13b9966/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= -github.com/ugorji/go v1.1.5-pre/go.mod h1:FwP/aQVg39TXzItUBMwnWp9T9gPQnXw4Poh4/oBQZ/0= github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= github.com/ugorji/go v1.2.7/go.mod h1:nF9osbDWLy6bDVv/Rtoh6QgnvNDpmCalQV5urGCCS6M= github.com/ugorji/go/codec v0.0.0-20181022190402-e5e69e061d4f/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= -github.com/ugorji/go/codec v1.1.5-pre/go.mod h1:tULtS6Gy1AE1yCENaw4Vb//HLH5njI2tfCQDUqRd8fI= github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= github.com/ugorji/go/codec v1.2.7 h1:YPXUKf7fYbp/y8xloBqZOw2qaVggbfwMlI8WM3wZUJ0= github.com/ugorji/go/codec v1.2.7/go.mod h1:WGN1fab3R1fzQlVQTkfxVtIBhWDRqOviHU95kRgeqEY= github.com/unrolled/render v1.0.1 h1:VDDnQQVfBMsOsp3VaCJszSO0nkBIVEYoPWeRThk9spY= github.com/unrolled/render v1.0.1/go.mod h1:gN9T0NhL4Bfbwu8ann7Ry/TGHYfosul+J0obPf6NBdM= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= +github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= github.com/urfave/negroni v0.3.0 h1:PaXOb61mWeZJxc1Ji2xJjpVg9QfPo0rrB+lHyBxGNSU= github.com/urfave/negroni v0.3.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4= github.com/vmihailenco/msgpack/v5 v5.3.5 h1:5gO0H1iULLWGhs2H5tbAHIZTV8/cYafcFOr9znI5mJU= @@ -521,6 +542,7 @@ github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1: github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg= github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= @@ -543,7 +565,6 @@ go.uber.org/fx v1.12.0 h1:+1+3Cz9M0dFMPy9SW9XUIUHye8bnPUm7q7DroNGWYG4= go.uber.org/fx v1.12.0/go.mod h1:egT3Kyg1JFYQkvKLZ3EsykxkNrZxgXS+gKoKo7abERY= go.uber.org/goleak v0.10.0/go.mod h1:VCZuO8V8mFPlL0F5J5GK1rtHV3DrFcQ1R8ryq7FK0aI= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= -go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= @@ -551,16 +572,17 @@ go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+ go.uber.org/multierr v1.4.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= -go.uber.org/multierr v1.7.0 h1:zaiO/rmgFjbmCXdSYJWQcdvOCsthmdaHfr3Gm2Kx4Ec= go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.12.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= -go.uber.org/zap v1.20.0 h1:N4oPlghZwYG55MlU6LXk/Zp00FVNE9X9wrYO8CEs4lc= -go.uber.org/zap v1.20.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw= +go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= +go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -574,8 +596,8 @@ golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.1.0 h1:MDRAIl0xIo9Io2xV565hzXHw3zVseKrJKodhohM5CjU= golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20230108222341-4b8118a2686a h1:tlXy25amD5A7gOfbXdqCGN5k8ESEed/Ee1E5RcrYnqU= -golang.org/x/exp v0.0.0-20230108222341-4b8118a2686a/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= +golang.org/x/exp v0.0.0-20230711005742-c3f37128e5a4 h1:QLureRX3moex6NVu/Lr4MGakp9FdA7sBHGBmvRW7NaM= +golang.org/x/exp v0.0.0-20230711005742-c3f37128e5a4/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= golang.org/x/image v0.0.0-20200119044424-58c23975cae1/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/image v0.5.0 h1:5JMiNunQeQw++mMOz48/ISeNu3Iweh/JaZU8ZLqHRrI= golang.org/x/image v0.5.0/go.mod h1:FVC7BI/5Ym8R25iw5OLsgshdUBbT1h5jZTpA+mvAdZ4= @@ -591,7 +613,7 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.6.0 h1:b9gGHsz9/HhJ3HF5DHQytPpuwocVTChQJK3AvoLRD5I= +golang.org/x/mod v0.11.0 h1:bUO06HqtnRcc/7l71XBe4WcqTZ+3AH1J59zWDDwLKgU= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -602,13 +624,10 @@ golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= -golang.org/x/net v0.0.0-20190611141213-3f473d35a33a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= @@ -617,15 +636,16 @@ golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM= +golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.2.0 h1:sZfSu1wtKLGlWI4ZZayP0ck9Y73K1ynO6gqzTdBVdPU= -golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= +golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783 h1:nt+Q6cXKz4MosCSpnbMtqiQ8Oz0pxTef2B4Vca2lvfk= -golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= +golang.org/x/oauth2 v0.4.0 h1:NF0gk8LVPg1Ml7SSbGyySuoxdsXitj7TvgvuRxIMc/M= +golang.org/x/oauth2 v0.4.0/go.mod h1:RznEsdpjGAINPTOF0UH/t+xJ75L18YO3Ho6Pyn+uRec= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -644,10 +664,8 @@ golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181228144115-9a3f9b0469bb/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190610200419-93c9922d18ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -659,7 +677,6 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210217105451-b926d437f341/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -668,13 +685,17 @@ golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210816074244-15123e1e1f71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211013075003-97ac67df715c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ= -golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -682,8 +703,9 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.1.0 h1:xYY+Bajn2a7VBmTM5GikTmnK8ZuX8YgnQCqZpbBNtmA= @@ -693,10 +715,9 @@ golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524210228-3d17549cdc6b/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190606050223-4d9ae51c2468/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190611222205-d73e1c7e250b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -711,9 +732,10 @@ golang.org/x/tools v0.0.0-20201125231158-b5590deeca9b/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210112230658-8b4aab62c064/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.2.0 h1:G6AHpWxTMGY1KyEYoAQ5WTtIekUUvDNjan3ugu60JvE= -golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA= +golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -726,8 +748,8 @@ google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoA google.golang.org/genproto v0.0.0-20181004005441-af9cb2a35e7f/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20221202195650-67e5cbc046fd h1:OjndDrsik+Gt+e6fs45z9AxiewiKyLKYpA45W5Kpkks= -google.golang.org/genproto v0.0.0-20221202195650-67e5cbc046fd/go.mod h1:cTsE614GARnxrLsqKREzmNYJACSWWpAWdNMwnD7c2BE= +google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 h1:KpwkzHKEF7B9Zxg18WzOa7djJ+Ha5DzthMyZYQfEn2A= +google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU= google.golang.org/grpc v0.0.0-20180607172857-7a6a684ca69e/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -747,8 +769,8 @@ google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= -google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= +google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -762,8 +784,9 @@ gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= gopkg.in/go-playground/validator.v8 v8.18.2/go.mod h1:RX2a/7Ha8BgOhfk7j780h4/u/RRjR0eouCJSH80/M2Y= -gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= +gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= +gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= diff --git a/tests/client/keyspace_test.go b/tests/integrations/client/keyspace_test.go similarity index 65% rename from tests/client/keyspace_test.go rename to tests/integrations/client/keyspace_test.go index a9fb953cdf5..df396808d8e 100644 --- a/tests/client/keyspace_test.go +++ b/tests/integrations/client/keyspace_test.go @@ -16,33 +16,37 @@ package client_test import ( "fmt" + "math" "time" "github.com/pingcap/kvproto/pkg/keyspacepb" "github.com/stretchr/testify/require" + "github.com/tikv/pd/pkg/keyspace" + "github.com/tikv/pd/pkg/mcs/utils" "github.com/tikv/pd/pkg/slice" "github.com/tikv/pd/server" - "github.com/tikv/pd/server/keyspace" ) const ( - testConfig1 = "config_entry_1" - testConfig2 = "config_entry_2" + testConfig1 = "config_entry_1" + testConfig2 = "config_entry_2" + testKeyspaceCount = 10 ) -func mustMakeTestKeyspaces(re *require.Assertions, server *server.Server, start, count int) []*keyspacepb.KeyspaceMeta { +func mustMakeTestKeyspaces(re *require.Assertions, server *server.Server, start int) []*keyspacepb.KeyspaceMeta { now := time.Now().Unix() var err error - keyspaces := make([]*keyspacepb.KeyspaceMeta, count) + keyspaces := make([]*keyspacepb.KeyspaceMeta, testKeyspaceCount) manager := server.GetKeyspaceManager() - for i := 0; i < count; i++ { + for i := 0; i < testKeyspaceCount; i++ { keyspaces[i], err = manager.CreateKeyspace(&keyspace.CreateKeyspaceRequest{ - Name: fmt.Sprintf("test_keyspace%d", start+i), + Name: fmt.Sprintf("test_keyspace_%d", start+i), Config: map[string]string{ testConfig1: "100", testConfig2: "200", }, - Now: now, + CreateTime: now, + IsPreAlloc: true, // skip wait region split }) re.NoError(err) } @@ -51,7 +55,7 @@ func mustMakeTestKeyspaces(re *require.Assertions, server *server.Server, start, func (suite *clientTestSuite) TestLoadKeyspace() { re := suite.Require() - metas := mustMakeTestKeyspaces(re, suite.srv, 0, 10) + metas := mustMakeTestKeyspaces(re, suite.srv, 0) for _, expected := range metas { loaded, err := suite.client.LoadKeyspace(suite.ctx, expected.GetName()) re.NoError(err) @@ -61,68 +65,46 @@ func (suite *clientTestSuite) TestLoadKeyspace() { _, err := suite.client.LoadKeyspace(suite.ctx, "non-existing keyspace") re.Error(err) // Loading default keyspace should be successful. - keyspaceDefault, err := suite.client.LoadKeyspace(suite.ctx, keyspace.DefaultKeyspaceName) + keyspaceDefault, err := suite.client.LoadKeyspace(suite.ctx, utils.DefaultKeyspaceName) re.NoError(err) - re.Equal(keyspace.DefaultKeyspaceID, keyspaceDefault.GetId()) - re.Equal(keyspace.DefaultKeyspaceName, keyspaceDefault.GetName()) + re.Equal(utils.DefaultKeyspaceID, keyspaceDefault.GetId()) + re.Equal(utils.DefaultKeyspaceName, keyspaceDefault.GetName()) } -func (suite *clientTestSuite) TestWatchKeyspaces() { +func (suite *clientTestSuite) TestGetAllKeyspaces() { re := suite.Require() - initialKeyspaces := mustMakeTestKeyspaces(re, suite.srv, 10, 10) - watchChan, err := suite.client.WatchKeyspaces(suite.ctx) - re.NoError(err) - // First batch of watchChan message should contain all existing keyspaces. - initialLoaded := <-watchChan - for i := range initialKeyspaces { - re.Contains(initialLoaded, initialKeyspaces[i]) + metas := mustMakeTestKeyspaces(re, suite.srv, 20) + for _, expected := range metas { + loaded, err := suite.client.LoadKeyspace(suite.ctx, expected.GetName()) + re.NoError(err) + re.Equal(expected, loaded) } - // Each additional message contains extra put events. - additionalKeyspaces := mustMakeTestKeyspaces(re, suite.srv, 30, 10) + // Get all keyspaces. + resKeyspaces, err := suite.client.GetAllKeyspaces(suite.ctx, 1, math.MaxUint32) re.NoError(err) - // Checks that all additional keyspaces are captured by watch channel. - for i := 0; i < 10; { - loadedKeyspaces := <-watchChan - re.NotEmpty(loadedKeyspaces) - for j := range loadedKeyspaces { - re.Equal(additionalKeyspaces[i+j], loadedKeyspaces[j]) + re.Equal(len(metas), len(resKeyspaces)) + // Check expected keyspaces all in resKeyspaces. + for _, expected := range metas { + var isExists bool + for _, resKeyspace := range resKeyspaces { + if expected.GetName() == resKeyspace.GetName() { + isExists = true + continue + } + } + if !isExists { + re.Fail("not exists keyspace") } - i += len(loadedKeyspaces) } - // Updates to state should also be captured. - expected, err := suite.srv.GetKeyspaceManager().UpdateKeyspaceState(initialKeyspaces[0].Name, keyspacepb.KeyspaceState_DISABLED, time.Now().Unix()) - re.NoError(err) - loaded := <-watchChan - re.Equal([]*keyspacepb.KeyspaceMeta{expected}, loaded) - // Updates to config should also be captured. - expected, err = suite.srv.GetKeyspaceManager().UpdateKeyspaceConfig(initialKeyspaces[0].Name, []*keyspace.Mutation{ - { - Op: keyspace.OpDel, - Key: testConfig1, - }, - }) - re.NoError(err) - loaded = <-watchChan - re.Equal([]*keyspacepb.KeyspaceMeta{expected}, loaded) - // Updates to default keyspace's config should also be captured. - expected, err = suite.srv.GetKeyspaceManager().UpdateKeyspaceConfig(keyspace.DefaultKeyspaceName, []*keyspace.Mutation{ - { - Op: keyspace.OpPut, - Key: "config", - Value: "value", - }, - }) - re.NoError(err) - loaded = <-watchChan - re.Equal([]*keyspacepb.KeyspaceMeta{expected}, loaded) } func mustCreateKeyspaceAtState(re *require.Assertions, server *server.Server, index int, state keyspacepb.KeyspaceState) *keyspacepb.KeyspaceMeta { manager := server.GetKeyspaceManager() meta, err := manager.CreateKeyspace(&keyspace.CreateKeyspaceRequest{ - Name: fmt.Sprintf("test_keyspace%d", index), - Config: nil, - Now: 0, // Use 0 to indicate unchanged keyspace. + Name: fmt.Sprintf("test_keyspace_%d", index), + Config: nil, + CreateTime: 0, // Use 0 to indicate unchanged keyspace. + IsPreAlloc: true, // skip wait region split }) re.NoError(err) switch state { diff --git a/tests/integrations/mcs/Makefile b/tests/integrations/mcs/Makefile new file mode 100644 index 00000000000..01e63b5baec --- /dev/null +++ b/tests/integrations/mcs/Makefile @@ -0,0 +1,49 @@ +# Copyright 2023 TiKV Project Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ROOT_PATH := ../../.. +GO_TOOLS_BIN_PATH := $(ROOT_PATH)/.tools/bin +PATH := $(GO_TOOLS_BIN_PATH):$(PATH) +SHELL := env PATH='$(PATH)' GOBIN='$(GO_TOOLS_BIN_PATH)' $(shell which bash) + +static: install-tools + @ echo "gofmt ..." + @ gofmt -s -l -d . 2>&1 | awk '{ print } END { if (NR > 0) { exit 1 } }' + @ echo "golangci-lint ..." + @ golangci-lint run -c $(ROOT_PATH)/.golangci.yml --verbose ./... --allow-parallel-runners + @ echo "revive ..." + @ revive -formatter friendly -config $(ROOT_PATH)/revive.toml ./... + +tidy: + @ go mod tidy + git diff go.mod go.sum | cat + git diff --quiet go.mod go.sum + +test: failpoint-enable + CGO_ENABLED=1 go test ./... -v -tags deadlock -race -cover || { $(MAKE) failpoint-disable && exit 1; } + $(MAKE) failpoint-disable + +ci-test-job: + CGO_ENABLED=1 go test ./... -v -timeout=15m -tags deadlock -race -covermode=atomic -coverprofile=covprofile -coverpkg=$(ROOT_PATH)/... github.com/tikv/pd/tests/integrations/mcs + +install-tools: + cd $(ROOT_PATH) && $(MAKE) install-tools + +failpoint-enable: + cd $(ROOT_PATH) && $(MAKE) failpoint-enable + go mod tidy + +failpoint-disable: + cd $(ROOT_PATH) && $(MAKE) failpoint-disable + go mod tidy diff --git a/tests/integrations/mcs/discovery/register_test.go b/tests/integrations/mcs/discovery/register_test.go new file mode 100644 index 00000000000..2cdcccedddd --- /dev/null +++ b/tests/integrations/mcs/discovery/register_test.go @@ -0,0 +1,164 @@ +// Copyright 2023 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package register_test + +import ( + "context" + "strconv" + "testing" + "time" + + "github.com/stretchr/testify/suite" + bs "github.com/tikv/pd/pkg/basicserver" + "github.com/tikv/pd/pkg/mcs/discovery" + "github.com/tikv/pd/pkg/mcs/utils" + "github.com/tikv/pd/pkg/utils/tempurl" + "github.com/tikv/pd/pkg/utils/testutil" + "github.com/tikv/pd/tests" + "go.uber.org/goleak" +) + +func TestMain(m *testing.M) { + goleak.VerifyTestMain(m, testutil.LeakOptions...) +} + +type serverRegisterTestSuite struct { + suite.Suite + ctx context.Context + cancel context.CancelFunc + cluster *tests.TestCluster + pdLeader *tests.TestServer + clusterID string + backendEndpoints string +} + +func TestServerRegisterTestSuite(t *testing.T) { + suite.Run(t, new(serverRegisterTestSuite)) +} + +func (suite *serverRegisterTestSuite) SetupSuite() { + var err error + re := suite.Require() + + suite.ctx, suite.cancel = context.WithCancel(context.Background()) + suite.cluster, err = tests.NewTestAPICluster(suite.ctx, 1) + re.NoError(err) + + err = suite.cluster.RunInitialServers() + re.NoError(err) + + leaderName := suite.cluster.WaitLeader() + suite.pdLeader = suite.cluster.GetServer(leaderName) + suite.clusterID = strconv.FormatUint(suite.pdLeader.GetClusterID(), 10) + suite.backendEndpoints = suite.pdLeader.GetAddr() +} + +func (suite *serverRegisterTestSuite) TearDownSuite() { + suite.cluster.Destroy() + suite.cancel() +} + +func (suite *serverRegisterTestSuite) TestServerRegister() { + // test register, primary and unregister when start tso and resource-manager with only one server + for i := 0; i < 3; i++ { + suite.checkServerRegister(utils.TSOServiceName) + } + // TODO: uncomment after resource-manager is ready + // for i := 0; i < 3; i++ { + // suite.checkServerRegister(utils.ResourceManagerServiceName) + // } +} + +func (suite *serverRegisterTestSuite) checkServerRegister(serviceName string) { + re := suite.Require() + s, cleanup := suite.addServer(serviceName) + + addr := s.GetAddr() + client := suite.pdLeader.GetEtcdClient() + // test API server discovery + + endpoints, err := discovery.Discover(client, suite.clusterID, serviceName) + re.NoError(err) + returnedEntry := &discovery.ServiceRegistryEntry{} + returnedEntry.Deserialize([]byte(endpoints[0])) + re.Equal(addr, returnedEntry.ServiceAddr) + + // test primary when only one server + expectedPrimary := tests.WaitForPrimaryServing(suite.Require(), map[string]bs.Server{addr: s}) + primary, exist := suite.pdLeader.GetServer().GetServicePrimaryAddr(suite.ctx, serviceName) + re.True(exist) + re.Equal(primary, expectedPrimary) + + // test API server discovery after unregister + cleanup() + endpoints, err = discovery.Discover(client, suite.clusterID, serviceName) + re.NoError(err) + re.Empty(endpoints) + testutil.Eventually(re, func() bool { + return !s.IsServing() + }, testutil.WithWaitFor(3*time.Second), testutil.WithTickInterval(50*time.Millisecond)) +} + +func (suite *serverRegisterTestSuite) TestServerPrimaryChange() { + suite.checkServerPrimaryChange(utils.TSOServiceName, 3) + // TODO: uncomment after resource-manager is ready + // suite.checkServerPrimaryChange(utils.ResourceManagerServiceName, 3) +} + +func (suite *serverRegisterTestSuite) checkServerPrimaryChange(serviceName string, serverNum int) { + re := suite.Require() + primary, exist := suite.pdLeader.GetServer().GetServicePrimaryAddr(suite.ctx, serviceName) + re.False(exist) + re.Empty(primary) + + serverMap := make(map[string]bs.Server) + for i := 0; i < serverNum; i++ { + s, cleanup := suite.addServer(serviceName) + defer cleanup() + serverMap[s.GetAddr()] = s + } + + expectedPrimary := tests.WaitForPrimaryServing(suite.Require(), serverMap) + primary, exist = suite.pdLeader.GetServer().GetServicePrimaryAddr(suite.ctx, serviceName) + re.True(exist) + re.Equal(expectedPrimary, primary) + // close old primary + serverMap[primary].Close() + delete(serverMap, primary) + + expectedPrimary = tests.WaitForPrimaryServing(suite.Require(), serverMap) + // test API server discovery + client := suite.pdLeader.GetEtcdClient() + endpoints, err := discovery.Discover(client, suite.clusterID, serviceName) + re.NoError(err) + re.Len(endpoints, serverNum-1) + + // test primary changed + primary, exist = suite.pdLeader.GetServer().GetServicePrimaryAddr(suite.ctx, serviceName) + re.True(exist) + re.Equal(expectedPrimary, primary) +} + +func (suite *serverRegisterTestSuite) addServer(serviceName string) (bs.Server, func()) { + re := suite.Require() + switch serviceName { + case utils.TSOServiceName: + return tests.StartSingleTSOTestServer(suite.ctx, re, suite.backendEndpoints, tempurl.Alloc()) + case utils.ResourceManagerServiceName: + return tests.StartSingleResourceManagerTestServer(suite.ctx, re, suite.backendEndpoints, tempurl.Alloc()) + default: + return nil, nil + } +} diff --git a/tests/mcs/go.mod b/tests/integrations/mcs/go.mod similarity index 73% rename from tests/mcs/go.mod rename to tests/integrations/mcs/go.mod index 82b1f6cc5d2..738c9f56a74 100644 --- a/tests/mcs/go.mod +++ b/tests/integrations/mcs/go.mod @@ -1,18 +1,26 @@ -module github.com/tikv/pd/tests/mcs +module github.com/tikv/pd/tests/integrations/mcs go 1.20 +replace ( + github.com/tikv/pd => ../../../ + github.com/tikv/pd/client => ../../../client +) + +// reset grpc and protobuf deps in order to import client and server at the same time +replace google.golang.org/grpc v1.54.0 => google.golang.org/grpc v1.26.0 + require ( github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00 - github.com/pingcap/kvproto v0.0.0-20230228041042-1e9aca94bab6 + github.com/pingcap/kvproto v0.0.0-20230727073445-53e1f8730c30 github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 - github.com/spf13/cobra v1.0.0 - github.com/stretchr/testify v1.8.1 + github.com/stretchr/testify v1.8.2 github.com/tikv/pd v0.0.0-00010101000000-000000000000 github.com/tikv/pd/client v0.0.0-00010101000000-000000000000 go.etcd.io/etcd v0.5.0-alpha.5.0.20220915004622-85b640cee793 go.uber.org/goleak v1.1.12 - google.golang.org/grpc v1.51.0 + go.uber.org/zap v1.24.0 + google.golang.org/grpc v1.54.0 ) require ( @@ -22,28 +30,40 @@ require ( github.com/PuerkitoBio/purell v1.1.1 // indirect github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect github.com/ReneKroon/ttlcache/v2 v2.3.0 // indirect - github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect github.com/VividCortex/mysqlerr v1.0.0 // indirect github.com/Xeoncross/go-aesctr-with-hmac v0.0.0-20200623134604-12b17a7ff502 // indirect - github.com/aws/aws-sdk-go v1.35.3 // indirect - github.com/benbjohnson/clock v1.1.0 // indirect + github.com/aws/aws-sdk-go-v2 v1.17.7 // indirect + github.com/aws/aws-sdk-go-v2/config v1.18.19 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.13.18 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.1 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.31 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.25 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.3.32 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.25 // indirect + github.com/aws/aws-sdk-go-v2/service/kms v1.20.8 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.12.6 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.6 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.18.7 // indirect + github.com/aws/smithy-go v1.13.5 // indirect + github.com/benbjohnson/clock v1.3.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bitly/go-simplejson v0.5.0 // indirect github.com/breeswish/gin-jwt/v2 v2.6.4-jwt-patch // indirect github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 // indirect github.com/cenkalti/backoff/v4 v4.0.2 // indirect - github.com/cespare/xxhash/v2 v2.1.2 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/cloudfoundry/gosigar v1.3.6 // indirect github.com/coreos/go-semver v0.3.0 // indirect github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f // indirect github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/docker/go-units v0.4.0 // indirect github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4 // indirect - github.com/elastic/gosigar v0.14.2 // indirect github.com/elliotchance/pie/v2 v2.1.0 // indirect github.com/fogleman/gg v1.3.0 // indirect github.com/gin-contrib/cors v1.4.0 // indirect github.com/gin-contrib/gzip v0.0.1 // indirect + github.com/gin-contrib/pprof v1.4.0 // indirect github.com/gin-contrib/sse v0.1.0 // indirect github.com/gin-gonic/gin v1.8.1 // indirect github.com/go-ole/go-ole v1.2.6 // indirect @@ -61,7 +81,7 @@ require ( github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt v3.2.1+incompatible // indirect github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 // indirect - github.com/golang/protobuf v1.5.2 // indirect + github.com/golang/protobuf v1.5.3 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/btree v1.1.2 // indirect github.com/google/pprof v0.0.0-20211122183932-1daafda22083 // indirect @@ -76,7 +96,6 @@ require ( github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect - github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/joho/godotenv v1.4.0 // indirect github.com/jonboulle/clockwork v0.2.2 // indirect github.com/joomcode/errorx v1.0.1 // indirect @@ -84,7 +103,7 @@ require ( github.com/json-iterator/go v1.1.12 // indirect github.com/konsorten/go-windows-terminal-sequences v1.0.3 // indirect github.com/leodido/go-urn v1.2.1 // indirect - github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect + github.com/lufia/plan9stats v0.0.0-20230326075908-cb1d2100619a // indirect github.com/mailru/easyjson v0.7.6 // indirect github.com/mattn/go-isatty v0.0.14 // indirect github.com/mattn/go-sqlite3 v1.14.15 // indirect @@ -99,12 +118,12 @@ require ( github.com/phf/go-queue v0.0.0-20170504031614-9abe38d0371d // indirect github.com/pingcap/errcode v0.3.0 // indirect github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c // indirect - github.com/pingcap/sysutil v0.0.0-20211208032423-041a72e5860d // indirect - github.com/pingcap/tidb-dashboard v0.0.0-20230209052558-a58fc2a7e924 // indirect + github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21 // indirect + github.com/pingcap/tidb-dashboard v0.0.0-20230705095454-5e220f970f27 // indirect github.com/pingcap/tipb v0.0.0-20220718022156-3e2483c20a9e // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect + github.com/power-devops/perfstat v0.0.0-20221212215047-62379fc7944b // indirect github.com/prometheus/client_golang v1.11.1 // indirect github.com/prometheus/client_model v0.2.0 // indirect github.com/prometheus/common v0.26.0 // indirect @@ -112,16 +131,17 @@ require ( github.com/rs/cors v1.7.0 // indirect github.com/samber/lo v1.37.0 // indirect github.com/sasha-s/go-deadlock v0.2.0 // indirect - github.com/shirou/gopsutil v3.21.3+incompatible // indirect - github.com/shirou/gopsutil/v3 v3.22.12 // indirect + github.com/shirou/gopsutil/v3 v3.23.3 // indirect + github.com/shoenig/go-m1cpu v0.1.5 // indirect github.com/shurcooL/httpgzip v0.0.0-20190720172056-320755c1c1b0 // indirect github.com/sirupsen/logrus v1.6.0 // indirect github.com/smallnest/chanx v0.0.0-20221229104322-eb4c998d2072 // indirect github.com/soheilhy/cmux v0.1.4 // indirect + github.com/spf13/cobra v1.0.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/stretchr/objx v0.5.0 // indirect - github.com/swaggo/files v0.0.0-20190704085106-630677cd5c14 // indirect - github.com/swaggo/http-swagger v0.0.0-20200308142732-58ac5e232fba // indirect + github.com/swaggo/files v0.0.0-20210815190702-a29dd2bc99b2 // indirect + github.com/swaggo/http-swagger v1.2.6 // indirect github.com/swaggo/swag v1.8.3 // indirect github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965 // indirect github.com/tklauser/go-sysconf v0.3.11 // indirect @@ -138,22 +158,21 @@ require ( go.uber.org/atomic v1.10.0 // indirect go.uber.org/dig v1.9.0 // indirect go.uber.org/fx v1.12.0 // indirect - go.uber.org/multierr v1.7.0 // indirect - go.uber.org/zap v1.20.0 // indirect + go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.1.0 // indirect - golang.org/x/exp v0.0.0-20230108222341-4b8118a2686a // indirect - golang.org/x/image v0.0.0-20200119044424-58c23975cae1 // indirect - golang.org/x/net v0.2.0 // indirect - golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783 // indirect + golang.org/x/exp v0.0.0-20230711005742-c3f37128e5a4 // indirect + golang.org/x/image v0.5.0 // indirect + golang.org/x/net v0.10.0 // indirect + golang.org/x/oauth2 v0.4.0 // indirect golang.org/x/sync v0.1.0 // indirect - golang.org/x/sys v0.3.0 // indirect - golang.org/x/text v0.4.0 // indirect + golang.org/x/sys v0.8.0 // indirect + golang.org/x/text v0.9.0 // indirect golang.org/x/time v0.1.0 // indirect - golang.org/x/tools v0.2.0 // indirect + golang.org/x/tools v0.6.0 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20221202195650-67e5cbc046fd // indirect - google.golang.org/protobuf v1.28.1 // indirect - gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect + google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect + google.golang.org/protobuf v1.30.0 // indirect + gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect gorm.io/datatypes v1.1.0 // indirect @@ -163,11 +182,3 @@ require ( moul.io/zapgorm2 v1.1.0 // indirect sigs.k8s.io/yaml v1.2.0 // indirect ) - -replace ( - github.com/tikv/pd => ../../ - github.com/tikv/pd/client => ../../client -) - -// reset grpc and protobuf deps in order to import client and server at the same time -replace google.golang.org/grpc v1.51.0 => google.golang.org/grpc v1.26.0 diff --git a/tests/integrations/mcs/go.sum b/tests/integrations/mcs/go.sum new file mode 100644 index 00000000000..225d23d4e92 --- /dev/null +++ b/tests/integrations/mcs/go.sum @@ -0,0 +1,826 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc= +github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE= +github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww= +github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI= +github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M= +github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= +github.com/ReneKroon/ttlcache/v2 v2.3.0 h1:qZnUjRKIrbKHH6vF5T7Y9Izn5ObfTZfyYpGhvz2BKPo= +github.com/ReneKroon/ttlcache/v2 v2.3.0/go.mod h1:zbo6Pv/28e21Z8CzzqgYRArQYGYtjONRxaAKGxzQvG4= +github.com/VividCortex/mysqlerr v1.0.0 h1:5pZ2TZA+YnzPgzBfiUWGqWmKDVNBdrkf9g+DNe1Tiq8= +github.com/VividCortex/mysqlerr v1.0.0/go.mod h1:xERx8E4tBhLvpjzdUyQiSfUxeMcATEQrflDAfXsqcAE= +github.com/Xeoncross/go-aesctr-with-hmac v0.0.0-20200623134604-12b17a7ff502 h1:L8IbaI/W6h5Cwgh0n4zGeZpVK78r/jBf9ASurHo9+/o= +github.com/Xeoncross/go-aesctr-with-hmac v0.0.0-20200623134604-12b17a7ff502/go.mod h1:pmnBM9bxWSiHvC/gSWunUIyDvGn33EkP2CUjxFKtTTM= +github.com/agiledragon/gomonkey/v2 v2.3.1/go.mod h1:ap1AmDzcVOAz1YpeJ3TCzIgstoaWLA6jbbgxfB4w2iY= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= +github.com/alvaroloes/enumer v1.1.2/go.mod h1:FxrjvuXoDAx9isTJrv4c+T410zFi0DtXIT0m65DJ+Wo= +github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= +github.com/appleboy/gofight/v2 v2.1.2 h1:VOy3jow4vIK8BRQJoC/I9muxyYlJ2yb9ht2hZoS3rf4= +github.com/appleboy/gofight/v2 v2.1.2/go.mod h1:frW+U1QZEdDgixycTj4CygQ48yLTUhplt43+Wczp3rw= +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/aws/aws-sdk-go-v2 v1.17.7 h1:CLSjnhJSTSogvqUGhIC6LqFKATMRexcxLZ0i/Nzk9Eg= +github.com/aws/aws-sdk-go-v2 v1.17.7/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= +github.com/aws/aws-sdk-go-v2/config v1.18.19 h1:AqFK6zFNtq4i1EYu+eC7lcKHYnZagMn6SW171la0bGw= +github.com/aws/aws-sdk-go-v2/config v1.18.19/go.mod h1:XvTmGMY8d52ougvakOv1RpiTLPz9dlG/OQHsKU/cMmY= +github.com/aws/aws-sdk-go-v2/credentials v1.13.18 h1:EQMdtHwz0ILTW1hoP+EwuWhwCG1hD6l3+RWFQABET4c= +github.com/aws/aws-sdk-go-v2/credentials v1.13.18/go.mod h1:vnwlwjIe+3XJPBYKu1et30ZPABG3VaXJYr8ryohpIyM= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.1 h1:gt57MN3liKiyGopcqgNzJb2+d9MJaKT/q1OksHNXVE4= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.1/go.mod h1:lfUx8puBRdM5lVVMQlwt2v+ofiG/X6Ms+dy0UkG/kXw= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.31 h1:sJLYcS+eZn5EeNINGHSCRAwUJMFVqklwkH36Vbyai7M= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.31/go.mod h1:QT0BqUvX1Bh2ABdTGnjqEjvjzrCfIniM9Sc8zn9Yndo= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.25 h1:1mnRASEKnkqsntcxHaysxwgVoUUp5dkiB+l3llKnqyg= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.25/go.mod h1:zBHOPwhBc3FlQjQJE/D3IfPWiWaQmT06Vq9aNukDo0k= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.32 h1:p5luUImdIqywn6JpQsW3tq5GNOxKmOnEpybzPx+d1lk= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.32/go.mod h1:XGhIBZDEgfqmFIugclZ6FU7v75nHhBDtzuB4xB/tEi4= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.25 h1:5LHn8JQ0qvjD9L9JhMtylnkcw7j05GDZqM9Oin6hpr0= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.25/go.mod h1:/95IA+0lMnzW6XzqYJRpjjsAbKEORVeO0anQqjd2CNU= +github.com/aws/aws-sdk-go-v2/service/kms v1.20.8 h1:R5f4VOFi3ScTe7TtePyxLqEhNqTJIAxL57MzrXFNs6I= +github.com/aws/aws-sdk-go-v2/service/kms v1.20.8/go.mod h1:OtP3pBOgmJM+acQyQcQXtQHets3yJoVuanCx2T5M7v4= +github.com/aws/aws-sdk-go-v2/service/sso v1.12.6 h1:5V7DWLBd7wTELVz5bPpwzYy/sikk0gsgZfj40X+l5OI= +github.com/aws/aws-sdk-go-v2/service/sso v1.12.6/go.mod h1:Y1VOmit/Fn6Tz1uFAeCO6Q7M2fmfXSCLeL5INVYsLuY= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.6 h1:B8cauxOH1W1v7rd8RdI/MWnoR4Ze0wIHWrb90qczxj4= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.6/go.mod h1:Lh/bc9XUf8CfOY6Jp5aIkQtN+j1mc+nExc+KXj9jx2s= +github.com/aws/aws-sdk-go-v2/service/sts v1.18.7 h1:bWNgNdRko2x6gqa0blfATqAZKZokPIeM1vfmQt2pnvM= +github.com/aws/aws-sdk-go-v2/service/sts v1.18.7/go.mod h1:JuTnSoeePXmMVe9G8NcjjwgOKEfZ4cOjMuT2IBT/2eI= +github.com/aws/smithy-go v1.13.5 h1:hgz0X/DX0dGqTYpGALqXJoRKRj5oQ7150i5FdTePzO8= +github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= +github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= +github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= +github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/bitly/go-simplejson v0.5.0 h1:6IH+V8/tVMab511d5bn4M7EwGXZf9Hj6i2xSwkNEM+Y= +github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngEKAMDJEczWVA= +github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY= +github.com/breeswish/gin-jwt/v2 v2.6.4-jwt-patch h1:KLE/YeX+9FNaGVW5MtImRVPhjDpfpgJhvkuYWBmOYbo= +github.com/breeswish/gin-jwt/v2 v2.6.4-jwt-patch/go.mod h1:KjBLriHXe7L6fGceqWzTod8HUB/TP1WWDtfuSYtYXaI= +github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 h1:BjkPE3785EwPhhyuFkbINB+2a1xATwk8SNDWnJiD41g= +github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5/go.mod h1:jtAfVaU/2cu1+wdSRPWE2c1N2qeAA3K4RH9pYgqwets= +github.com/cenkalti/backoff/v4 v4.0.2 h1:JIufpQLbh4DkbQoii76ItQIUFzevQSqOLZca4eamEDs= +github.com/cenkalti/backoff/v4 v4.0.2/go.mod h1:eEew/i+1Q6OrCDZh3WiXYv3+nJwBASZ8Bog/87DQnVg= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cloudfoundry/gosigar v1.3.6 h1:gIc08FbB3QPb+nAQhINIK/qhf5REKkY0FTGgRGXkcVc= +github.com/cloudfoundry/gosigar v1.3.6/go.mod h1:lNWstu5g5gw59O09Y+wsMNFzBSnU8a0u+Sfx4dq360E= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa h1:OaNxuTZr7kxeODyLWsRMC+OD03aFUH+mW6r2d+MWa5Y= +github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= +github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= +github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= +github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f h1:JOrtw2xFKzlg+cbHpyrpLDmnN1HqhBfnX7WDiW7eG2c= +github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbpBpLoyyu8B6e44T7hJy6potg= +github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/corona10/goimagehash v1.0.2 h1:pUfB0LnsJASMPGEZLj7tGY251vF+qLGqOgEP4rUs6kA= +github.com/corona10/goimagehash v1.0.2/go.mod h1:/l9umBhvcHQXVtQO1V6Gp1yD20STawkhRnnX0D1bvVI= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= +github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= +github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4 h1:qk/FSDDxo05wdJH28W+p5yivv7LuLYLRXPPD8KQCtZs= +github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385 h1:clC1lXBpe2kTj2VHdaIu9ajZQe4kcEY9j0NsnDDBZ3o= +github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM= +github.com/elliotchance/pie/v2 v2.1.0 h1:KEVAAzxYxTyFs4hvebFZVzBdEo3YeMzl2HYDWn+P3F4= +github.com/elliotchance/pie/v2 v2.1.0/go.mod h1:18t0dgGFH006g4eVdDtWfgFZPQEgl10IoEO8YWEq3Og= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fogleman/gg v1.3.0 h1:/7zJX8F6AaYQc57WQCyN9cAIz+4bCJGO9B+dyW29am8= +github.com/fogleman/gg v1.3.0/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/gin-contrib/cors v1.4.0 h1:oJ6gwtUl3lqV0WEIwM/LxPF1QZ5qe2lGWdY2+bz7y0g= +github.com/gin-contrib/cors v1.4.0/go.mod h1:bs9pNM0x/UsmHPBWT2xZz9ROh8xYjYkiURUfmBoMlcs= +github.com/gin-contrib/gzip v0.0.1 h1:ezvKOL6jH+jlzdHNE4h9h8q8uMpDQjyl0NN0Jd7jozc= +github.com/gin-contrib/gzip v0.0.1/go.mod h1:fGBJBCdt6qCZuCAOwWuFhBB4OOq9EFqlo5dEaFhhu5w= +github.com/gin-contrib/pprof v1.4.0 h1:XxiBSf5jWZ5i16lNOPbMTVdgHBdhfGRD5PZ1LWazzvg= +github.com/gin-contrib/pprof v1.4.0/go.mod h1:RrehPJasUVBPK6yTUwOl8/NP6i0vbUgmxtis+Z5KE90= +github.com/gin-contrib/sse v0.0.0-20170109093832-22d885f9ecc7/go.mod h1:VJ0WA2NBN22VlZ2dKZQPAPnyWw5XTlK1KymzLKsr59s= +github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= +github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= +github.com/gin-gonic/gin v1.3.0/go.mod h1:7cKuhb5qV2ggCFctp2fJQ+ErvciLZrIeoOSOm6mUr7Y= +github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M= +github.com/gin-gonic/gin v1.8.1 h1:4+fr/el88TOO3ewCmQr8cx/CtZ/umlIRIs5M4NTNjf8= +github.com/gin-gonic/gin v1.8.1/go.mod h1:ji8BvRH1azfM+SYow9zQ6SZMvR8qOMZHmsCuWR9tTTk= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= +github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= +github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonreference v0.19.6 h1:UBIxjkht+AWIgYzCDSv2GN+E/togfwXUJFRTWhl2Jjs= +github.com/go-openapi/jsonreference v0.19.6/go.mod h1:diGHMEHg2IqXZGKxqyvWdfWU/aim5Dprw5bqpKkTvns= +github.com/go-openapi/spec v0.20.4 h1:O8hJrt0UMnhHcluhIdUgCLRWyM2x7QkBXRvOs7m+O1M= +github.com/go-openapi/spec v0.20.4/go.mod h1:faYFR1CvsJZ0mNsmsphTMSoRrNV3TEDoAM7FOEWeq8I= +github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-openapi/swag v0.19.15 h1:D2NRCBzS9/pEY3gP9Nl8aDqGUcPFrwG2p+CNFrLyrCM= +github.com/go-openapi/swag v0.19.15/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= +github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A= +github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= +github.com/go-playground/locales v0.14.0 h1:u50s323jtVGugKlcYeyzC0etD1HifMjqmJqb8WugfUU= +github.com/go-playground/locales v0.14.0/go.mod h1:sawfccIbzZTqEDETgFXqTho0QybSa7l++s0DH+LDiLs= +github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= +github.com/go-playground/universal-translator v0.18.0 h1:82dyy6p4OuJq4/CByFNOn/jYrnRPArHwAcmLoJZxyho= +github.com/go-playground/universal-translator v0.18.0/go.mod h1:UvRDBj+xPUEGrFYl+lu/H90nyDXpg0fqeB/AQUGNTVA= +github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI= +github.com/go-playground/validator/v10 v10.10.0 h1:I7mrTYv78z8k8VXa/qJlOlEXn/nBh+BF8dHX5nt/dr0= +github.com/go-playground/validator/v10 v10.10.0/go.mod h1:74x4gJWsvQexRdW8Pn3dXSGrTK4nAUsbPlLADvpJkos= +github.com/go-resty/resty/v2 v2.6.0 h1:joIR5PNLM2EFqqESUjCMGXrWmXNHEU9CEiK813oKYS4= +github.com/go-resty/resty/v2 v2.6.0/go.mod h1:PwvJS6hvaPkjtjNg9ph+VrSD92bi5Zq73w/BIH7cC3Q= +github.com/go-sql-driver/mysql v1.7.0 h1:ueSltNNllEqE3qcWBTD0iQd3IpL/6U+mJxLkazJ7YPc= +github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/goccy/go-graphviz v0.0.9 h1:s/FMMJ1Joj6La3S5ApO3Jk2cwM4LpXECC2muFx3IPQQ= +github.com/goccy/go-graphviz v0.0.9/go.mod h1:wXVsXxmyMQU6TN3zGRttjNn3h+iCAS7xQFC6TlNvLhk= +github.com/goccy/go-json v0.9.7 h1:IcB+Aqpx/iMHu5Yooh7jEzJk1JZ7Pjtmys2ukPr7EeM= +github.com/goccy/go-json v0.9.7/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= +github.com/gogo/protobuf v0.0.0-20171007142547-342cbe0a0415/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v0.0.0-20180717141946-636bf0302bc9/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang-jwt/jwt v3.2.1+incompatible h1:73Z+4BJcrTC+KczS6WvTPvRGOp1WmfEP4Q1lOd9Z/+c= +github.com/golang-jwt/jwt v3.2.1+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= +github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 h1:au07oEsX2xN0ktxqI+Sida1w446QrXBRJ0nee3SNZlA= +github.com/golang-sql/sqlexp v0.1.0 h1:ZCD6MBpcuOVfGVqsEmY5/4FtYiKz6tSyUv9LPEDei6A= +github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 h1:DACJavvAHhabrF08vX0COfcOBJRhZ8lUbR+ZWIs0Y5g= +github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/glog v1.1.0 h1:/d3pCKDPWNnvIWe0vVUpNP32qc8U3PDVxySP/y360qE= +github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef h1:veQD95Isof8w9/WXiA+pa3tz3fJXkt5B7QaRBrM62gk= +github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v0.0.0-20180814211427-aa810b61a9c7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= +github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU= +github.com/google/btree v1.1.2/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/pprof v0.0.0-20211122183932-1daafda22083 h1:c8EUapQFi+kjzedr4c6WqbwMdmB95+oDBWZ5XFHFYxY= +github.com/google/pprof v0.0.0-20211122183932-1daafda22083/go.mod h1:KgnwoLYCZ8IQu3XUZ8Nc/bM9CCZFOyjUNOSygVozoDg= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gorilla/mux v1.7.4 h1:VuZ8uybHlWmqV03+zRzdwKL4tUnIp1MAQtp1mIFE1bc= +github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= +github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= +github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4 h1:z53tR0945TRRQO/fLEVPI6SMv7ZflF0TEaTAoU7tOzg= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= +github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= +github.com/gtank/cryptopasta v0.0.0-20170601214702-1f550f6f2f69 h1:7xsUJsB2NrdcttQPa7JLEaGzvdbk7KvfrjgHZXOQRo0= +github.com/gtank/cryptopasta v0.0.0-20170601214702-1f550f6f2f69/go.mod h1:YLEMZOtU+AZ7dhN9T/IpGhXVGly2bvkJQ+zxj3WeVQo= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d h1:uGg2frlt3IcT7kbV6LEp5ONv4vmoO2FW4qSO+my/aoM= +github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w= +github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/jackc/chunkreader/v2 v2.0.1 h1:i+RDz65UE+mmpjTfyz0MoVTnzeYxroil2G82ki7MGG8= +github.com/jackc/pgconn v1.13.0 h1:3L1XMNV2Zvca/8BYhzcRFS70Lr0WlDg16Di6SFGAbys= +github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgproto3/v2 v2.3.1 h1:nwj7qwf0S+Q7ISFfBndqeLwSwxs+4DPsbRFjECT1Y4Y= +github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b h1:C8S2+VttkHFdOOCXJe+YGfa4vHYwlt4Zx+IVXQ97jYg= +github.com/jackc/pgtype v1.12.0 h1:Dlq8Qvcch7kiehm8wPGIW0W3KsCCHJnRacKW0UM8n5w= +github.com/jackc/pgx/v4 v4.17.2 h1:0Ut0rpeKwvIVbMQ1KbMBU4h6wxehBI535LK6Flheh8E= +github.com/jarcoal/httpmock v1.0.8 h1:8kI16SoO6LQKgPE7PvQuV+YuD/inwHd7fOOe2zMbo4k= +github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= +github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= +github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= +github.com/jinzhu/now v1.1.2/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +github.com/jinzhu/now v1.1.4/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= +github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/joho/godotenv v1.4.0 h1:3l4+N6zfMWnkbPEXKng2o2/MR5mSwTrBih4ZEkkz1lg= +github.com/joho/godotenv v1.4.0/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= +github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/jonboulle/clockwork v0.2.2 h1:UOGuzwb1PwsrDAObMuhUnj0p5ULPj8V/xJ7Kx9qUBdQ= +github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= +github.com/joomcode/errorx v1.0.1 h1:CalpDWz14ZHd68fIqluJasJosAewpz2TFaJALrUxjrk= +github.com/joomcode/errorx v1.0.1/go.mod h1:kgco15ekB6cs+4Xjzo7SPeXzx38PbJzBwbnu9qfVNHQ= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA= +github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= +github.com/json-iterator/go v1.1.5/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8= +github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= +github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w= +github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= +github.com/lufia/plan9stats v0.0.0-20230326075908-cb1d2100619a h1:N9zuLhTvBSRt0gWSiJswwQ2HqDmtX/ZCDJURnKUt1Ik= +github.com/lufia/plan9stats v0.0.0-20230326075908-cb1d2100619a/go.mod h1:JKx41uQRwqlTZabZc+kILPrO/3jlKnQ2Z8b7YiVw5cE= +github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA= +github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= +github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= +github.com/mattn/go-sqlite3 v1.14.15 h1:vfoHhTN1af61xCRSWzFIWzx2YskyMTwHLrExkBOjvxI= +github.com/mattn/go-sqlite3 v1.14.15/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= +github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/microsoft/go-mssqldb v0.17.0 h1:Fto83dMZPnYv1Zwx5vHHxpNraeEaUlQ/hhHLgZiaenE= +github.com/minio/sio v0.3.0 h1:syEFBewzOMOYVzSTFpp1MqpSZk8rUNbz8VIIc+PNzus= +github.com/minio/sio v0.3.0/go.mod h1:8b0yPp2avGThviy/+OCJBI6OMpvxoUuiLvE6F1lebhw= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/nfnt/resize v0.0.0-20160724205520-891127d8d1b5 h1:BvoENQQU+fZ9uukda/RzCAL/191HHwJA5b13R6diVlY= +github.com/nfnt/resize v0.0.0-20160724205520-891127d8d1b5/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= +github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= +github.com/oleiade/reflections v1.0.1 h1:D1XO3LVEYroYskEsoSiGItp9RUxG6jWnCVvrqH0HHQM= +github.com/oleiade/reflections v1.0.1/go.mod h1:rdFxbxq4QXVZWj0F+e9jqjDkc7dbp97vkRixKo2JR60= +github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= +github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.20.1 h1:PA/3qinGoukvymdIDV8pii6tiZgC8kbmJO6Z5+b002Q= +github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= +github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= +github.com/otiai10/copy v1.7.0/go.mod h1:rmRl6QPdJj6EiUqXQ/4Nn2lLXoNQjFCQbbNrxgc/t3U= +github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95/go.mod h1:9qAhocn7zKJG+0mI8eUu6xqkFDYS2kb2saOteoSB3cE= +github.com/otiai10/curr v1.0.0/go.mod h1:LskTG5wDwr8Rs+nNQ+1LlxRjAtTZZjtJW4rMXl6j4vs= +github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT91xUo= +github.com/otiai10/mint v1.3.3/go.mod h1:/yxELlJQ0ufhjUwhshSj+wFjZ78CnZ48/1wtmBH1OTc= +github.com/pascaldekloe/name v0.0.0-20180628100202-0fd16699aae1/go.mod h1:eD5JxqMiuNYyFNmyY9rkJ/slN8y59oEu4Ei7F8OoKWQ= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pelletier/go-toml/v2 v2.0.1 h1:8e3L2cCQzLFi2CR4g7vGFuFxX7Jl1kKX8gW+iV0GUKU= +github.com/pelletier/go-toml/v2 v2.0.1/go.mod h1:r9LEWfGN8R5k0VXJ+0BkIe7MYkRdwZOjgMj2KwnJFUo= +github.com/petermattis/goid v0.0.0-20211229010228-4d14c490ee36 h1:64bxqeTEN0/xoEqhKGowgihNuzISS9rEG6YUMU4bzJo= +github.com/petermattis/goid v0.0.0-20211229010228-4d14c490ee36/go.mod h1:pxMtw7cyUw6B2bRH0ZBANSPg+AoSud1I1iyJHI69jH4= +github.com/phf/go-queue v0.0.0-20170504031614-9abe38d0371d h1:U+PMnTlV2tu7RuMK5etusZG3Cf+rpow5hqQByeCzJ2g= +github.com/phf/go-queue v0.0.0-20170504031614-9abe38d0371d/go.mod h1:lXfE4PvvTW5xOjO6Mba8zDPyw8M93B6AQ7frTGnMlA8= +github.com/pingcap/check v0.0.0-20191216031241-8a5a85928f12 h1:rfD9v3+ppLPzoQBgZev0qYCpegrwyFx/BUpkApEiKdY= +github.com/pingcap/errcode v0.3.0 h1:IF6LC/4+b1KNwrMlr2rBTUrojFPMexXBcDWZSpNwxjg= +github.com/pingcap/errcode v0.3.0/go.mod h1:4b2X8xSqxIroj/IZ9MX/VGZhAwc11wB9wRIzHvz6SeM= +github.com/pingcap/errors v0.11.0/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= +github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= +github.com/pingcap/errors v0.11.5-0.20190809092503-95897b64e011/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= +github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c h1:xpW9bvK+HuuTmyFqUwr+jcCvpVkK7sumiz+ko5H9eq4= +github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c/go.mod h1:X2r9ueLEUZgtx2cIogM0v4Zj5uvvzhuuiu7Pn8HzMPg= +github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00 h1:C3N3itkduZXDZFh4N3vQ5HEtld3S+Y+StULhWVvumU0= +github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00/go.mod h1:4qGtCB0QK0wBzKtFEGDhxXnSnbQApw1gc9siScUl8ew= +github.com/pingcap/kvproto v0.0.0-20191211054548-3c6b38ea5107/go.mod h1:WWLmULLO7l8IOcQG+t+ItJ3fEcrL5FxF0Wu+HrMy26w= +github.com/pingcap/kvproto v0.0.0-20230727073445-53e1f8730c30 h1:EvqKcDT7ceGLW0mXqM8Cp5Z8DfgQRnwj2YTnlCLj2QI= +github.com/pingcap/kvproto v0.0.0-20230727073445-53e1f8730c30/go.mod h1:r0q/CFcwvyeRhKtoqzmWMBebrtpIziQQ9vR+JKh1knc= +github.com/pingcap/log v0.0.0-20210625125904-98ed8e2eb1c7/go.mod h1:8AanEdAHATuRurdGxZXBz0At+9avep+ub7U1AGYLIMM= +github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 h1:HR/ylkkLmGdSSDaD8IDP+SZrdhV1Kibl9KrHxJ9eciw= +github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4= +github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21 h1:QV6jqlfOkh8hqvEAgwBZa+4bSgO0EeKC7s5c6Luam2I= +github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21/go.mod h1:QYnjfA95ZaMefyl1NO8oPtKeb8pYUdnDVhQgf+qdpjM= +github.com/pingcap/tidb-dashboard v0.0.0-20230705095454-5e220f970f27 h1:K9lZMYuDuAiR5kOjFESwJ8KfSb4ui5zX6vZGbUp58uk= +github.com/pingcap/tidb-dashboard v0.0.0-20230705095454-5e220f970f27/go.mod h1:OUzFMMVjR1GKlf4LWLqza9QNKjCrYJ7stVn/3PN0djM= +github.com/pingcap/tipb v0.0.0-20220718022156-3e2483c20a9e h1:FBaTXU8C3xgt/drM58VHxojHo/QoG1oPsgWTGvaSpO4= +github.com/pingcap/tipb v0.0.0-20220718022156-3e2483c20a9e/go.mod h1:A7mrd7WHBl1o63LE2bIBGEJMTNWXqhgmYiOvMLxozfs= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= +github.com/power-devops/perfstat v0.0.0-20221212215047-62379fc7944b h1:0LFwY6Q3gMACTjAbMZBjXAqTOzOwFaj2Ld6cjeQ7Rig= +github.com/power-devops/perfstat v0.0.0-20221212215047-62379fc7944b/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= +github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= +github.com/prometheus/client_golang v1.11.1 h1:+4eQaD7vAZ6DsfsxB15hbE0odUjGI5ARs9yskGu1v4s= +github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= +github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= +github.com/prometheus/common v0.26.0 h1:iMAkS2TDoNWnKM+Kopnx/8tnEStIfpYA0ur0xQzzhMQ= +github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4= +github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= +github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUAtL9R8= +github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= +github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik= +github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/samber/lo v1.37.0 h1:XjVcB8g6tgUp8rsPsJ2CvhClfImrpL04YpQHXeHPhRw= +github.com/samber/lo v1.37.0/go.mod h1:9vaz2O4o8oOnK23pd2TrXufcbdbJIa3b6cstBWKpopA= +github.com/sasha-s/go-deadlock v0.2.0 h1:lMqc+fUb7RrFS3gQLtoQsJ7/6TV/pAIFvBsqX73DK8Y= +github.com/sasha-s/go-deadlock v0.2.0/go.mod h1:StQn567HiB1fF2yJ44N9au7wOhrPS3iZqiDbRupzT10= +github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= +github.com/shirou/gopsutil/v3 v3.21.12/go.mod h1:BToYZVTlSVlfazpDDYFnsVZLaoRG+g8ufT6fPQLdJzA= +github.com/shirou/gopsutil/v3 v3.23.3 h1:Syt5vVZXUDXPEXpIBt5ziWsJ4LdSAAxF4l/xZeQgSEE= +github.com/shirou/gopsutil/v3 v3.23.3/go.mod h1:lSBNN6t3+D6W5e5nXTxc8KIMMVxAcS+6IJlffjRRlMU= +github.com/shoenig/go-m1cpu v0.1.4/go.mod h1:Wwvst4LR89UxjeFtLRMrpgRiyY4xPsejnVZym39dbAQ= +github.com/shoenig/go-m1cpu v0.1.5 h1:LF57Z/Fpb/WdGLjt2HZilNnmZOxg/q2bSKTQhgbrLrQ= +github.com/shoenig/go-m1cpu v0.1.5/go.mod h1:Wwvst4LR89UxjeFtLRMrpgRiyY4xPsejnVZym39dbAQ= +github.com/shoenig/test v0.6.3 h1:GVXWJFk9PiOjN0KoJ7VrJGH6uLPnqxR7/fe3HUPfE0c= +github.com/shoenig/test v0.6.3/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k= +github.com/shurcooL/httpgzip v0.0.0-20190720172056-320755c1c1b0 h1:mj/nMDAwTBiaCqMEs4cYCqF7pO6Np7vhy1D1wcQGz+E= +github.com/shurcooL/httpgzip v0.0.0-20190720172056-320755c1c1b0/go.mod h1:919LwcH0M7/W4fcZ0/jy0qGght1GIhqyS/EgWGH2j5Q= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I= +github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= +github.com/smallnest/chanx v0.0.0-20221229104322-eb4c998d2072 h1:Txo4SXVJq/OgEjwgkWoxkMoTjGlcrgsQE/XSghjmu0w= +github.com/smallnest/chanx v0.0.0-20221229104322-eb4c998d2072/go.mod h1:+4nWMF0+CqEcU74SnX2NxaGqZ8zX4pcQ8Jcs77DbX5A= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/soheilhy/cmux v0.1.4 h1:0HKaf1o97UwFjHH9o5XsHUOF+tqmdA7KEzXLpiyaw0E= +github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= +github.com/spf13/cobra v1.0.0 h1:6m/oheQuQ13N9ks4hubMG6BnvwOeaJrqSPLahSnczz8= +github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= +github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/swaggo/files v0.0.0-20210815190702-a29dd2bc99b2 h1:+iNTcqQJy0OZ5jk6a5NLib47eqXK8uYcPX+O4+cBpEM= +github.com/swaggo/files v0.0.0-20210815190702-a29dd2bc99b2/go.mod h1:lKJPbtWzJ9JhsTN1k1gZgleJWY/cqq0psdoMmaThG3w= +github.com/swaggo/http-swagger v1.2.6 h1:ihTjChUoSRMpFMjWw+0AkL1Ti4r6v8pCgVYLmQVRlRw= +github.com/swaggo/http-swagger v1.2.6/go.mod h1:CcoICgY3yVDk2u1LQUCMHbAj0fjlxIX+873psXlIKNA= +github.com/swaggo/swag v1.7.9/go.mod h1:gZ+TJ2w/Ve1RwQsA2IRoSOTidHz6DX+PIG8GWvbnoLU= +github.com/swaggo/swag v1.8.3 h1:3pZSSCQ//gAH88lfmxM3Cd1+JCsxV8Md6f36b9hrZ5s= +github.com/swaggo/swag v1.8.3/go.mod h1:jMLeXOOmYyjk8PvHTsXBdrubsNd9gUJTTCzL5iBnseg= +github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965 h1:1oFLiOyVl+W7bnBzGhf7BbIv9loSFQcieWWYIjLqcAw= +github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965/go.mod h1:9OrXJhf154huy1nPWmuSrkgjPUtUNhA+Zmy+6AESzuA= +github.com/tidwall/gjson v1.6.0/go.mod h1:P256ACg0Mn+j1RXIDXoss50DeIABTYK1PULOJHhxOls= +github.com/tidwall/gjson v1.9.3 h1:hqzS9wAHMO+KVBBkLxYdkEeeFHuqr95GfClRLKlgK0E= +github.com/tidwall/match v1.0.1 h1:PnKP62LPNxHKTwvHHZZzdOAOCtsJTjo6dZLCwpKm5xc= +github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E= +github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4= +github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= +github.com/tklauser/go-sysconf v0.3.9/go.mod h1:11DU/5sG7UexIrp/O6g35hrWzu0JxlwQ3LSFUzyeuhs= +github.com/tklauser/go-sysconf v0.3.11 h1:89WgdJhk5SNwJfu+GKyYveZ4IaJ7xAkecBo+KdJV0CM= +github.com/tklauser/go-sysconf v0.3.11/go.mod h1:GqXfhXY3kiPa0nAXPDIQIWzJbMCB7AmcWpGR8lSZfqI= +github.com/tklauser/numcpus v0.3.0/go.mod h1:yFGUr7TUHQRAhyqBcEg0Ge34zDBAsIvJJcyE6boqnA8= +github.com/tklauser/numcpus v0.6.0 h1:kebhY2Qt+3U6RNK7UqpYNA+tJ23IBEGKkB7JQBfDYms= +github.com/tklauser/numcpus v0.6.0/go.mod h1:FEZLMke0lhOUG6w2JadTzp0a+Nl8PF/GFkQ5UVIcaL4= +github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/tmc/grpc-websocket-proxy v0.0.0-20200427203606-3cfed13b9966 h1:j6JEOq5QWFker+d7mFQYOhjTZonQ7YkLTHm56dbn+yM= +github.com/tmc/grpc-websocket-proxy v0.0.0-20200427203606-3cfed13b9966/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= +github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= +github.com/ugorji/go v1.2.7/go.mod h1:nF9osbDWLy6bDVv/Rtoh6QgnvNDpmCalQV5urGCCS6M= +github.com/ugorji/go/codec v0.0.0-20181022190402-e5e69e061d4f/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= +github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= +github.com/ugorji/go/codec v1.2.7 h1:YPXUKf7fYbp/y8xloBqZOw2qaVggbfwMlI8WM3wZUJ0= +github.com/ugorji/go/codec v1.2.7/go.mod h1:WGN1fab3R1fzQlVQTkfxVtIBhWDRqOviHU95kRgeqEY= +github.com/unrolled/render v1.0.1 h1:VDDnQQVfBMsOsp3VaCJszSO0nkBIVEYoPWeRThk9spY= +github.com/unrolled/render v1.0.1/go.mod h1:gN9T0NhL4Bfbwu8ann7Ry/TGHYfosul+J0obPf6NBdM= +github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= +github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= +github.com/urfave/negroni v0.3.0 h1:PaXOb61mWeZJxc1Ji2xJjpVg9QfPo0rrB+lHyBxGNSU= +github.com/urfave/negroni v0.3.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4= +github.com/vmihailenco/msgpack/v5 v5.3.5 h1:5gO0H1iULLWGhs2H5tbAHIZTV8/cYafcFOr9znI5mJU= +github.com/vmihailenco/msgpack/v5 v5.3.5/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q/V5KbhBonMG9jc= +github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g= +github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg= +github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU= +go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= +go.etcd.io/etcd v0.5.0-alpha.5.0.20220915004622-85b640cee793 h1:fqmtdYQlwZ/vKWSz5amW+a4cnjg23ojz5iL7rjf08Wg= +go.etcd.io/etcd v0.5.0-alpha.5.0.20220915004622-85b640cee793/go.mod h1:eBhtbxXP1qpW0F6+WxoJ64DM1Mrfx46PHtVxEdkLe0I= +go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= +go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= +go.uber.org/dig v1.9.0 h1:pJTDXKEhRqBI8W7rU7kwT5EgyRZuSMVSFcZolOvKK9U= +go.uber.org/dig v1.9.0/go.mod h1:X34SnWGr8Fyla9zQNO2GSO2D+TIuqB14OS8JhYocIyw= +go.uber.org/fx v1.12.0 h1:+1+3Cz9M0dFMPy9SW9XUIUHye8bnPUm7q7DroNGWYG4= +go.uber.org/fx v1.12.0/go.mod h1:egT3Kyg1JFYQkvKLZ3EsykxkNrZxgXS+gKoKo7abERY= +go.uber.org/goleak v0.10.0/go.mod h1:VCZuO8V8mFPlL0F5J5GK1rtHV3DrFcQ1R8ryq7FK0aI= +go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= +go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= +go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= +go.uber.org/multierr v1.4.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= +go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= +go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= +go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= +go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.12.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= +go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= +go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= +go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= +go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200204104054-c9f3fb736b72/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.1.0 h1:MDRAIl0xIo9Io2xV565hzXHw3zVseKrJKodhohM5CjU= +golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20230711005742-c3f37128e5a4 h1:QLureRX3moex6NVu/Lr4MGakp9FdA7sBHGBmvRW7NaM= +golang.org/x/exp v0.0.0-20230711005742-c3f37128e5a4/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= +golang.org/x/image v0.0.0-20200119044424-58c23975cae1/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.5.0 h1:5JMiNunQeQw++mMOz48/ISeNu3Iweh/JaZU8ZLqHRrI= +golang.org/x/image v0.5.0/go.mod h1:FVC7BI/5Ym8R25iw5OLsgshdUBbT1h5jZTpA+mvAdZ4= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5 h1:2M3HP5CCK1Si9FQhwnzYhXdG6DXeebvUHFpre8QvbyI= +golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.11.0 h1:bUO06HqtnRcc/7l71XBe4WcqTZ+3AH1J59zWDDwLKgU= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181005035420-146acd28ed58/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM= +golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.4.0 h1:NF0gk8LVPg1Ml7SSbGyySuoxdsXitj7TvgvuRxIMc/M= +golang.org/x/oauth2 v0.4.0/go.mod h1:RznEsdpjGAINPTOF0UH/t+xJ75L18YO3Ho6Pyn+uRec= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181228144115-9a3f9b0469bb/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210816074244-15123e1e1f71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211013075003-97ac67df715c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.1.0 h1:xYY+Bajn2a7VBmTM5GikTmnK8ZuX8YgnQCqZpbBNtmA= +golang.org/x/time v0.1.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190524210228-3d17549cdc6b/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191030062658-86caa796c7ab/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191114200427-caa0b0f7d508/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20201125231158-b5590deeca9b/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210112230658-8b4aab62c064/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20181004005441-af9cb2a35e7f/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 h1:KpwkzHKEF7B9Zxg18WzOa7djJ+Ha5DzthMyZYQfEn2A= +google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU= +google.golang.org/grpc v0.0.0-20180607172857-7a6a684ca69e/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.26.0 h1:2dTRdpdFEEhJYQD8EMLB61nnrzSCTbG38PhqdhvOltg= +google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= +google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= +gopkg.in/go-playground/validator.v8 v8.18.2/go.mod h1:RX2a/7Ha8BgOhfk7j780h4/u/RRjR0eouCJSH80/M2Y= +gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= +gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= +gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= +gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gorm.io/datatypes v1.1.0 h1:EVp1Z28N4ACpYFK1nHboEIJGIFfjY7vLeieDk8jSHJA= +gorm.io/datatypes v1.1.0/go.mod h1:SH2K9R+2RMjuX1CkCONrPwoe9JzVv2hkQvEu4bXGojE= +gorm.io/driver/mysql v1.4.5 h1:u1lytId4+o9dDaNcPCFzNv7h6wvmc92UjNk3z8enSBU= +gorm.io/driver/mysql v1.4.5/go.mod h1:SxzItlnT1cb6e1e4ZRpgJN2VYtcqJgqnHxWr4wsP8oc= +gorm.io/driver/postgres v1.4.5 h1:mTeXTTtHAgnS9PgmhN2YeUbazYpLhUI1doLnw42XUZc= +gorm.io/driver/sqlite v1.4.3 h1:HBBcZSDnWi5BW3B3rwvVTc510KGkBkexlOg0QrmLUuU= +gorm.io/driver/sqlite v1.4.3/go.mod h1:0Aq3iPO+v9ZKbcdiz8gLWRw5VOPcBOPUQJFLq5e2ecI= +gorm.io/driver/sqlserver v1.4.1 h1:t4r4r6Jam5E6ejqP7N82qAJIJAht27EGT41HyPfXRw0= +gorm.io/gorm v1.21.9/go.mod h1:F+OptMscr0P2F2qU97WT1WimdH9GaQPoDW7AYd5i2Y0= +gorm.io/gorm v1.23.8/go.mod h1:l2lP/RyAtc1ynaTjFksBde/O8v9oOGIApu2/xRitmZk= +gorm.io/gorm v1.24.0/go.mod h1:DVrVomtaYTbqs7gB/x2uVvqnXzv0nqjB396B8cG4dBA= +gorm.io/gorm v1.24.3 h1:WL2ifUmzR/SLp85CSURAfybcHnGZ+yLSGSxgYXlFBHg= +gorm.io/gorm v1.24.3/go.mod h1:DVrVomtaYTbqs7gB/x2uVvqnXzv0nqjB396B8cG4dBA= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +moul.io/zapgorm2 v1.1.0 h1:qwAlMBYf+qJkJ7PAzJl4oCe6eS6QGiKAXUPeis0+RBE= +moul.io/zapgorm2 v1.1.0/go.mod h1:emRfKjNqSzVj5lcgasBdovIXY1jSOwFz2GQZn1Rddks= +sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= +sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= +sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= diff --git a/tests/integrations/mcs/keyspace/tso_keyspace_group_test.go b/tests/integrations/mcs/keyspace/tso_keyspace_group_test.go new file mode 100644 index 00000000000..59aabb260ae --- /dev/null +++ b/tests/integrations/mcs/keyspace/tso_keyspace_group_test.go @@ -0,0 +1,365 @@ +// Copyright 2023 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package keyspace_test + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "testing" + "time" + + "github.com/pingcap/failpoint" + "github.com/stretchr/testify/suite" + bs "github.com/tikv/pd/pkg/basicserver" + "github.com/tikv/pd/pkg/mcs/utils" + "github.com/tikv/pd/pkg/storage/endpoint" + "github.com/tikv/pd/pkg/utils/tempurl" + "github.com/tikv/pd/pkg/utils/testutil" + "github.com/tikv/pd/server/apiv2/handlers" + "github.com/tikv/pd/tests" +) + +const ( + keyspaceGroupsPrefix = "/pd/api/v2/tso/keyspace-groups" +) + +type keyspaceGroupTestSuite struct { + suite.Suite + ctx context.Context + cleanupFunc testutil.CleanupFunc + cluster *tests.TestCluster + server *tests.TestServer + backendEndpoints string + dialClient *http.Client +} + +func TestKeyspaceGroupTestSuite(t *testing.T) { + suite.Run(t, new(keyspaceGroupTestSuite)) +} + +func (suite *keyspaceGroupTestSuite) SetupTest() { + suite.NoError(failpoint.Enable("github.com/tikv/pd/pkg/keyspace/acceleratedAllocNodes", `return(true)`)) + ctx, cancel := context.WithCancel(context.Background()) + suite.ctx = ctx + cluster, err := tests.NewTestAPICluster(suite.ctx, 1) + suite.cluster = cluster + suite.NoError(err) + suite.NoError(cluster.RunInitialServers()) + suite.NotEmpty(cluster.WaitLeader()) + suite.server = cluster.GetServer(cluster.GetLeader()) + suite.NoError(suite.server.BootstrapCluster()) + suite.backendEndpoints = suite.server.GetAddr() + suite.dialClient = &http.Client{ + Transport: &http.Transport{ + DisableKeepAlives: true, + }, + } + suite.cleanupFunc = func() { + cancel() + } +} + +func (suite *keyspaceGroupTestSuite) TearDownTest() { + suite.cleanupFunc() + suite.cluster.Destroy() + suite.NoError(failpoint.Disable("github.com/tikv/pd/pkg/keyspace/acceleratedAllocNodes")) +} + +func (suite *keyspaceGroupTestSuite) TestAllocNodesUpdate() { + // add three nodes. + nodes := make(map[string]bs.Server) + for i := 0; i < utils.DefaultKeyspaceGroupReplicaCount+1; i++ { + s, cleanup := tests.StartSingleTSOTestServer(suite.ctx, suite.Require(), suite.backendEndpoints, tempurl.Alloc()) + defer cleanup() + nodes[s.GetAddr()] = s + } + tests.WaitForPrimaryServing(suite.Require(), nodes) + + // create a keyspace group. + kgs := &handlers.CreateKeyspaceGroupParams{KeyspaceGroups: []*endpoint.KeyspaceGroup{ + { + ID: uint32(1), + UserKind: endpoint.Standard.String(), + }, + }} + code := suite.tryCreateKeyspaceGroup(kgs) + suite.Equal(http.StatusOK, code) + + // alloc nodes for the keyspace group. + id := 1 + params := &handlers.AllocNodesForKeyspaceGroupParams{ + Replica: utils.DefaultKeyspaceGroupReplicaCount, + } + got, code := suite.tryAllocNodesForKeyspaceGroup(id, params) + suite.Equal(http.StatusOK, code) + suite.Equal(utils.DefaultKeyspaceGroupReplicaCount, len(got)) + oldMembers := make(map[string]struct{}) + for _, member := range got { + suite.Contains(nodes, member.Address) + oldMembers[member.Address] = struct{}{} + } + + // alloc node update to 3. + params.Replica = utils.DefaultKeyspaceGroupReplicaCount + 1 + got, code = suite.tryAllocNodesForKeyspaceGroup(id, params) + suite.Equal(http.StatusOK, code) + suite.Equal(params.Replica, len(got)) + newMembers := make(map[string]struct{}) + for _, member := range got { + suite.Contains(nodes, member.Address) + newMembers[member.Address] = struct{}{} + } + for member := range oldMembers { + // old members should be in new members. + suite.Contains(newMembers, member) + } +} + +func (suite *keyspaceGroupTestSuite) TestAllocReplica() { + nodes := make(map[string]bs.Server) + for i := 0; i < utils.DefaultKeyspaceGroupReplicaCount; i++ { + s, cleanup := tests.StartSingleTSOTestServer(suite.ctx, suite.Require(), suite.backendEndpoints, tempurl.Alloc()) + defer cleanup() + nodes[s.GetAddr()] = s + } + tests.WaitForPrimaryServing(suite.Require(), nodes) + + // miss replica. + id := 1 + params := &handlers.AllocNodesForKeyspaceGroupParams{} + got, code := suite.tryAllocNodesForKeyspaceGroup(id, params) + suite.Equal(http.StatusBadRequest, code) + suite.Empty(got) + + // replica is less than default replica. + params = &handlers.AllocNodesForKeyspaceGroupParams{ + Replica: utils.DefaultKeyspaceGroupReplicaCount - 1, + } + _, code = suite.tryAllocNodesForKeyspaceGroup(id, params) + suite.Equal(http.StatusBadRequest, code) + + // there is no any keyspace group. + params = &handlers.AllocNodesForKeyspaceGroupParams{ + Replica: utils.DefaultKeyspaceGroupReplicaCount, + } + _, code = suite.tryAllocNodesForKeyspaceGroup(id, params) + suite.Equal(http.StatusBadRequest, code) + + // create a keyspace group. + kgs := &handlers.CreateKeyspaceGroupParams{KeyspaceGroups: []*endpoint.KeyspaceGroup{ + { + ID: uint32(id), + UserKind: endpoint.Standard.String(), + }, + }} + code = suite.tryCreateKeyspaceGroup(kgs) + suite.Equal(http.StatusOK, code) + params = &handlers.AllocNodesForKeyspaceGroupParams{ + Replica: utils.DefaultKeyspaceGroupReplicaCount, + } + got, code = suite.tryAllocNodesForKeyspaceGroup(id, params) + suite.Equal(http.StatusOK, code) + for _, member := range got { + suite.Contains(nodes, member.Address) + } + + // the keyspace group is exist, but the replica is more than the num of nodes. + params = &handlers.AllocNodesForKeyspaceGroupParams{ + Replica: utils.DefaultKeyspaceGroupReplicaCount + 1, + } + _, code = suite.tryAllocNodesForKeyspaceGroup(id, params) + suite.Equal(http.StatusBadRequest, code) + + // the keyspace group is exist, the new replica is more than the old replica. + s2, cleanup2 := tests.StartSingleTSOTestServer(suite.ctx, suite.Require(), suite.backendEndpoints, tempurl.Alloc()) + defer cleanup2() + nodes[s2.GetAddr()] = s2 + tests.WaitForPrimaryServing(suite.Require(), nodes) + params = &handlers.AllocNodesForKeyspaceGroupParams{ + Replica: utils.DefaultKeyspaceGroupReplicaCount + 1, + } + got, code = suite.tryAllocNodesForKeyspaceGroup(id, params) + suite.Equal(http.StatusOK, code) + for _, member := range got { + suite.Contains(nodes, member.Address) + } + + // the keyspace group is exist, the new replica is equal to the old replica. + params = &handlers.AllocNodesForKeyspaceGroupParams{ + Replica: utils.DefaultKeyspaceGroupReplicaCount + 1, + } + _, code = suite.tryAllocNodesForKeyspaceGroup(id, params) + suite.Equal(http.StatusBadRequest, code) + + // the keyspace group is exist, the new replica is less than the old replica. + params = &handlers.AllocNodesForKeyspaceGroupParams{ + Replica: utils.DefaultKeyspaceGroupReplicaCount, + } + _, code = suite.tryAllocNodesForKeyspaceGroup(id, params) + suite.Equal(http.StatusBadRequest, code) + + // the keyspace group is not exist. + id = 2 + params = &handlers.AllocNodesForKeyspaceGroupParams{ + Replica: utils.DefaultKeyspaceGroupReplicaCount, + } + _, code = suite.tryAllocNodesForKeyspaceGroup(id, params) + suite.Equal(http.StatusBadRequest, code) +} + +func (suite *keyspaceGroupTestSuite) TestSetNodes() { + nodes := make(map[string]bs.Server) + nodesList := []string{} + for i := 0; i < utils.DefaultKeyspaceGroupReplicaCount; i++ { + s, cleanup := tests.StartSingleTSOTestServer(suite.ctx, suite.Require(), suite.backendEndpoints, tempurl.Alloc()) + defer cleanup() + nodes[s.GetAddr()] = s + nodesList = append(nodesList, s.GetAddr()) + } + tests.WaitForPrimaryServing(suite.Require(), nodes) + + // the keyspace group is not exist. + id := 1 + params := &handlers.SetNodesForKeyspaceGroupParams{ + Nodes: nodesList, + } + _, code := suite.trySetNodesForKeyspaceGroup(id, params) + suite.Equal(http.StatusBadRequest, code) + + // the keyspace group is exist. + kgs := &handlers.CreateKeyspaceGroupParams{KeyspaceGroups: []*endpoint.KeyspaceGroup{ + { + ID: uint32(id), + UserKind: endpoint.Standard.String(), + }, + }} + code = suite.tryCreateKeyspaceGroup(kgs) + suite.Equal(http.StatusOK, code) + params = &handlers.SetNodesForKeyspaceGroupParams{ + Nodes: nodesList, + } + kg, code := suite.trySetNodesForKeyspaceGroup(id, params) + suite.Equal(http.StatusOK, code) + suite.Len(kg.Members, 2) + for _, member := range kg.Members { + suite.Contains(nodes, member.Address) + } + + // the keyspace group is exist, but the nodes is not exist. + params = &handlers.SetNodesForKeyspaceGroupParams{ + Nodes: append(nodesList, "pingcap.com:2379"), + } + _, code = suite.trySetNodesForKeyspaceGroup(id, params) + suite.Equal(http.StatusBadRequest, code) + + // the keyspace group is exist, but the count of nodes is less than the default replica. + params = &handlers.SetNodesForKeyspaceGroupParams{ + Nodes: []string{nodesList[0]}, + } + _, code = suite.trySetNodesForKeyspaceGroup(id, params) + suite.Equal(http.StatusBadRequest, code) + + // the keyspace group is not exist. + id = 2 + params = &handlers.SetNodesForKeyspaceGroupParams{ + Nodes: nodesList, + } + _, code = suite.trySetNodesForKeyspaceGroup(id, params) + suite.Equal(http.StatusBadRequest, code) +} + +func (suite *keyspaceGroupTestSuite) TestDefaultKeyspaceGroup() { + nodes := make(map[string]bs.Server) + for i := 0; i < utils.DefaultKeyspaceGroupReplicaCount; i++ { + s, cleanup := tests.StartSingleTSOTestServer(suite.ctx, suite.Require(), suite.backendEndpoints, tempurl.Alloc()) + defer cleanup() + nodes[s.GetAddr()] = s + } + tests.WaitForPrimaryServing(suite.Require(), nodes) + + // the default keyspace group is exist. + var kg *endpoint.KeyspaceGroup + var code int + testutil.Eventually(suite.Require(), func() bool { + kg, code = suite.tryGetKeyspaceGroup(utils.DefaultKeyspaceGroupID) + return code == http.StatusOK && kg != nil + }, testutil.WithWaitFor(time.Second*1)) + suite.Equal(utils.DefaultKeyspaceGroupID, kg.ID) + suite.Len(kg.Members, utils.DefaultKeyspaceGroupReplicaCount) + for _, member := range kg.Members { + suite.Contains(nodes, member.Address) + } +} + +func (suite *keyspaceGroupTestSuite) tryAllocNodesForKeyspaceGroup(id int, request *handlers.AllocNodesForKeyspaceGroupParams) ([]endpoint.KeyspaceGroupMember, int) { + data, err := json.Marshal(request) + suite.NoError(err) + httpReq, err := http.NewRequest(http.MethodPost, suite.server.GetAddr()+keyspaceGroupsPrefix+fmt.Sprintf("/%d/alloc", id), bytes.NewBuffer(data)) + suite.NoError(err) + resp, err := suite.dialClient.Do(httpReq) + suite.NoError(err) + defer resp.Body.Close() + nodes := make([]endpoint.KeyspaceGroupMember, 0) + if resp.StatusCode == http.StatusOK { + bodyBytes, err := io.ReadAll(resp.Body) + suite.NoError(err) + suite.NoError(json.Unmarshal(bodyBytes, &nodes)) + } + return nodes, resp.StatusCode +} + +func (suite *keyspaceGroupTestSuite) tryCreateKeyspaceGroup(request *handlers.CreateKeyspaceGroupParams) int { + data, err := json.Marshal(request) + suite.NoError(err) + httpReq, err := http.NewRequest(http.MethodPost, suite.server.GetAddr()+keyspaceGroupsPrefix, bytes.NewBuffer(data)) + suite.NoError(err) + resp, err := suite.dialClient.Do(httpReq) + suite.NoError(err) + defer resp.Body.Close() + return resp.StatusCode +} + +func (suite *keyspaceGroupTestSuite) tryGetKeyspaceGroup(id uint32) (*endpoint.KeyspaceGroup, int) { + httpReq, err := http.NewRequest(http.MethodGet, suite.server.GetAddr()+keyspaceGroupsPrefix+fmt.Sprintf("/%d", id), nil) + suite.NoError(err) + resp, err := suite.dialClient.Do(httpReq) + suite.NoError(err) + defer resp.Body.Close() + kg := &endpoint.KeyspaceGroup{} + if resp.StatusCode == http.StatusOK { + bodyBytes, err := io.ReadAll(resp.Body) + suite.NoError(err) + suite.NoError(json.Unmarshal(bodyBytes, kg)) + } + return kg, resp.StatusCode +} + +func (suite *keyspaceGroupTestSuite) trySetNodesForKeyspaceGroup(id int, request *handlers.SetNodesForKeyspaceGroupParams) (*endpoint.KeyspaceGroup, int) { + data, err := json.Marshal(request) + suite.NoError(err) + httpReq, err := http.NewRequest(http.MethodPatch, suite.server.GetAddr()+keyspaceGroupsPrefix+fmt.Sprintf("/%d", id), bytes.NewBuffer(data)) + suite.NoError(err) + resp, err := suite.dialClient.Do(httpReq) + suite.NoError(err) + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil, resp.StatusCode + } + return suite.tryGetKeyspaceGroup(uint32(id)) +} diff --git a/tests/integrations/mcs/resourcemanager/resource_manager_test.go b/tests/integrations/mcs/resourcemanager/resource_manager_test.go new file mode 100644 index 00000000000..6c3a0a39146 --- /dev/null +++ b/tests/integrations/mcs/resourcemanager/resource_manager_test.go @@ -0,0 +1,1254 @@ +// Copyright 2022 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package resourcemanager_test + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "strconv" + "strings" + "testing" + "time" + + "github.com/pingcap/failpoint" + rmpb "github.com/pingcap/kvproto/pkg/resource_manager" + "github.com/stretchr/testify/suite" + pd "github.com/tikv/pd/client" + "github.com/tikv/pd/client/resource_group/controller" + "github.com/tikv/pd/pkg/mcs/resourcemanager/server" + "github.com/tikv/pd/pkg/utils/testutil" + "github.com/tikv/pd/tests" + "go.uber.org/goleak" + + // Register Service + _ "github.com/tikv/pd/pkg/mcs/registry" + _ "github.com/tikv/pd/pkg/mcs/resourcemanager/server/install" +) + +func TestMain(m *testing.M) { + goleak.VerifyTestMain(m, testutil.LeakOptions...) +} + +type resourceManagerClientTestSuite struct { + suite.Suite + ctx context.Context + clean context.CancelFunc + cluster *tests.TestCluster + client pd.Client + initGroups []*rmpb.ResourceGroup +} + +func TestResourceManagerClientTestSuite(t *testing.T) { + suite.Run(t, new(resourceManagerClientTestSuite)) +} + +func (suite *resourceManagerClientTestSuite) SetupSuite() { + var err error + re := suite.Require() + + re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/mcs/resourcemanager/server/enableDegradedMode", `return(true)`)) + + suite.ctx, suite.clean = context.WithCancel(context.Background()) + + suite.cluster, err = tests.NewTestCluster(suite.ctx, 2) + re.NoError(err) + + err = suite.cluster.RunInitialServers() + re.NoError(err) + + suite.client, err = pd.NewClientWithContext(suite.ctx, suite.cluster.GetConfig().GetClientURLs(), pd.SecurityOption{}) + re.NoError(err) + leader := suite.cluster.GetServer(suite.cluster.WaitLeader()) + suite.waitLeader(suite.client, leader.GetAddr()) + + suite.initGroups = []*rmpb.ResourceGroup{ + { + Name: "test1", + Mode: rmpb.GroupMode_RUMode, + RUSettings: &rmpb.GroupRequestUnitSettings{ + RU: &rmpb.TokenBucket{ + Settings: &rmpb.TokenLimitSettings{ + FillRate: 10000, + }, + Tokens: 100000, + }, + }, + }, + { + Name: "test2", + Mode: rmpb.GroupMode_RUMode, + RUSettings: &rmpb.GroupRequestUnitSettings{ + RU: &rmpb.TokenBucket{ + Settings: &rmpb.TokenLimitSettings{ + FillRate: 20000, + BurstLimit: -1, + }, + Tokens: 100000, + }, + }, + }, + { + Name: "test3", + Mode: rmpb.GroupMode_RUMode, + RUSettings: &rmpb.GroupRequestUnitSettings{ + RU: &rmpb.TokenBucket{ + Settings: &rmpb.TokenLimitSettings{ + FillRate: 100, + BurstLimit: 5000000, + }, + Tokens: 5000000, + }, + }, + }, + { + Name: "test4", + Mode: rmpb.GroupMode_RUMode, + RUSettings: &rmpb.GroupRequestUnitSettings{ + RU: &rmpb.TokenBucket{ + Settings: &rmpb.TokenLimitSettings{ + FillRate: 1000, + BurstLimit: 5000000, + }, + Tokens: 5000000, + }, + }, + }, + { + Name: "background_job", + Mode: rmpb.GroupMode_RUMode, + RUSettings: &rmpb.GroupRequestUnitSettings{ + RU: &rmpb.TokenBucket{ + Settings: &rmpb.TokenLimitSettings{ + FillRate: 10000, + BurstLimit: -1, + }, + Tokens: 100000, + }, + }, + BackgroundSettings: &rmpb.BackgroundSettings{JobTypes: []string{"br", "lightning"}}, + }, + } +} + +func (suite *resourceManagerClientTestSuite) waitLeader(cli pd.Client, leaderAddr string) { + innerCli, ok := cli.(interface{ GetServiceDiscovery() pd.ServiceDiscovery }) + suite.True(ok) + suite.NotNil(innerCli) + testutil.Eventually(suite.Require(), func() bool { + innerCli.GetServiceDiscovery().ScheduleCheckMemberChanged() + return innerCli.GetServiceDiscovery().GetServingAddr() == leaderAddr + }) +} + +func (suite *resourceManagerClientTestSuite) TearDownSuite() { + re := suite.Require() + suite.client.Close() + suite.cluster.Destroy() + suite.clean() + re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/mcs/resourcemanager/server/enableDegradedMode")) +} + +func (suite *resourceManagerClientTestSuite) TearDownTest() { + suite.cleanupResourceGroups() +} + +func (suite *resourceManagerClientTestSuite) cleanupResourceGroups() { + cli := suite.client + groups, err := cli.ListResourceGroups(suite.ctx) + suite.NoError(err) + for _, group := range groups { + deleteResp, err := cli.DeleteResourceGroup(suite.ctx, group.GetName()) + if group.Name == "default" { + suite.Contains(err.Error(), "cannot delete reserved group") + continue + } + suite.NoError(err) + suite.Contains(deleteResp, "Success!") + } +} + +func (suite *resourceManagerClientTestSuite) resignAndWaitLeader() { + suite.NoError(suite.cluster.ResignLeader()) + newLeader := suite.cluster.GetServer(suite.cluster.WaitLeader()) + suite.NotNil(newLeader) + suite.waitLeader(suite.client, newLeader.GetAddr()) +} + +func (suite *resourceManagerClientTestSuite) TestWatchResourceGroup() { + re := suite.Require() + cli := suite.client + group := &rmpb.ResourceGroup{ + Name: "test", + Mode: rmpb.GroupMode_RUMode, + RUSettings: &rmpb.GroupRequestUnitSettings{ + RU: &rmpb.TokenBucket{ + Settings: &rmpb.TokenLimitSettings{ + FillRate: 10000, + }, + Tokens: 100000, + }, + }, + } + // Mock get revision by listing + for i := 0; i < 3; i++ { + group.Name += strconv.Itoa(i) + resp, err := cli.AddResourceGroup(suite.ctx, group) + group.Name = "test" + re.NoError(err) + re.Contains(resp, "Success!") + } + lresp, revision, err := cli.LoadResourceGroups(suite.ctx) + re.NoError(err) + re.Equal(len(lresp), 4) + re.Greater(revision, int64(0)) + tcs := tokenConsumptionPerSecond{rruTokensAtATime: 100} + re.NoError(failpoint.Enable("github.com/tikv/pd/client/resource_group/controller/disableWatch", "return(true)")) + defer func() { + re.NoError(failpoint.Disable("github.com/tikv/pd/client/resource_group/controller/disableWatch")) + }() + controllerKeySpace, _ := controller.NewResourceGroupController(suite.ctx, 1, cli, nil, controller.EnableSingleGroupByKeyspace()) + controller, _ := controller.NewResourceGroupController(suite.ctx, 1, cli, nil) + controller.Start(suite.ctx) + defer controller.Stop() + controller.OnRequestWait(suite.ctx, "test0", tcs.makeReadRequest()) + meta := controller.GetActiveResourceGroup("test0") + metaShadow, err := controller.GetResourceGroup("test0") + re.NoError(err) + re.Equal(meta.RUSettings.RU, group.RUSettings.RU) + re.Equal(metaShadow.RUSettings.RU, group.RUSettings.RU) + + controllerKeySpace.OnRequestWait(suite.ctx, "test0", tcs.makeReadRequest()) + metaKeySpace := controllerKeySpace.GetActiveResourceGroup("test0") + re.Equal(metaKeySpace.RUSettings.RU, group.RUSettings.RU) + + controller.OnRequestWait(suite.ctx, "test1", tcs.makeReadRequest()) + meta = controller.GetActiveResourceGroup("test1") + metaShadow, err = controller.GetResourceGroup("test1") + re.NoError(err) + re.Equal(meta.RUSettings.RU, group.RUSettings.RU) + re.Equal(metaShadow.RUSettings.RU, group.RUSettings.RU) + suite.NoError(err) + // Mock add resource groups + for i := 3; i < 9; i++ { + group.Name = "test" + strconv.Itoa(i) + resp, err := cli.AddResourceGroup(suite.ctx, group) + re.NoError(err) + re.Contains(resp, "Success!") + } + // Mock modify resource groups + modifySettings := func(gs *rmpb.ResourceGroup) { + gs.RUSettings = &rmpb.GroupRequestUnitSettings{ + RU: &rmpb.TokenBucket{ + Settings: &rmpb.TokenLimitSettings{ + FillRate: 20000, + }, + }, + } + } + re.NoError(failpoint.Enable("github.com/tikv/pd/client/resource_group/controller/watchStreamError", "return(true)")) + for i := 0; i < 2; i++ { + if i == 1 { + testutil.Eventually(re, func() bool { + meta = controller.GetActiveResourceGroup("test0") + return meta.RUSettings.RU.Settings.FillRate == uint64(20000) + }, testutil.WithTickInterval(50*time.Millisecond)) + metaKeySpace = controllerKeySpace.GetActiveResourceGroup("test0") + re.Equal(metaKeySpace.RUSettings.RU.Settings.FillRate, uint64(10000)) + re.NoError(failpoint.Enable("github.com/tikv/pd/client/watchStreamError", "return(true)")) + } + group.Name = "test" + strconv.Itoa(i) + modifySettings(group) + resp, err := cli.ModifyResourceGroup(suite.ctx, group) + re.NoError(err) + re.Contains(resp, "Success!") + } + time.Sleep(time.Millisecond * 50) + meta = controller.GetActiveResourceGroup("test1") + re.Equal(meta.RUSettings.RU.Settings.FillRate, uint64(10000)) + re.NoError(failpoint.Disable("github.com/tikv/pd/client/watchStreamError")) + testutil.Eventually(re, func() bool { + meta = controller.GetActiveResourceGroup("test1") + return meta.RUSettings.RU.Settings.FillRate == uint64(20000) + }, testutil.WithTickInterval(100*time.Millisecond)) + re.NoError(failpoint.Disable("github.com/tikv/pd/client/resource_group/controller/watchStreamError")) + + for i := 2; i < 9; i++ { + group.Name = "test" + strconv.Itoa(i) + modifySettings(group) + resp, err := cli.ModifyResourceGroup(suite.ctx, group) + re.NoError(err) + re.Contains(resp, "Success!") + } + // Mock delete resource groups + suite.cleanupResourceGroups() + time.Sleep(time.Second) + meta = controller.GetActiveResourceGroup(group.Name) + re.Nil(meta) + + // Check watch result + watchChan, err := suite.client.WatchResourceGroup(suite.ctx, revision) + re.NoError(err) + i := 0 + for { + select { + case <-time.After(time.Second): + return + case res := <-watchChan: + for _, r := range res { + if i < 6 { + suite.Equal(uint64(10000), r.RUSettings.RU.Settings.FillRate) + } else { + suite.Equal(uint64(20000), r.RUSettings.RU.Settings.FillRate) + } + i++ + } + } + } +} + +const buffDuration = time.Millisecond * 300 + +type tokenConsumptionPerSecond struct { + rruTokensAtATime float64 + wruTokensAtATime float64 + times int + waitDuration time.Duration +} + +func (t tokenConsumptionPerSecond) makeReadRequest() *controller.TestRequestInfo { + return controller.NewTestRequestInfo(false, 0, 0) +} + +func (t tokenConsumptionPerSecond) makeWriteRequest() *controller.TestRequestInfo { + return controller.NewTestRequestInfo(true, uint64(t.wruTokensAtATime-1), 0) +} + +func (t tokenConsumptionPerSecond) makeReadResponse() *controller.TestResponseInfo { + return controller.NewTestResponseInfo( + uint64((t.rruTokensAtATime-1)/2), + time.Duration(t.rruTokensAtATime/2)*time.Millisecond, + false, + ) +} + +func (t tokenConsumptionPerSecond) makeWriteResponse() *controller.TestResponseInfo { + return controller.NewTestResponseInfo( + 0, + time.Duration(0), + true, + ) +} + +func (suite *resourceManagerClientTestSuite) TestResourceGroupController() { + re := suite.Require() + cli := suite.client + + for _, group := range suite.initGroups { + resp, err := cli.AddResourceGroup(suite.ctx, group) + re.NoError(err) + re.Contains(resp, "Success!") + } + + cfg := &controller.RequestUnitConfig{ + ReadBaseCost: 1, + ReadCostPerByte: 1, + WriteBaseCost: 1, + WriteCostPerByte: 1, + CPUMsCost: 1, + } + + controller, _ := controller.NewResourceGroupController(suite.ctx, 1, cli, cfg, controller.EnableSingleGroupByKeyspace()) + controller.Start(suite.ctx) + + testCases := []struct { + resourceGroupName string + tcs []tokenConsumptionPerSecond + len int + }{ + { + resourceGroupName: suite.initGroups[0].Name, + len: 8, + tcs: []tokenConsumptionPerSecond{ + {rruTokensAtATime: 50, wruTokensAtATime: 20, times: 100, waitDuration: 0}, + {rruTokensAtATime: 50, wruTokensAtATime: 100, times: 100, waitDuration: 0}, + {rruTokensAtATime: 50, wruTokensAtATime: 100, times: 100, waitDuration: 0}, + {rruTokensAtATime: 20, wruTokensAtATime: 40, times: 250, waitDuration: 0}, + {rruTokensAtATime: 25, wruTokensAtATime: 50, times: 200, waitDuration: 0}, + {rruTokensAtATime: 30, wruTokensAtATime: 60, times: 165, waitDuration: 0}, + {rruTokensAtATime: 40, wruTokensAtATime: 80, times: 125, waitDuration: 0}, + {rruTokensAtATime: 50, wruTokensAtATime: 100, times: 100, waitDuration: 0}, + }, + }, + } + tricker := time.NewTicker(time.Second) + defer tricker.Stop() + i := 0 + for { + v := false + <-tricker.C + for _, cas := range testCases { + if i >= cas.len { + continue + } + v = true + sum := time.Duration(0) + for j := 0; j < cas.tcs[i].times; j++ { + rreq := cas.tcs[i].makeReadRequest() + wreq := cas.tcs[i].makeWriteRequest() + rres := cas.tcs[i].makeReadResponse() + wres := cas.tcs[i].makeWriteResponse() + startTime := time.Now() + _, _, err := controller.OnRequestWait(suite.ctx, cas.resourceGroupName, rreq) + re.NoError(err) + _, _, err = controller.OnRequestWait(suite.ctx, cas.resourceGroupName, wreq) + re.NoError(err) + sum += time.Since(startTime) + controller.OnResponse(cas.resourceGroupName, rreq, rres) + controller.OnResponse(cas.resourceGroupName, wreq, wres) + time.Sleep(1000 * time.Microsecond) + } + re.LessOrEqual(sum, buffDuration+cas.tcs[i].waitDuration) + } + i++ + if !v { + break + } + } + controller.Stop() +} + +// TestSwitchBurst is used to test https://github.com/tikv/pd/issues/6209 +func (suite *resourceManagerClientTestSuite) TestSwitchBurst() { + re := suite.Require() + cli := suite.client + re.NoError(failpoint.Enable("github.com/tikv/pd/client/resource_group/controller/acceleratedReportingPeriod", "return(true)")) + + for _, group := range suite.initGroups { + resp, err := cli.AddResourceGroup(suite.ctx, group) + re.NoError(err) + re.Contains(resp, "Success!") + } + + cfg := &controller.RequestUnitConfig{ + ReadBaseCost: 1, + ReadCostPerByte: 1, + WriteBaseCost: 1, + WriteCostPerByte: 1, + CPUMsCost: 1, + } + testCases := []struct { + resourceGroupName string + tcs []tokenConsumptionPerSecond + len int + }{ + { + resourceGroupName: suite.initGroups[0].Name, + len: 8, + tcs: []tokenConsumptionPerSecond{ + {rruTokensAtATime: 50, wruTokensAtATime: 20, times: 100, waitDuration: 0}, + {rruTokensAtATime: 50, wruTokensAtATime: 100, times: 100, waitDuration: 0}, + {rruTokensAtATime: 50, wruTokensAtATime: 100, times: 100, waitDuration: 0}, + {rruTokensAtATime: 20, wruTokensAtATime: 40, times: 250, waitDuration: 0}, + {rruTokensAtATime: 25, wruTokensAtATime: 50, times: 200, waitDuration: 0}, + {rruTokensAtATime: 30, wruTokensAtATime: 60, times: 165, waitDuration: 0}, + {rruTokensAtATime: 40, wruTokensAtATime: 80, times: 125, waitDuration: 0}, + {rruTokensAtATime: 50, wruTokensAtATime: 100, times: 100, waitDuration: 0}, + }, + }, + } + controller, _ := controller.NewResourceGroupController(suite.ctx, 1, cli, cfg, controller.EnableSingleGroupByKeyspace()) + controller.Start(suite.ctx) + resourceGroupName := suite.initGroups[1].Name + tcs := tokenConsumptionPerSecond{rruTokensAtATime: 1, wruTokensAtATime: 2, times: 100, waitDuration: 0} + for j := 0; j < tcs.times; j++ { + rreq := tcs.makeReadRequest() + wreq := tcs.makeWriteRequest() + rres := tcs.makeReadResponse() + wres := tcs.makeWriteResponse() + _, _, err := controller.OnRequestWait(suite.ctx, resourceGroupName, rreq) + re.NoError(err) + _, _, err = controller.OnRequestWait(suite.ctx, resourceGroupName, wreq) + re.NoError(err) + controller.OnResponse(resourceGroupName, rreq, rres) + controller.OnResponse(resourceGroupName, wreq, wres) + } + time.Sleep(2 * time.Second) + cli.ModifyResourceGroup(suite.ctx, &rmpb.ResourceGroup{ + Name: "test2", + Mode: rmpb.GroupMode_RUMode, + RUSettings: &rmpb.GroupRequestUnitSettings{ + RU: &rmpb.TokenBucket{ + Settings: &rmpb.TokenLimitSettings{ + FillRate: 20000, + BurstLimit: 20000, + }, + }, + }, + }) + time.Sleep(100 * time.Millisecond) + tricker := time.NewTicker(time.Second) + defer tricker.Stop() + i := 0 + for { + v := false + <-tricker.C + for _, cas := range testCases { + if i >= cas.len { + continue + } + v = true + sum := time.Duration(0) + for j := 0; j < cas.tcs[i].times; j++ { + rreq := cas.tcs[i].makeReadRequest() + wreq := cas.tcs[i].makeWriteRequest() + rres := cas.tcs[i].makeReadResponse() + wres := cas.tcs[i].makeWriteResponse() + startTime := time.Now() + _, _, err := controller.OnRequestWait(suite.ctx, resourceGroupName, rreq) + re.NoError(err) + _, _, err = controller.OnRequestWait(suite.ctx, resourceGroupName, wreq) + re.NoError(err) + sum += time.Since(startTime) + controller.OnResponse(resourceGroupName, rreq, rres) + controller.OnResponse(resourceGroupName, wreq, wres) + time.Sleep(1000 * time.Microsecond) + } + re.LessOrEqual(sum, buffDuration+cas.tcs[i].waitDuration) + } + i++ + if !v { + break + } + } + + resourceGroupName2 := suite.initGroups[2].Name + tcs = tokenConsumptionPerSecond{rruTokensAtATime: 1, wruTokensAtATime: 100000, times: 1, waitDuration: 0} + wreq := tcs.makeWriteRequest() + _, _, err := controller.OnRequestWait(suite.ctx, resourceGroupName2, wreq) + re.NoError(err) + + re.NoError(failpoint.Enable("github.com/tikv/pd/client/resource_group/controller/acceleratedSpeedTrend", "return(true)")) + resourceGroupName3 := suite.initGroups[3].Name + tcs = tokenConsumptionPerSecond{rruTokensAtATime: 1, wruTokensAtATime: 1000, times: 1, waitDuration: 0} + wreq = tcs.makeWriteRequest() + _, _, err = controller.OnRequestWait(suite.ctx, resourceGroupName3, wreq) + re.NoError(err) + time.Sleep(110 * time.Millisecond) + tcs = tokenConsumptionPerSecond{rruTokensAtATime: 1, wruTokensAtATime: 10, times: 1010, waitDuration: 0} + duration := time.Duration(0) + for i := 0; i < tcs.times; i++ { + wreq = tcs.makeWriteRequest() + startTime := time.Now() + _, _, err = controller.OnRequestWait(suite.ctx, resourceGroupName3, wreq) + duration += time.Since(startTime) + re.NoError(err) + } + re.Less(duration, 100*time.Millisecond) + re.NoError(failpoint.Disable("github.com/tikv/pd/client/resource_group/controller/acceleratedReportingPeriod")) + re.NoError(failpoint.Disable("github.com/tikv/pd/client/resource_group/controller/acceleratedSpeedTrend")) + controller.Stop() +} + +func (suite *resourceManagerClientTestSuite) TestResourcePenalty() { + re := suite.Require() + cli := suite.client + + for _, group := range suite.initGroups { + resp, err := cli.AddResourceGroup(suite.ctx, group) + re.NoError(err) + re.Contains(resp, "Success!") + } + + cfg := &controller.RequestUnitConfig{ + ReadBaseCost: 1, + ReadCostPerByte: 1, + WriteBaseCost: 1, + WriteCostPerByte: 1, + CPUMsCost: 1, + } + c, _ := controller.NewResourceGroupController(suite.ctx, 1, cli, cfg, controller.EnableSingleGroupByKeyspace()) + c.Start(suite.ctx) + + resourceGroupName := suite.initGroups[1].Name + // init + req := controller.NewTestRequestInfo(false, 0, 2 /* store2 */) + resp := controller.NewTestResponseInfo(0, time.Duration(30), true) + _, penalty, err := c.OnRequestWait(suite.ctx, resourceGroupName, req) + re.NoError(err) + re.Equal(penalty.WriteBytes, 0.0) + re.Equal(penalty.TotalCpuTimeMs, 0.0) + _, err = c.OnResponse(resourceGroupName, req, resp) + re.NoError(err) + + req = controller.NewTestRequestInfo(true, 60, 1 /* store1 */) + resp = controller.NewTestResponseInfo(0, time.Duration(10), true) + _, penalty, err = c.OnRequestWait(suite.ctx, resourceGroupName, req) + re.NoError(err) + re.Equal(penalty.WriteBytes, 0.0) + re.Equal(penalty.TotalCpuTimeMs, 0.0) + _, err = c.OnResponse(resourceGroupName, req, resp) + re.NoError(err) + + // failed request, shouldn't be counted in penalty + req = controller.NewTestRequestInfo(true, 20, 1 /* store1 */) + resp = controller.NewTestResponseInfo(0, time.Duration(0), false) + _, penalty, err = c.OnRequestWait(suite.ctx, resourceGroupName, req) + re.NoError(err) + re.Equal(penalty.WriteBytes, 0.0) + re.Equal(penalty.TotalCpuTimeMs, 0.0) + _, err = c.OnResponse(resourceGroupName, req, resp) + re.NoError(err) + + // from same store, should be zero + req1 := controller.NewTestRequestInfo(false, 0, 1 /* store1 */) + resp1 := controller.NewTestResponseInfo(0, time.Duration(10), true) + _, penalty, err = c.OnRequestWait(suite.ctx, resourceGroupName, req1) + re.NoError(err) + re.Equal(penalty.WriteBytes, 0.0) + _, err = c.OnResponse(resourceGroupName, req1, resp1) + re.NoError(err) + + // from different store, should be non-zero + req2 := controller.NewTestRequestInfo(true, 50, 2 /* store2 */) + resp2 := controller.NewTestResponseInfo(0, time.Duration(10), true) + _, penalty, err = c.OnRequestWait(suite.ctx, resourceGroupName, req2) + re.NoError(err) + re.Equal(penalty.WriteBytes, 60.0) + re.InEpsilon(penalty.TotalCpuTimeMs, 10.0/1000.0/1000.0, 1e-6) + _, err = c.OnResponse(resourceGroupName, req2, resp2) + re.NoError(err) + + // from new store, should be zero + req3 := controller.NewTestRequestInfo(true, 0, 3 /* store3 */) + resp3 := controller.NewTestResponseInfo(0, time.Duration(10), true) + _, penalty, err = c.OnRequestWait(suite.ctx, resourceGroupName, req3) + re.NoError(err) + re.Equal(penalty.WriteBytes, 0.0) + _, err = c.OnResponse(resourceGroupName, req3, resp3) + re.NoError(err) + + // from different group, should be zero + resourceGroupName = suite.initGroups[2].Name + req4 := controller.NewTestRequestInfo(true, 50, 1 /* store2 */) + resp4 := controller.NewTestResponseInfo(0, time.Duration(10), true) + _, penalty, err = c.OnRequestWait(suite.ctx, resourceGroupName, req4) + re.NoError(err) + re.Equal(penalty.WriteBytes, 0.0) + _, err = c.OnResponse(resourceGroupName, req4, resp4) + re.NoError(err) + + c.Stop() +} + +func (suite *resourceManagerClientTestSuite) TestAcquireTokenBucket() { + re := suite.Require() + cli := suite.client + + groups := make([]*rmpb.ResourceGroup, 0) + groups = append(groups, suite.initGroups...) + for _, group := range groups { + resp, err := cli.AddResourceGroup(suite.ctx, group) + re.NoError(err) + re.Contains(resp, "Success!") + } + reqs := &rmpb.TokenBucketsRequest{ + Requests: make([]*rmpb.TokenBucketRequest, 0), + TargetRequestPeriodMs: uint64(time.Second * 10 / time.Millisecond), + } + re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/mcs/resourcemanager/server/fastPersist", `return(true)`)) + suite.resignAndWaitLeader() + groups = append(groups, &rmpb.ResourceGroup{Name: "test3"}) + for i := 0; i < 3; i++ { + for _, group := range groups { + requests := make([]*rmpb.RequestUnitItem, 0) + requests = append(requests, &rmpb.RequestUnitItem{ + Type: rmpb.RequestUnitType_RU, + Value: 30000, + }) + req := &rmpb.TokenBucketRequest{ + ResourceGroupName: group.Name, + Request: &rmpb.TokenBucketRequest_RuItems{ + RuItems: &rmpb.TokenBucketRequest_RequestRU{ + RequestRU: requests, + }, + }, + } + reqs.Requests = append(reqs.Requests, req) + } + aresp, err := cli.AcquireTokenBuckets(suite.ctx, reqs) + re.NoError(err) + for _, resp := range aresp { + re.Len(resp.GrantedRUTokens, 1) + re.Equal(resp.GrantedRUTokens[0].GrantedTokens.Tokens, 30000.) + if resp.ResourceGroupName == "test2" { + re.Equal(int64(-1), resp.GrantedRUTokens[0].GrantedTokens.GetSettings().GetBurstLimit()) + } + } + gresp, err := cli.GetResourceGroup(suite.ctx, groups[0].GetName()) + re.NoError(err) + re.Less(gresp.RUSettings.RU.Tokens, groups[0].RUSettings.RU.Tokens) + + checkFunc := func(g1 *rmpb.ResourceGroup, g2 *rmpb.ResourceGroup) { + re.Equal(g1.GetName(), g2.GetName()) + re.Equal(g1.GetMode(), g2.GetMode()) + re.Equal(g1.GetRUSettings().RU.Settings.FillRate, g2.GetRUSettings().RU.Settings.FillRate) + // now we don't persistent tokens in running state, so tokens is original. + re.Less(g1.GetRUSettings().RU.Tokens, g2.GetRUSettings().RU.Tokens) + re.NoError(err) + } + time.Sleep(250 * time.Millisecond) + // to test persistent + suite.resignAndWaitLeader() + gresp, err = cli.GetResourceGroup(suite.ctx, groups[0].GetName()) + re.NoError(err) + checkFunc(gresp, groups[0]) + } + + // Test for background request upload. + reqs.Requests = nil + reqs.Requests = append(reqs.Requests, &rmpb.TokenBucketRequest{ + ResourceGroupName: "background_job", + IsBackground: true, + }) + aresp, err := cli.AcquireTokenBuckets(suite.ctx, reqs) + re.NoError(err) + for _, resp := range aresp { + re.Len(resp.GrantedRUTokens, 0) + } + + re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/mcs/resourcemanager/server/fastPersist")) +} + +func (suite *resourceManagerClientTestSuite) TestBasicResourceGroupCURD() { + re := suite.Require() + cli := suite.client + testCasesSet1 := []struct { + name string + mode rmpb.GroupMode + isNewGroup bool + modifySuccess bool + expectMarshal string + modifySettings func(*rmpb.ResourceGroup) + }{ + {"test1", rmpb.GroupMode_RUMode, true, true, + `{"name":"test1","mode":1,"r_u_settings":{"r_u":{"settings":{"fill_rate":10000},"state":{"initialized":false}}},"priority":0}`, + func(gs *rmpb.ResourceGroup) { + gs.RUSettings = &rmpb.GroupRequestUnitSettings{ + RU: &rmpb.TokenBucket{ + Settings: &rmpb.TokenLimitSettings{ + FillRate: 10000, + }, + }, + } + }, + }, + + {"test2", rmpb.GroupMode_RUMode, true, true, + `{"name":"test2","mode":1,"r_u_settings":{"r_u":{"settings":{"fill_rate":20000},"state":{"initialized":false}}},"priority":0,"runaway_settings":{"rule":{"exec_elapsed_time_ms":10000},"action":1},"background_settings":{"job_types":["test"]}}`, + func(gs *rmpb.ResourceGroup) { + gs.RUSettings = &rmpb.GroupRequestUnitSettings{ + RU: &rmpb.TokenBucket{ + Settings: &rmpb.TokenLimitSettings{ + FillRate: 20000, + }, + }, + } + gs.RunawaySettings = &rmpb.RunawaySettings{ + Rule: &rmpb.RunawayRule{ + ExecElapsedTimeMs: 10000, + }, + Action: rmpb.RunawayAction_CoolDown, + } + gs.BackgroundSettings = &rmpb.BackgroundSettings{ + JobTypes: []string{"test"}, + } + }, + }, + {"test2", rmpb.GroupMode_RUMode, false, true, + `{"name":"test2","mode":1,"r_u_settings":{"r_u":{"settings":{"fill_rate":30000,"burst_limit":-1},"state":{"initialized":false}}},"priority":0,"runaway_settings":{"rule":{"exec_elapsed_time_ms":1000},"action":2,"watch":{"lasting_duration_ms":100000,"type":2}},"background_settings":{"job_types":["br","lightning"]}}`, + func(gs *rmpb.ResourceGroup) { + gs.RUSettings = &rmpb.GroupRequestUnitSettings{ + RU: &rmpb.TokenBucket{ + Settings: &rmpb.TokenLimitSettings{ + FillRate: 30000, + BurstLimit: -1, + }, + }, + } + gs.RunawaySettings = &rmpb.RunawaySettings{ + Rule: &rmpb.RunawayRule{ + ExecElapsedTimeMs: 1000, + }, + Action: rmpb.RunawayAction_Kill, + Watch: &rmpb.RunawayWatch{ + Type: rmpb.RunawayWatchType_Similar, + LastingDurationMs: 100000, + }, + } + gs.BackgroundSettings = &rmpb.BackgroundSettings{ + JobTypes: []string{"br", "lightning"}, + } + }, + }, + {"default", rmpb.GroupMode_RUMode, false, true, + `{"name":"default","mode":1,"r_u_settings":{"r_u":{"settings":{"fill_rate":10000,"burst_limit":-1},"state":{"initialized":false}}},"priority":0,"background_settings":{"job_types":["br"]}}`, + func(gs *rmpb.ResourceGroup) { + gs.RUSettings = &rmpb.GroupRequestUnitSettings{ + RU: &rmpb.TokenBucket{ + Settings: &rmpb.TokenLimitSettings{ + FillRate: 10000, + BurstLimit: -1, + }, + }, + } + gs.BackgroundSettings = &rmpb.BackgroundSettings{ + JobTypes: []string{"br"}, + } + }, + }, + } + + checkErr := func(err error, success bool) { + if success { + re.NoError(err) + } else { + re.Error(err) + } + } + + finalNum := 1 + // Test Resource Group CURD via gRPC + for i, tcase := range testCasesSet1 { + group := &rmpb.ResourceGroup{ + Name: tcase.name, + Mode: tcase.mode, + } + // Create Resource Group + resp, err := cli.AddResourceGroup(suite.ctx, group) + checkErr(err, true) + if tcase.isNewGroup { + finalNum++ + re.Contains(resp, "Success!") + } + + // Modify Resource Group + tcase.modifySettings(group) + mresp, err := cli.ModifyResourceGroup(suite.ctx, group) + checkErr(err, tcase.modifySuccess) + if tcase.modifySuccess { + re.Contains(mresp, "Success!") + } + + // Get Resource Group + gresp, err := cli.GetResourceGroup(suite.ctx, tcase.name) + re.NoError(err) + re.Equal(tcase.name, gresp.Name) + if tcase.modifySuccess { + re.Equal(group, gresp) + } + + // Last one, Check list and delete all resource groups + if i == len(testCasesSet1)-1 { + // List Resource Groups + lresp, err := cli.ListResourceGroups(suite.ctx) + re.NoError(err) + re.Equal(finalNum, len(lresp)) + + for _, g := range lresp { + // Delete Resource Group + dresp, err := cli.DeleteResourceGroup(suite.ctx, g.Name) + if g.Name == "default" { + re.Contains(err.Error(), "cannot delete reserved group") + continue + } + re.NoError(err) + re.Contains(dresp, "Success!") + _, err = cli.GetResourceGroup(suite.ctx, g.Name) + re.EqualError(err, fmt.Sprintf("get resource group %v failed, rpc error: code = Unknown desc = resource group not found", g.Name)) + } + + // to test the deletion of persistence + suite.resignAndWaitLeader() + // List Resource Group + lresp, err = cli.ListResourceGroups(suite.ctx) + re.NoError(err) + re.Equal(1, len(lresp)) + } + } + + // Test Resource Group CURD via HTTP + finalNum = 1 + getAddr := func(i int) string { + server := suite.cluster.GetServer(suite.cluster.GetLeader()) + if i%2 == 1 { + server = suite.cluster.GetServer(suite.cluster.GetFollower()) + } + return server.GetAddr() + } + for i, tcase := range testCasesSet1 { + // Create Resource Group + group := &rmpb.ResourceGroup{ + Name: tcase.name, + Mode: tcase.mode, + } + createJSON, err := json.Marshal(group) + re.NoError(err) + resp, err := http.Post(getAddr(i)+"/resource-manager/api/v1/config/group", "application/json", strings.NewReader(string(createJSON))) + re.NoError(err) + defer resp.Body.Close() + re.Equal(http.StatusOK, resp.StatusCode) + if tcase.isNewGroup { + finalNum++ + } + + // Modify Resource Group + tcase.modifySettings(group) + modifyJSON, err := json.Marshal(group) + re.NoError(err) + req, err := http.NewRequest(http.MethodPut, getAddr(i+1)+"/resource-manager/api/v1/config/group", strings.NewReader(string(modifyJSON))) + re.NoError(err) + req.Header.Set("Content-Type", "application/json") + resp, err = http.DefaultClient.Do(req) + re.NoError(err) + defer resp.Body.Close() + if tcase.modifySuccess { + re.Equal(http.StatusOK, resp.StatusCode) + } else { + re.Equal(http.StatusInternalServerError, resp.StatusCode) + } + + // Get Resource Group + resp, err = http.Get(getAddr(i) + "/resource-manager/api/v1/config/group/" + tcase.name) + re.NoError(err) + defer resp.Body.Close() + re.Equal(http.StatusOK, resp.StatusCode) + respString, err := io.ReadAll(resp.Body) + re.NoError(err) + re.Contains(string(respString), tcase.name) + if tcase.modifySuccess { + re.Equal(string(respString), tcase.expectMarshal) + } + + // Last one, Check list and delete all resource groups + if i == len(testCasesSet1)-1 { + resp, err := http.Get(getAddr(i) + "/resource-manager/api/v1/config/groups") + re.NoError(err) + defer resp.Body.Close() + re.Equal(http.StatusOK, resp.StatusCode) + respString, err := io.ReadAll(resp.Body) + re.NoError(err) + groups := make([]*server.ResourceGroup, 0) + json.Unmarshal(respString, &groups) + re.Equal(finalNum, len(groups)) + + // Delete all resource groups + for _, g := range groups { + req, err := http.NewRequest(http.MethodDelete, getAddr(i+1)+"/resource-manager/api/v1/config/group/"+g.Name, nil) + re.NoError(err) + resp, err := http.DefaultClient.Do(req) + re.NoError(err) + defer resp.Body.Close() + respString, err := io.ReadAll(resp.Body) + re.NoError(err) + if g.Name == "default" { + re.Contains(string(respString), "cannot delete reserved group") + continue + } + re.Equal(http.StatusOK, resp.StatusCode) + re.Contains(string(respString), "Success!") + } + + // verify again + resp1, err := http.Get(getAddr(i) + "/resource-manager/api/v1/config/groups") + re.NoError(err) + defer resp1.Body.Close() + re.Equal(http.StatusOK, resp1.StatusCode) + respString1, err := io.ReadAll(resp1.Body) + re.NoError(err) + groups1 := make([]server.ResourceGroup, 0) + json.Unmarshal(respString1, &groups1) + re.Equal(1, len(groups1)) + } + } + + // test restart cluster + groups, err := cli.ListResourceGroups(suite.ctx) + re.NoError(err) + servers := suite.cluster.GetServers() + re.NoError(suite.cluster.StopAll()) + serverList := make([]*tests.TestServer, 0, len(servers)) + for _, s := range servers { + serverList = append(serverList, s) + } + re.NoError(suite.cluster.RunServers(serverList)) + suite.cluster.WaitLeader() + var newGroups []*rmpb.ResourceGroup + testutil.Eventually(suite.Require(), func() bool { + var err error + newGroups, err = cli.ListResourceGroups(suite.ctx) + return err == nil + }, testutil.WithWaitFor(time.Second)) + re.Equal(groups, newGroups) +} + +func (suite *resourceManagerClientTestSuite) TestResourceManagerClientFailover() { + re := suite.Require() + cli := suite.client + + group := &rmpb.ResourceGroup{ + Name: "test3", + Mode: rmpb.GroupMode_RUMode, + RUSettings: &rmpb.GroupRequestUnitSettings{ + RU: &rmpb.TokenBucket{ + Settings: &rmpb.TokenLimitSettings{ + FillRate: 10000, + }, + Tokens: 100000, + }, + }, + } + addResp, err := cli.AddResourceGroup(suite.ctx, group) + re.NoError(err) + re.Contains(addResp, "Success!") + getResp, err := cli.GetResourceGroup(suite.ctx, group.GetName()) + re.NoError(err) + re.NotNil(getResp) + re.Equal(*group, *getResp) + + // Change the leader after each time we modify the resource group. + for i := 0; i < 4; i++ { + group.RUSettings.RU.Settings.FillRate += uint64(i) + modifyResp, err := cli.ModifyResourceGroup(suite.ctx, group) + re.NoError(err) + re.Contains(modifyResp, "Success!") + suite.resignAndWaitLeader() + getResp, err = cli.GetResourceGroup(suite.ctx, group.GetName()) + re.NoError(err) + re.NotNil(getResp) + re.Equal(group.RUSettings.RU.Settings.FillRate, getResp.RUSettings.RU.Settings.FillRate) + } +} + +func (suite *resourceManagerClientTestSuite) TestResourceManagerClientDegradedMode() { + re := suite.Require() + cli := suite.client + + group := &rmpb.ResourceGroup{ + Name: "modetest", + Mode: rmpb.GroupMode_RUMode, + RUSettings: &rmpb.GroupRequestUnitSettings{ + RU: &rmpb.TokenBucket{ + Settings: &rmpb.TokenLimitSettings{ + FillRate: 10, + BurstLimit: 10, + }, + Tokens: 10, + }, + }, + } + addResp, err := cli.AddResourceGroup(suite.ctx, group) + re.NoError(err) + re.Contains(addResp, "Success!") + + cfg := &controller.RequestUnitConfig{ + ReadBaseCost: 1, + ReadCostPerByte: 1, + WriteBaseCost: 1, + WriteCostPerByte: 1, + CPUMsCost: 1, + } + re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/mcs/resourcemanager/server/acquireFailed", `return(true)`)) + re.NoError(failpoint.Enable("github.com/tikv/pd/client/resource_group/controller/degradedModeRU", "return(true)")) + controller, _ := controller.NewResourceGroupController(suite.ctx, 1, cli, cfg) + controller.Start(suite.ctx) + tc := tokenConsumptionPerSecond{ + rruTokensAtATime: 0, + wruTokensAtATime: 10000, + } + tc2 := tokenConsumptionPerSecond{ + rruTokensAtATime: 0, + wruTokensAtATime: 2, + } + resourceName := "modetest" + controller.OnRequestWait(suite.ctx, resourceName, tc.makeWriteRequest()) + time.Sleep(time.Second * 2) + beginTime := time.Now() + // This is used to make sure resource group in lowRU. + for i := 0; i < 100; i++ { + controller.OnRequestWait(suite.ctx, resourceName, tc2.makeWriteRequest()) + } + for i := 0; i < 100; i++ { + controller.OnRequestWait(suite.ctx, resourceName, tc.makeWriteRequest()) + } + endTime := time.Now() + // we can not check `inDegradedMode` because of data race. + re.True(endTime.Before(beginTime.Add(time.Second))) + controller.Stop() + re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/mcs/resourcemanager/server/acquireFailed")) + re.NoError(failpoint.Disable("github.com/tikv/pd/client/resource_group/controller/degradedModeRU")) +} + +func (suite *resourceManagerClientTestSuite) TestLoadRequestUnitConfig() { + re := suite.Require() + cli := suite.client + // Test load from resource manager. + ctr, err := controller.NewResourceGroupController(suite.ctx, 1, cli, nil) + re.NoError(err) + config := ctr.GetConfig() + re.NotNil(config) + expectedConfig := controller.DefaultRUConfig() + re.Equal(expectedConfig.ReadBaseCost, config.ReadBaseCost) + re.Equal(expectedConfig.ReadBytesCost, config.ReadBytesCost) + re.Equal(expectedConfig.WriteBaseCost, config.WriteBaseCost) + re.Equal(expectedConfig.WriteBytesCost, config.WriteBytesCost) + re.Equal(expectedConfig.CPUMsCost, config.CPUMsCost) + // Test init from given config. + ruConfig := &controller.RequestUnitConfig{ + ReadBaseCost: 1, + ReadCostPerByte: 2, + WriteBaseCost: 3, + WriteCostPerByte: 4, + CPUMsCost: 5, + } + ctr, err = controller.NewResourceGroupController(suite.ctx, 1, cli, ruConfig) + re.NoError(err) + config = ctr.GetConfig() + re.NotNil(config) + controllerConfig := controller.DefaultConfig() + controllerConfig.RequestUnit = *ruConfig + expectedConfig = controller.GenerateRUConfig(controllerConfig) + re.Equal(expectedConfig.ReadBaseCost, config.ReadBaseCost) + re.Equal(expectedConfig.ReadBytesCost, config.ReadBytesCost) + re.Equal(expectedConfig.WriteBaseCost, config.WriteBaseCost) + re.Equal(expectedConfig.WriteBytesCost, config.WriteBytesCost) + re.Equal(expectedConfig.CPUMsCost, config.CPUMsCost) + // refer github.com/tikv/pd/pkg/mcs/resourcemanager/server/enableDegradedMode, check with 1s. + re.Equal(time.Second, config.DegradedModeWaitDuration) +} + +func (suite *resourceManagerClientTestSuite) TestRemoveStaleResourceGroup() { + re := suite.Require() + cli := suite.client + + for _, group := range suite.initGroups { + resp, err := cli.AddResourceGroup(suite.ctx, group) + re.NoError(err) + re.Contains(resp, "Success!") + } + + ruConfig := &controller.RequestUnitConfig{ + ReadBaseCost: 1, + ReadCostPerByte: 1, + } + re.NoError(failpoint.Enable("github.com/tikv/pd/client/resource_group/controller/fastCleanup", `return(true)`)) + controller, _ := controller.NewResourceGroupController(suite.ctx, 1, cli, ruConfig) + controller.Start(suite.ctx) + + testConfig := struct { + tcs tokenConsumptionPerSecond + times int + }{ + tcs: tokenConsumptionPerSecond{ + rruTokensAtATime: 100, + }, + times: 100, + } + // Mock client binds one resource group and then closed + rreq := testConfig.tcs.makeReadRequest() + rres := testConfig.tcs.makeReadResponse() + for j := 0; j < testConfig.times; j++ { + controller.OnRequestWait(suite.ctx, suite.initGroups[0].Name, rreq) + controller.OnResponse(suite.initGroups[0].Name, rreq, rres) + time.Sleep(100 * time.Microsecond) + } + time.Sleep(1 * time.Second) + + re.Nil(controller.GetActiveResourceGroup(suite.initGroups[0].Name)) + + re.NoError(failpoint.Disable("github.com/tikv/pd/client/resource_group/controller/fastCleanup")) + controller.Stop() +} + +func (suite *resourceManagerClientTestSuite) TestSkipConsumptionForBackgroundJobs() { + re := suite.Require() + cli := suite.client + + for _, group := range suite.initGroups { + resp, err := cli.AddResourceGroup(suite.ctx, group) + re.NoError(err) + re.Contains(resp, "Success!") + } + + cfg := &controller.RequestUnitConfig{ + ReadBaseCost: 1, + ReadCostPerByte: 1, + WriteBaseCost: 1, + WriteCostPerByte: 1, + CPUMsCost: 1, + } + c, _ := controller.NewResourceGroupController(suite.ctx, 1, cli, cfg) + c.Start(suite.ctx) + + resourceGroupName := suite.initGroups[1].Name + re.False(c.IsBackgroundRequest(suite.ctx, resourceGroupName, "internal_default")) + // test fallback for nil. + re.False(c.IsBackgroundRequest(suite.ctx, resourceGroupName, "internal_lightning")) + re.False(c.IsBackgroundRequest(suite.ctx, resourceGroupName, "internal_ddl")) + re.False(c.IsBackgroundRequest(suite.ctx, resourceGroupName, "")) + + resourceGroupName = "background_job" + re.True(c.IsBackgroundRequest(suite.ctx, resourceGroupName, "internal_br")) + re.True(c.IsBackgroundRequest(suite.ctx, resourceGroupName, "internal_lightning")) + // test fallback for nil. + re.False(c.IsBackgroundRequest(suite.ctx, resourceGroupName, "internal_ddl")) + + // modify `Default` to check fallback. + resp, err := cli.ModifyResourceGroup(suite.ctx, &rmpb.ResourceGroup{ + Name: "default", + Mode: rmpb.GroupMode_RUMode, + RUSettings: &rmpb.GroupRequestUnitSettings{ + RU: &rmpb.TokenBucket{ + Settings: &rmpb.TokenLimitSettings{ + FillRate: 1, + BurstLimit: -1, + }, + Tokens: 1, + }, + }, + BackgroundSettings: &rmpb.BackgroundSettings{JobTypes: []string{"lightning", "ddl"}}, + }) + re.NoError(err) + re.Contains(resp, "Success!") + // wait for watch event modify. + time.Sleep(time.Millisecond * 100) + + resourceGroupName = suite.initGroups[1].Name + re.False(c.IsBackgroundRequest(suite.ctx, resourceGroupName, "internal_default")) + // test fallback for `"lightning", "ddl"`. + re.True(c.IsBackgroundRequest(suite.ctx, resourceGroupName, "internal_lightning")) + re.True(c.IsBackgroundRequest(suite.ctx, resourceGroupName, "internal_ddl")) + re.False(c.IsBackgroundRequest(suite.ctx, resourceGroupName, "")) + + resourceGroupName = "background_job" + re.True(c.IsBackgroundRequest(suite.ctx, resourceGroupName, "internal_br")) + re.True(c.IsBackgroundRequest(suite.ctx, resourceGroupName, "internal_lightning")) + // test fallback for `"lightning", "ddl"`. + re.False(c.IsBackgroundRequest(suite.ctx, resourceGroupName, "internal_ddl")) + + c.Stop() +} diff --git a/tests/mcs/resource_manager/server_test.go b/tests/integrations/mcs/resourcemanager/server_test.go similarity index 56% rename from tests/mcs/resource_manager/server_test.go rename to tests/integrations/mcs/resourcemanager/server_test.go index e12987dc023..309c511ee65 100644 --- a/tests/mcs/resource_manager/server_test.go +++ b/tests/integrations/mcs/resourcemanager/server_test.go @@ -21,15 +21,11 @@ import ( "net/http" "strings" "testing" - "time" rmpb "github.com/pingcap/kvproto/pkg/resource_manager" "github.com/stretchr/testify/require" "github.com/tikv/pd/client/grpcutil" - "github.com/tikv/pd/pkg/mcs/discovery" - rm "github.com/tikv/pd/pkg/mcs/resource_manager/server" "github.com/tikv/pd/pkg/utils/tempurl" - "github.com/tikv/pd/pkg/utils/testutil" "github.com/tikv/pd/tests" ) @@ -38,7 +34,7 @@ func TestResourceManagerServer(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - cluster, err := tests.NewTestCluster(ctx, 1) + cluster, err := tests.NewTestAPICluster(ctx, 1) defer cluster.Destroy() re.NoError(err) @@ -48,20 +44,12 @@ func TestResourceManagerServer(t *testing.T) { leaderName := cluster.WaitLeader() leader := cluster.GetServer(leaderName) - cfg, err := rm.NewTestDefaultConfig() - re.NoError(err) - cfg.BackendEndpoints = leader.GetAddr() - cfg.ListenAddr = tempurl.Alloc() - - s, cleanup, err := rm.NewTestServer(ctx, re, cfg) - re.NoError(err) + s, cleanup := tests.StartSingleResourceManagerTestServer(ctx, re, leader.GetAddr(), tempurl.Alloc()) + addr := s.GetAddr() defer cleanup() - testutil.Eventually(re, func() bool { - return s.IsServing() - }, testutil.WithWaitFor(5*time.Second), testutil.WithTickInterval(50*time.Millisecond)) // Test registered GRPC Service - cc, err := grpcutil.GetClientConn(ctx, cfg.ListenAddr, nil) + cc, err := grpcutil.GetClientConn(ctx, addr, nil) re.NoError(err) defer cc.Close() @@ -72,7 +60,7 @@ func TestResourceManagerServer(t *testing.T) { re.ErrorContains(err, "resource group not found") // Test registered REST HTTP Handler - url := cfg.ListenAddr + "/resource-manager/api/v1/config" + url := addr + "/resource-manager/api/v1/config" { resp, err := http.Get(url + "/groups") re.NoError(err) @@ -80,7 +68,7 @@ func TestResourceManagerServer(t *testing.T) { re.Equal(http.StatusOK, resp.StatusCode) respString, err := io.ReadAll(resp.Body) re.NoError(err) - re.Equal("[]", string(respString)) + re.Equal(`[{"name":"default","mode":1,"r_u_settings":{"r_u":{"settings":{"fill_rate":2147483647,"burst_limit":-1},"state":{"initialized":false}}},"priority":8}]`, string(respString)) } { group := &rmpb.ResourceGroup{ @@ -101,44 +89,17 @@ func TestResourceManagerServer(t *testing.T) { re.Equal(http.StatusOK, resp.StatusCode) respString, err := io.ReadAll(resp.Body) re.NoError(err) - re.Equal("{\"name\":\"pingcap\",\"mode\":1,\"r_u_settings\":{\"r_u\":{\"state\":{\"initialized\":false}}}}", string(respString)) + re.Equal("{\"name\":\"pingcap\",\"mode\":1,\"r_u_settings\":{\"r_u\":{\"state\":{\"initialized\":false}}},\"priority\":0}", string(respString)) } -} - -func TestResourceManagerRegister(t *testing.T) { - re := require.New(t) - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - cluster, err := tests.NewTestCluster(ctx, 1) - defer cluster.Destroy() - re.NoError(err) - - err = cluster.RunInitialServers() - re.NoError(err) - leaderName := cluster.WaitLeader() - leader := cluster.GetServer(leaderName) - - cfg, err := rm.NewTestDefaultConfig() - re.NoError(err) - cfg.BackendEndpoints = leader.GetAddr() - cfg.ListenAddr = tempurl.Alloc() - - s, cleanup, err := rm.NewTestServer(ctx, re, cfg) - re.NoError(err) - defer cleanup() - testutil.Eventually(re, func() bool { - return s.IsServing() - }, testutil.WithWaitFor(5*time.Second), testutil.WithTickInterval(50*time.Millisecond)) - - client := leader.GetEtcdClient() - endpoints, err := discovery.Discover(client, "resource_manager") - re.NoError(err) - re.Equal(cfg.ListenAddr, endpoints[0]) - - s.Close() - endpoints, err = discovery.Discover(client, "resource_manager") - re.NoError(err) - re.Empty(endpoints) + // Test metrics handler + { + resp, err := http.Get(addr + "/metrics") + re.NoError(err) + defer resp.Body.Close() + re.Equal(http.StatusOK, resp.StatusCode) + respBytes, err := io.ReadAll(resp.Body) + re.NoError(err) + re.Contains(string(respBytes), "resource_manager_server_info") + } } diff --git a/tests/integrations/mcs/scheduling/config_test.go b/tests/integrations/mcs/scheduling/config_test.go new file mode 100644 index 00000000000..29f99c66f4c --- /dev/null +++ b/tests/integrations/mcs/scheduling/config_test.go @@ -0,0 +1,106 @@ +// Copyright 2023 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scheduling + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + "github.com/tikv/pd/pkg/mcs/scheduling/server/config" + sc "github.com/tikv/pd/pkg/schedule/config" + "github.com/tikv/pd/pkg/storage/endpoint" + "github.com/tikv/pd/pkg/utils/testutil" + "github.com/tikv/pd/pkg/versioninfo" + "github.com/tikv/pd/tests" +) + +type configTestSuite struct { + suite.Suite + + ctx context.Context + cancel context.CancelFunc + + // The PD cluster. + cluster *tests.TestCluster + // pdLeaderServer is the leader server of the PD cluster. + pdLeaderServer *tests.TestServer +} + +func TestConfig(t *testing.T) { + suite.Run(t, &configTestSuite{}) +} + +func (suite *configTestSuite) SetupSuite() { + re := suite.Require() + + var err error + suite.ctx, suite.cancel = context.WithCancel(context.Background()) + suite.cluster, err = tests.NewTestAPICluster(suite.ctx, 1) + re.NoError(err) + err = suite.cluster.RunInitialServers() + re.NoError(err) + leaderName := suite.cluster.WaitLeader() + suite.pdLeaderServer = suite.cluster.GetServer(leaderName) + re.NoError(suite.pdLeaderServer.BootstrapCluster()) +} + +func (suite *configTestSuite) TearDownSuite() { + suite.cancel() + suite.cluster.Destroy() +} + +func (suite *configTestSuite) TestConfigWatch() { + re := suite.Require() + + // Make sure the config is persisted before the watcher is created. + persistConfig(re, suite.pdLeaderServer) + // Create a config watcher. + watcher, err := config.NewWatcher( + suite.ctx, + suite.pdLeaderServer.GetEtcdClient(), + endpoint.ConfigPath(suite.cluster.GetCluster().GetId()), + config.NewPersistConfig(config.NewConfig()), + ) + re.NoError(err) + // Check the initial config value. + re.Equal(uint64(sc.DefaultMaxReplicas), watcher.GetReplicationConfig().MaxReplicas) + re.Equal(sc.DefaultSplitMergeInterval, watcher.GetScheduleConfig().SplitMergeInterval.Duration) + re.Equal("0.0.0", watcher.GetClusterVersion().String()) + // Update the config and check if the scheduling config watcher can get the latest value. + suite.pdLeaderServer.GetPersistOptions().SetMaxReplicas(5) + persistConfig(re, suite.pdLeaderServer) + testutil.Eventually(re, func() bool { + return watcher.GetReplicationConfig().MaxReplicas == 5 + }) + suite.pdLeaderServer.GetPersistOptions().SetSplitMergeInterval(2 * sc.DefaultSplitMergeInterval) + persistConfig(re, suite.pdLeaderServer) + testutil.Eventually(re, func() bool { + return watcher.GetScheduleConfig().SplitMergeInterval.Duration == 2*sc.DefaultSplitMergeInterval + }) + suite.pdLeaderServer.GetPersistOptions().SetClusterVersion(versioninfo.MinSupportedVersion(versioninfo.Version4_0)) + persistConfig(re, suite.pdLeaderServer) + testutil.Eventually(re, func() bool { + return watcher.GetClusterVersion().String() == "4.0.0" + }) + watcher.Close() +} + +// Manually trigger the config persistence in the PD API server side. +func persistConfig(re *require.Assertions, pdLeaderServer *tests.TestServer) { + err := pdLeaderServer.GetPersistOptions().Persist(pdLeaderServer.GetServer().GetStorage()) + re.NoError(err) +} diff --git a/tests/integrations/mcs/testutil.go b/tests/integrations/mcs/testutil.go new file mode 100644 index 00000000000..bbedd65209d --- /dev/null +++ b/tests/integrations/mcs/testutil.go @@ -0,0 +1,124 @@ +// Copyright 2023 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mcs + +import ( + "context" + "sync" + + "github.com/stretchr/testify/require" + pd "github.com/tikv/pd/client" + "github.com/tikv/pd/pkg/utils/testutil" + "github.com/tikv/pd/pkg/utils/tsoutil" +) + +// SetupClientWithAPIContext creates a TSO client with api context name for test. +func SetupClientWithAPIContext( + ctx context.Context, re *require.Assertions, apiCtx pd.APIContext, endpoints []string, opts ...pd.ClientOption, +) pd.Client { + cli, err := pd.NewClientWithAPIContext(ctx, apiCtx, endpoints, pd.SecurityOption{}, opts...) + re.NoError(err) + return cli +} + +// SetupClientWithKeyspaceID creates a TSO client with the given keyspace id for test. +func SetupClientWithKeyspaceID( + ctx context.Context, re *require.Assertions, + keyspaceID uint32, endpoints []string, opts ...pd.ClientOption, +) pd.Client { + cli, err := pd.NewClientWithKeyspace(ctx, keyspaceID, endpoints, pd.SecurityOption{}, opts...) + re.NoError(err) + return cli +} + +// WaitForTSOServiceAvailable waits for the pd client being served by the tso server side +func WaitForTSOServiceAvailable( + ctx context.Context, re *require.Assertions, client pd.Client, +) { + testutil.Eventually(re, func() bool { + _, _, err := client.GetTS(ctx) + return err == nil + }) +} + +// CheckMultiKeyspacesTSO checks the correctness of TSO for multiple keyspaces. +func CheckMultiKeyspacesTSO( + ctx context.Context, re *require.Assertions, + clients []pd.Client, parallelAct func(), +) { + ctx, cancel := context.WithCancel(ctx) + wg := sync.WaitGroup{} + wg.Add(len(clients)) + + for _, client := range clients { + go func(cli pd.Client) { + defer wg.Done() + var ts, lastTS uint64 + for { + select { + case <-ctx.Done(): + // Make sure the lastTS is not empty + re.NotEmpty(lastTS) + return + default: + } + physical, logical, err := cli.GetTS(ctx) + // omit the error check since there are many kinds of errors + if err != nil { + continue + } + ts = tsoutil.ComposeTS(physical, logical) + re.Less(lastTS, ts) + lastTS = ts + } + }(client) + } + + wg.Add(1) + go func() { + defer wg.Done() + parallelAct() + cancel() + }() + + wg.Wait() +} + +// WaitForMultiKeyspacesTSOAvailable waits for the given keyspaces being served by the tso server side +func WaitForMultiKeyspacesTSOAvailable( + ctx context.Context, re *require.Assertions, + keyspaceIDs []uint32, backendEndpoints []string, +) []pd.Client { + wg := sync.WaitGroup{} + wg.Add(len(keyspaceIDs)) + + clients := make([]pd.Client, 0, len(keyspaceIDs)) + for _, keyspaceID := range keyspaceIDs { + cli := SetupClientWithKeyspaceID(ctx, re, keyspaceID, backendEndpoints) + re.NotNil(cli) + clients = append(clients, cli) + + go func() { + defer wg.Done() + testutil.Eventually(re, func() bool { + _, _, err := cli.GetTS(ctx) + return err == nil + }) + }() + } + + wg.Wait() + return clients +} diff --git a/tests/integrations/mcs/tso/api_test.go b/tests/integrations/mcs/tso/api_test.go new file mode 100644 index 00000000000..fde6bcb8da0 --- /dev/null +++ b/tests/integrations/mcs/tso/api_test.go @@ -0,0 +1,174 @@ +// Copyright 2023 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tso + +import ( + "bytes" + "context" + "encoding/json" + "io" + "net/http" + "testing" + "time" + + "github.com/pingcap/failpoint" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + tso "github.com/tikv/pd/pkg/mcs/tso/server" + apis "github.com/tikv/pd/pkg/mcs/tso/server/apis/v1" + mcsutils "github.com/tikv/pd/pkg/mcs/utils" + "github.com/tikv/pd/pkg/storage/endpoint" + "github.com/tikv/pd/server/config" + "github.com/tikv/pd/tests" +) + +const ( + tsoKeyspaceGroupsPrefix = "/tso/api/v1/keyspace-groups" +) + +// dialClient used to dial http request. +var dialClient = &http.Client{ + Transport: &http.Transport{ + DisableKeepAlives: true, + }, +} + +type tsoAPITestSuite struct { + suite.Suite + ctx context.Context + cancel context.CancelFunc + pdCluster *tests.TestCluster + tsoCluster *tests.TestTSOCluster +} + +func TestTSOAPI(t *testing.T) { + suite.Run(t, new(tsoAPITestSuite)) +} + +func (suite *tsoAPITestSuite) SetupTest() { + re := suite.Require() + + var err error + suite.ctx, suite.cancel = context.WithCancel(context.Background()) + suite.pdCluster, err = tests.NewTestAPICluster(suite.ctx, 1) + re.NoError(err) + err = suite.pdCluster.RunInitialServers() + re.NoError(err) + leaderName := suite.pdCluster.WaitLeader() + pdLeaderServer := suite.pdCluster.GetServer(leaderName) + re.NoError(pdLeaderServer.BootstrapCluster()) + suite.tsoCluster, err = tests.NewTestTSOCluster(suite.ctx, 1, pdLeaderServer.GetAddr()) + re.NoError(err) +} + +func (suite *tsoAPITestSuite) TearDownTest() { + suite.cancel() + suite.tsoCluster.Destroy() + suite.pdCluster.Destroy() +} + +func (suite *tsoAPITestSuite) TestGetKeyspaceGroupMembers() { + re := suite.Require() + + primary := suite.tsoCluster.WaitForDefaultPrimaryServing(re) + re.NotNil(primary) + members := mustGetKeyspaceGroupMembers(re, primary) + re.Len(members, 1) + defaultGroupMember := members[mcsutils.DefaultKeyspaceGroupID] + re.NotNil(defaultGroupMember) + re.Equal(mcsutils.DefaultKeyspaceGroupID, defaultGroupMember.Group.ID) + re.True(defaultGroupMember.IsPrimary) + primaryMember, err := primary.GetMember(mcsutils.DefaultKeyspaceID, mcsutils.DefaultKeyspaceGroupID) + re.NoError(err) + re.Equal(primaryMember.GetLeaderID(), defaultGroupMember.PrimaryID) +} + +func mustGetKeyspaceGroupMembers(re *require.Assertions, server *tso.Server) map[uint32]*apis.KeyspaceGroupMember { + httpReq, err := http.NewRequest(http.MethodGet, server.GetAddr()+tsoKeyspaceGroupsPrefix+"/members", nil) + re.NoError(err) + httpResp, err := dialClient.Do(httpReq) + re.NoError(err) + defer httpResp.Body.Close() + data, err := io.ReadAll(httpResp.Body) + re.NoError(err) + re.Equal(http.StatusOK, httpResp.StatusCode, string(data)) + var resp map[uint32]*apis.KeyspaceGroupMember + re.NoError(json.Unmarshal(data, &resp)) + return resp +} + +func TestTSOServerStartFirst(t *testing.T) { + re := require.New(t) + re.NoError(failpoint.Enable("github.com/tikv/pd/server/delayStartServerLoop", `return(true)`)) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + apiCluster, err := tests.NewTestAPICluster(ctx, 1, func(conf *config.Config, serverName string) { + conf.Keyspace.PreAlloc = []string{"k1", "k2"} + }) + defer apiCluster.Destroy() + re.NoError(err) + addr := apiCluster.GetConfig().GetClientURL() + ch := make(chan struct{}) + defer close(ch) + clusterCh := make(chan *tests.TestTSOCluster) + defer close(clusterCh) + go func() { + tsoCluster, err := tests.NewTestTSOCluster(ctx, 2, addr) + re.NoError(err) + primary := tsoCluster.WaitForDefaultPrimaryServing(re) + re.NotNil(primary) + clusterCh <- tsoCluster + ch <- struct{}{} + }() + err = apiCluster.RunInitialServers() + re.NoError(err) + leaderName := apiCluster.WaitLeader() + pdLeaderServer := apiCluster.GetServer(leaderName) + re.NoError(pdLeaderServer.BootstrapCluster()) + re.NoError(err) + tsoCluster := <-clusterCh + defer tsoCluster.Destroy() + <-ch + + time.Sleep(time.Second * 1) + input := make(map[string]interface{}) + input["new-id"] = 1 + input["keyspaces"] = []uint32{2} + jsonBody, err := json.Marshal(input) + re.NoError(err) + httpReq, err := http.NewRequest(http.MethodPost, addr+"/pd/api/v2/tso/keyspace-groups/0/split", bytes.NewBuffer(jsonBody)) + re.NoError(err) + httpResp, err := dialClient.Do(httpReq) + re.NoError(err) + defer httpResp.Body.Close() + re.Equal(http.StatusOK, httpResp.StatusCode) + + httpReq, err = http.NewRequest(http.MethodGet, addr+"/pd/api/v2/tso/keyspace-groups/0", nil) + re.NoError(err) + httpResp, err = dialClient.Do(httpReq) + re.NoError(err) + data, err := io.ReadAll(httpResp.Body) + re.NoError(err) + defer httpResp.Body.Close() + re.Equal(http.StatusOK, httpResp.StatusCode) + + var group endpoint.KeyspaceGroup + re.NoError(json.Unmarshal(data, &group)) + re.Len(group.Keyspaces, 2) + re.Len(group.Members, 2) + + re.NoError(failpoint.Disable("github.com/tikv/pd/server/delayStartServerLoop")) +} diff --git a/tests/integrations/mcs/tso/keyspace_group_manager_test.go b/tests/integrations/mcs/tso/keyspace_group_manager_test.go new file mode 100644 index 00000000000..4af10fa7107 --- /dev/null +++ b/tests/integrations/mcs/tso/keyspace_group_manager_test.go @@ -0,0 +1,800 @@ +// Copyright 2023 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tso + +import ( + "context" + "math/rand" + "net/http" + "strings" + "sync" + "testing" + "time" + + "github.com/pingcap/failpoint" + "github.com/pingcap/kvproto/pkg/pdpb" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + pd "github.com/tikv/pd/client" + "github.com/tikv/pd/pkg/election" + "github.com/tikv/pd/pkg/errs" + "github.com/tikv/pd/pkg/keyspace" + mcsutils "github.com/tikv/pd/pkg/mcs/utils" + "github.com/tikv/pd/pkg/member" + "github.com/tikv/pd/pkg/storage/endpoint" + tsopkg "github.com/tikv/pd/pkg/tso" + "github.com/tikv/pd/pkg/utils/testutil" + "github.com/tikv/pd/pkg/utils/tsoutil" + "github.com/tikv/pd/server/apiv2/handlers" + "github.com/tikv/pd/server/config" + "github.com/tikv/pd/tests" + "github.com/tikv/pd/tests/integrations/mcs" + handlersutil "github.com/tikv/pd/tests/server/apiv2/handlers" +) + +type tsoKeyspaceGroupManagerTestSuite struct { + suite.Suite + + ctx context.Context + cancel context.CancelFunc + + // The PD cluster. + cluster *tests.TestCluster + // pdLeaderServer is the leader server of the PD cluster. + pdLeaderServer *tests.TestServer + // tsoCluster is the TSO service cluster. + tsoCluster *tests.TestTSOCluster +} + +func TestTSOKeyspaceGroupManager(t *testing.T) { + suite.Run(t, &tsoKeyspaceGroupManagerTestSuite{}) +} + +func (suite *tsoKeyspaceGroupManagerTestSuite) SetupSuite() { + re := suite.Require() + re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/tso/fastGroupSplitPatroller", `return(true)`)) + + var err error + suite.ctx, suite.cancel = context.WithCancel(context.Background()) + suite.cluster, err = tests.NewTestAPICluster(suite.ctx, 1) + re.NoError(err) + err = suite.cluster.RunInitialServers() + re.NoError(err) + leaderName := suite.cluster.WaitLeader() + suite.pdLeaderServer = suite.cluster.GetServer(leaderName) + re.NoError(suite.pdLeaderServer.BootstrapCluster()) + suite.tsoCluster, err = tests.NewTestTSOCluster(suite.ctx, 2, suite.pdLeaderServer.GetAddr()) + re.NoError(err) +} + +func (suite *tsoKeyspaceGroupManagerTestSuite) TearDownSuite() { + suite.cancel() + suite.tsoCluster.Destroy() + suite.cluster.Destroy() + suite.Require().NoError(failpoint.Disable("github.com/tikv/pd/pkg/tso/fastGroupSplitPatroller")) +} + +func (suite *tsoKeyspaceGroupManagerTestSuite) TearDownTest() { + cleanupKeyspaceGroups(suite.Require(), suite.pdLeaderServer) +} + +func cleanupKeyspaceGroups(re *require.Assertions, server *tests.TestServer) { + keyspaceGroups := handlersutil.MustLoadKeyspaceGroups(re, server, "0", "0") + for _, group := range keyspaceGroups { + // Do not delete default keyspace group. + if group.ID == mcsutils.DefaultKeyspaceGroupID { + continue + } + handlersutil.MustDeleteKeyspaceGroup(re, server, group.ID) + } +} + +func (suite *tsoKeyspaceGroupManagerTestSuite) TestKeyspacesServedByDefaultKeyspaceGroup() { + // There is only default keyspace group. Any keyspace, which hasn't been assigned to + // a keyspace group before, will be served by the default keyspace group. + re := suite.Require() + testutil.Eventually(re, func() bool { + for _, keyspaceID := range []uint32{0, 1, 2} { + served := false + for _, server := range suite.tsoCluster.GetServers() { + if server.IsKeyspaceServing(keyspaceID, mcsutils.DefaultKeyspaceGroupID) { + tam, err := server.GetTSOAllocatorManager(mcsutils.DefaultKeyspaceGroupID) + re.NoError(err) + re.NotNil(tam) + served = true + break + } + } + if !served { + return false + } + } + return true + }, testutil.WithWaitFor(5*time.Second), testutil.WithTickInterval(50*time.Millisecond)) + + // Any keyspace that was assigned to a keyspace group before, except default keyspace, + // won't be served at this time. Default keyspace will be served by default keyspace group + // all the time. + for _, server := range suite.tsoCluster.GetServers() { + server.IsKeyspaceServing(mcsutils.DefaultKeyspaceID, mcsutils.DefaultKeyspaceGroupID) + for _, keyspaceGroupID := range []uint32{1, 2, 3} { + server.IsKeyspaceServing(mcsutils.DefaultKeyspaceID, keyspaceGroupID) + server.IsKeyspaceServing(mcsutils.DefaultKeyspaceID, keyspaceGroupID) + for _, keyspaceID := range []uint32{1, 2, 3} { + if server.IsKeyspaceServing(keyspaceID, keyspaceGroupID) { + tam, err := server.GetTSOAllocatorManager(keyspaceGroupID) + re.NoError(err) + re.NotNil(tam) + } + } + } + } + + // Create a client for each keyspace and make sure they can successfully discover the service + // provided by the default keyspace group. + keyspaceIDs := []uint32{0, 1, 2, 3, 1000} + clients := mcs.WaitForMultiKeyspacesTSOAvailable( + suite.ctx, re, keyspaceIDs, []string{suite.pdLeaderServer.GetAddr()}) + re.Equal(len(keyspaceIDs), len(clients)) + mcs.CheckMultiKeyspacesTSO(suite.ctx, re, clients, func() { + time.Sleep(3 * time.Second) + }) +} + +func (suite *tsoKeyspaceGroupManagerTestSuite) TestKeyspacesServedByNonDefaultKeyspaceGroups() { + // Create multiple keyspace groups, and every keyspace should be served by one of them + // on a tso server. + re := suite.Require() + + // Create keyspace groups. + params := []struct { + keyspaceGroupID uint32 + keyspaceIDs []uint32 + }{ + {0, []uint32{0, 10}}, + {1, []uint32{1, 11}}, + {2, []uint32{2, 12}}, + } + + for _, param := range params { + if param.keyspaceGroupID == 0 { + // we have already created default keyspace group, so we can skip it. + // keyspace 10 isn't assigned to any keyspace group, so they will be + // served by default keyspace group. + continue + } + handlersutil.MustCreateKeyspaceGroup(re, suite.pdLeaderServer, &handlers.CreateKeyspaceGroupParams{ + KeyspaceGroups: []*endpoint.KeyspaceGroup{ + { + ID: param.keyspaceGroupID, + UserKind: endpoint.Standard.String(), + Members: suite.tsoCluster.GetKeyspaceGroupMember(), + Keyspaces: param.keyspaceIDs, + }, + }, + }) + } + + // Wait until all keyspace groups are ready. + testutil.Eventually(re, func() bool { + for _, param := range params { + for _, keyspaceID := range param.keyspaceIDs { + served := false + for _, server := range suite.tsoCluster.GetServers() { + if server.IsKeyspaceServing(keyspaceID, param.keyspaceGroupID) { + am, err := server.GetTSOAllocatorManager(param.keyspaceGroupID) + re.NoError(err) + re.NotNil(am) + + // Make sure every keyspace group is using the right timestamp path + // for loading/saving timestamp from/to etcd and the right primary path + // for primary election. + clusterID := suite.pdLeaderServer.GetClusterID() + rootPath := endpoint.TSOSvcRootPath(clusterID) + primaryPath := endpoint.KeyspaceGroupPrimaryPath(rootPath, param.keyspaceGroupID) + timestampPath := endpoint.FullTimestampPath(clusterID, param.keyspaceGroupID) + re.Equal(timestampPath, am.GetTimestampPath(tsopkg.GlobalDCLocation)) + re.Equal(primaryPath, am.GetMember().GetLeaderPath()) + + served = true + } + } + if !served { + return false + } + } + } + return true + }, testutil.WithWaitFor(5*time.Second), testutil.WithTickInterval(50*time.Millisecond)) + + // Create a client for each keyspace and make sure they can successfully discover the service + // provided by the corresponding keyspace group. + keyspaceIDs := make([]uint32, 0) + for _, param := range params { + keyspaceIDs = append(keyspaceIDs, param.keyspaceIDs...) + } + + clients := mcs.WaitForMultiKeyspacesTSOAvailable( + suite.ctx, re, keyspaceIDs, []string{suite.pdLeaderServer.GetAddr()}) + re.Equal(len(keyspaceIDs), len(clients)) + mcs.CheckMultiKeyspacesTSO(suite.ctx, re, clients, func() { + time.Sleep(3 * time.Second) + }) +} + +func (suite *tsoKeyspaceGroupManagerTestSuite) TestTSOKeyspaceGroupSplit() { + re := suite.Require() + // Create the keyspace group 1 with keyspaces [111, 222, 333]. + handlersutil.MustCreateKeyspaceGroup(re, suite.pdLeaderServer, &handlers.CreateKeyspaceGroupParams{ + KeyspaceGroups: []*endpoint.KeyspaceGroup{ + { + ID: 1, + UserKind: endpoint.Standard.String(), + Members: suite.tsoCluster.GetKeyspaceGroupMember(), + Keyspaces: []uint32{111, 222, 333}, + }, + }, + }) + kg1 := handlersutil.MustLoadKeyspaceGroupByID(re, suite.pdLeaderServer, 1) + re.Equal(uint32(1), kg1.ID) + re.Equal([]uint32{111, 222, 333}, kg1.Keyspaces) + re.False(kg1.IsSplitting()) + // Get a TSO from the keyspace group 1. + var ( + ts pdpb.Timestamp + err error + ) + testutil.Eventually(re, func() bool { + ts, err = suite.requestTSO(re, 222, 1) + return err == nil && tsoutil.CompareTimestamp(&ts, &pdpb.Timestamp{}) > 0 + }) + ts.Physical += time.Hour.Milliseconds() + // Set the TSO of the keyspace group 1 to a large value. + err = suite.tsoCluster.GetPrimaryServer(222, 1).GetHandler().ResetTS(tsoutil.GenerateTS(&ts), false, true, 1) + re.NoError(err) + // Split the keyspace group 1 to 2. + handlersutil.MustSplitKeyspaceGroup(re, suite.pdLeaderServer, 1, &handlers.SplitKeyspaceGroupByIDParams{ + NewID: 2, + Keyspaces: []uint32{222, 333}, + }) + // Wait for the split to complete automatically even there is no TSO request from the outside. + testutil.Eventually(re, func() bool { + kg2 := handlersutil.MustLoadKeyspaceGroupByID(re, suite.pdLeaderServer, 2) + re.Equal(uint32(2), kg2.ID) + re.Equal([]uint32{222, 333}, kg2.Keyspaces) + return !kg2.IsSplitting() + }) + // Check the split TSO from keyspace group 2 now. + splitTS, err := suite.requestTSO(re, 222, 2) + re.NoError(err) + re.Greater(tsoutil.CompareTimestamp(&splitTS, &ts), 0) +} + +func (suite *tsoKeyspaceGroupManagerTestSuite) requestTSO( + re *require.Assertions, + keyspaceID, keyspaceGroupID uint32, +) (pdpb.Timestamp, error) { + primary := suite.tsoCluster.WaitForPrimaryServing(re, keyspaceID, keyspaceGroupID) + kgm := primary.GetKeyspaceGroupManager() + re.NotNil(kgm) + ts, _, err := kgm.HandleTSORequest(keyspaceID, keyspaceGroupID, tsopkg.GlobalDCLocation, 1) + return ts, err +} + +func (suite *tsoKeyspaceGroupManagerTestSuite) TestTSOKeyspaceGroupSplitElection() { + re := suite.Require() + // Create the keyspace group 1 with keyspaces [111, 222, 333]. + handlersutil.MustCreateKeyspaceGroup(re, suite.pdLeaderServer, &handlers.CreateKeyspaceGroupParams{ + KeyspaceGroups: []*endpoint.KeyspaceGroup{ + { + ID: 1, + UserKind: endpoint.Standard.String(), + Members: suite.tsoCluster.GetKeyspaceGroupMember(), + Keyspaces: []uint32{111, 222, 333}, + }, + }, + }) + kg1 := handlersutil.MustLoadKeyspaceGroupByID(re, suite.pdLeaderServer, 1) + re.Equal(uint32(1), kg1.ID) + re.Equal([]uint32{111, 222, 333}, kg1.Keyspaces) + re.False(kg1.IsSplitting()) + // Split the keyspace group 1 to 2. + handlersutil.MustSplitKeyspaceGroup(re, suite.pdLeaderServer, 1, &handlers.SplitKeyspaceGroupByIDParams{ + NewID: 2, + Keyspaces: []uint32{222, 333}, + }) + kg2 := handlersutil.MustLoadKeyspaceGroupByID(re, suite.pdLeaderServer, 2) + re.Equal(uint32(2), kg2.ID) + re.Equal([]uint32{222, 333}, kg2.Keyspaces) + re.True(kg2.IsSplitTarget()) + // Check the leadership. + member1, err := suite.tsoCluster.WaitForPrimaryServing(re, 111, 1).GetMember(111, 1) + re.NoError(err) + re.NotNil(member1) + member2, err := suite.tsoCluster.WaitForPrimaryServing(re, 222, 2).GetMember(222, 2) + re.NoError(err) + re.NotNil(member2) + // Wait for the leader of the keyspace group 1 and 2 to be elected. + testutil.Eventually(re, func() bool { + return len(member1.GetLeaderListenUrls()) > 0 && len(member2.GetLeaderListenUrls()) > 0 + }) + // Check if the leader of the keyspace group 1 and 2 are the same. + re.Equal(member1.GetLeaderListenUrls(), member2.GetLeaderListenUrls()) + // Resign and block the leader of the keyspace group 1 from being elected. + member1.(*member.Participant).SetCampaignChecker(func(*election.Leadership) bool { + return false + }) + member1.ResetLeader() + // The leader of the keyspace group 2 should be resigned also. + testutil.Eventually(re, func() bool { + return member2.IsLeader() == false + }) + // Check if the leader of the keyspace group 1 and 2 are the same again. + member1.(*member.Participant).SetCampaignChecker(nil) + testutil.Eventually(re, func() bool { + return len(member1.GetLeaderListenUrls()) > 0 && len(member2.GetLeaderListenUrls()) > 0 + }) + re.Equal(member1.GetLeaderListenUrls(), member2.GetLeaderListenUrls()) + // Wait for the keyspace groups to finish the split. + waitFinishSplit(re, suite.pdLeaderServer, 1, 2, []uint32{111}, []uint32{222, 333}) +} + +func waitFinishSplit( + re *require.Assertions, + server *tests.TestServer, + splitSourceID, splitTargetID uint32, + splitSourceKeyspaces, splitTargetKeyspaces []uint32, +) { + testutil.Eventually(re, func() bool { + kg, code := handlersutil.TryLoadKeyspaceGroupByID(re, server, splitTargetID) + if code != http.StatusOK { + return false + } + re.Equal(splitTargetID, kg.ID) + re.Equal(splitTargetKeyspaces, kg.Keyspaces) + return !kg.IsSplitTarget() + }) + testutil.Eventually(re, func() bool { + kg, code := handlersutil.TryLoadKeyspaceGroupByID(re, server, splitSourceID) + if code != http.StatusOK { + return false + } + re.Equal(splitSourceID, kg.ID) + re.Equal(splitSourceKeyspaces, kg.Keyspaces) + return !kg.IsSplitSource() + }) +} + +func (suite *tsoKeyspaceGroupManagerTestSuite) TestTSOKeyspaceGroupSplitClient() { + re := suite.Require() + // Enable the failpoint to slow down the system time to test whether the TSO is monotonic. + re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/tso/systemTimeSlow", `return(true)`)) + // Create the keyspace group 1 with keyspaces [444, 555, 666]. + handlersutil.MustCreateKeyspaceGroup(re, suite.pdLeaderServer, &handlers.CreateKeyspaceGroupParams{ + KeyspaceGroups: []*endpoint.KeyspaceGroup{ + { + ID: 1, + UserKind: endpoint.Standard.String(), + Members: suite.tsoCluster.GetKeyspaceGroupMember(), + Keyspaces: []uint32{444, 555, 666}, + }, + }, + }) + kg1 := handlersutil.MustLoadKeyspaceGroupByID(re, suite.pdLeaderServer, 1) + re.Equal(uint32(1), kg1.ID) + re.Equal([]uint32{444, 555, 666}, kg1.Keyspaces) + re.False(kg1.IsSplitting()) + // Request the TSO for keyspace 555 concurrently via client. + cancel := suite.dispatchClient(re, 555, 1) + // Split the keyspace group 1 to 2. + handlersutil.MustSplitKeyspaceGroup(re, suite.pdLeaderServer, 1, &handlers.SplitKeyspaceGroupByIDParams{ + NewID: 2, + Keyspaces: []uint32{555, 666}, + }) + // Wait for the keyspace groups to finish the split. + waitFinishSplit(re, suite.pdLeaderServer, 1, 2, []uint32{444}, []uint32{555, 666}) + // Stop the client. + cancel() + re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/tso/systemTimeSlow")) +} + +func (suite *tsoKeyspaceGroupManagerTestSuite) dispatchClient( + re *require.Assertions, keyspaceID, keyspaceGroupID uint32, +) context.CancelFunc { + // Make sure the leader of the keyspace group is elected. + member, err := suite.tsoCluster. + WaitForPrimaryServing(re, keyspaceID, keyspaceGroupID). + GetMember(keyspaceID, keyspaceGroupID) + re.NoError(err) + re.NotNil(member) + // Prepare the client for keyspace. + tsoClient, err := pd.NewClientWithKeyspace(suite.ctx, keyspaceID, []string{suite.pdLeaderServer.GetAddr()}, pd.SecurityOption{}) + re.NoError(err) + re.NotNil(tsoClient) + var ( + wg sync.WaitGroup + ctx, cancel = context.WithCancel(suite.ctx) + lastPhysical, lastLogical int64 + ) + wg.Add(1) + go func() { + defer wg.Done() + for { + select { + case <-ctx.Done(): + return + default: + } + physical, logical, err := tsoClient.GetTS(ctx) + if err != nil { + errMsg := err.Error() + // Ignore the errors caused by the split and context cancellation. + if strings.Contains(errMsg, "context canceled") || + strings.Contains(errMsg, "not leader") || + strings.Contains(errMsg, "not served") || + strings.Contains(errMsg, "ErrKeyspaceNotAssigned") || + strings.Contains(errMsg, "ErrKeyspaceGroupIsMerging") { + continue + } + re.FailNow(errMsg) + } + if physical == lastPhysical { + re.Greater(logical, lastLogical) + } else { + re.Greater(physical, lastPhysical) + } + lastPhysical, lastLogical = physical, logical + } + }() + return func() { + // Wait for a while to make sure the client has sent more TSO requests. + time.Sleep(time.Second) + // Cancel the context to stop the client. + cancel() + // Wait for the client to stop. + wg.Wait() + } +} + +func (suite *tsoKeyspaceGroupManagerTestSuite) TestTSOKeyspaceGroupMembers() { + re := suite.Require() + re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/keyspace/skipSplitRegion", "return(true)")) + re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/keyspace/acceleratedAllocNodes", `return(true)`)) + // wait for finishing alloc nodes + waitFinishAllocNodes(re, suite.pdLeaderServer, mcsutils.DefaultKeyspaceGroupID) + testConfig := map[string]string{ + "config": "1", + "tso_keyspace_group_id": "0", + "user_kind": "basic", + } + handlersutil.MustCreateKeyspace(re, suite.pdLeaderServer, &handlers.CreateKeyspaceParams{ + Name: "test_keyspace", + Config: testConfig, + }) + waitFinishAllocNodes(re, suite.pdLeaderServer, mcsutils.DefaultKeyspaceGroupID) + re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/keyspace/skipSplitRegion")) + re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/keyspace/acceleratedAllocNodes")) +} + +func waitFinishAllocNodes(re *require.Assertions, server *tests.TestServer, groupID uint32) { + testutil.Eventually(re, func() bool { + kg := handlersutil.MustLoadKeyspaceGroupByID(re, server, groupID) + re.Equal(groupID, kg.ID) + return len(kg.Members) == mcsutils.DefaultKeyspaceGroupReplicaCount + }) +} + +func TestTwiceSplitKeyspaceGroup(t *testing.T) { + re := require.New(t) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/keyspace/acceleratedAllocNodes", `return(true)`)) + re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/tso/fastGroupSplitPatroller", `return(true)`)) + + // Init api server config but not start. + tc, err := tests.NewTestAPICluster(ctx, 1, func(conf *config.Config, _ string) { + conf.Keyspace.PreAlloc = []string{ + "keyspace_a", "keyspace_b", + } + }) + re.NoError(err) + pdAddr := tc.GetConfig().GetClientURL() + + // Start api server and tso server. + err = tc.RunInitialServers() + re.NoError(err) + defer tc.Destroy() + tc.WaitLeader() + leaderServer := tc.GetServer(tc.GetLeader()) + re.NoError(leaderServer.BootstrapCluster()) + + tsoCluster, err := tests.NewTestTSOCluster(ctx, 2, pdAddr) + re.NoError(err) + defer tsoCluster.Destroy() + tsoCluster.WaitForDefaultPrimaryServing(re) + + // First split keyspace group 0 to 1 with keyspace 2. + kgm := leaderServer.GetServer().GetKeyspaceGroupManager() + re.NotNil(kgm) + testutil.Eventually(re, func() bool { + err = kgm.SplitKeyspaceGroupByID(0, 1, []uint32{2}) + return err == nil + }) + + waitFinishSplit(re, leaderServer, 0, 1, []uint32{mcsutils.DefaultKeyspaceID, 1}, []uint32{2}) + + // Then split keyspace group 0 to 2 with keyspace 1. + testutil.Eventually(re, func() bool { + err = kgm.SplitKeyspaceGroupByID(0, 2, []uint32{1}) + return err == nil + }) + + waitFinishSplit(re, leaderServer, 0, 2, []uint32{mcsutils.DefaultKeyspaceID}, []uint32{1}) + + // Check the keyspace group 0 is split to 1 and 2. + kg0 := handlersutil.MustLoadKeyspaceGroupByID(re, leaderServer, 0) + kg1 := handlersutil.MustLoadKeyspaceGroupByID(re, leaderServer, 1) + kg2 := handlersutil.MustLoadKeyspaceGroupByID(re, leaderServer, 2) + re.Equal([]uint32{0}, kg0.Keyspaces) + re.Equal([]uint32{2}, kg1.Keyspaces) + re.Equal([]uint32{1}, kg2.Keyspaces) + re.False(kg0.IsSplitting()) + re.False(kg1.IsSplitting()) + re.False(kg2.IsSplitting()) + + re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/tso/fastGroupSplitPatroller")) + re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/keyspace/acceleratedAllocNodes")) +} + +func (suite *tsoKeyspaceGroupManagerTestSuite) TestTSOKeyspaceGroupMerge() { + re := suite.Require() + // Create the keyspace group 1 and 2 with keyspaces [111, 222] and [333]. + handlersutil.MustCreateKeyspaceGroup(re, suite.pdLeaderServer, &handlers.CreateKeyspaceGroupParams{ + KeyspaceGroups: []*endpoint.KeyspaceGroup{ + { + ID: 1, + UserKind: endpoint.Standard.String(), + Members: suite.tsoCluster.GetKeyspaceGroupMember(), + Keyspaces: []uint32{111, 222}, + }, + { + ID: 2, + UserKind: endpoint.Standard.String(), + Members: suite.tsoCluster.GetKeyspaceGroupMember(), + Keyspaces: []uint32{333}, + }, + }, + }) + // Get a TSO from the keyspace group 1. + var ( + ts pdpb.Timestamp + err error + ) + testutil.Eventually(re, func() bool { + ts, err = suite.requestTSO(re, 222, 1) + return err == nil && tsoutil.CompareTimestamp(&ts, &pdpb.Timestamp{}) > 0 + }) + ts.Physical += time.Hour.Milliseconds() + // Set the TSO of the keyspace group 1 to a large value. + err = suite.tsoCluster.GetPrimaryServer(222, 1).GetHandler().ResetTS(tsoutil.GenerateTS(&ts), false, true, 1) + re.NoError(err) + // Merge the keyspace group 1 and 2 to the default keyspace group. + handlersutil.MustMergeKeyspaceGroup(re, suite.pdLeaderServer, mcsutils.DefaultKeyspaceGroupID, &handlers.MergeKeyspaceGroupsParams{ + MergeList: []uint32{1, 2}, + }) + // Check the keyspace group 1 and 2 are merged to the default keyspace group. + kg := handlersutil.MustLoadKeyspaceGroupByID(re, suite.pdLeaderServer, mcsutils.DefaultKeyspaceGroupID) + re.Equal(mcsutils.DefaultKeyspaceGroupID, kg.ID) + for _, keyspaceID := range []uint32{111, 222, 333} { + re.Contains(kg.Keyspaces, keyspaceID) + } + re.True(kg.IsMergeTarget()) + // Check the merged TSO from the default keyspace group is greater than the TSO from the keyspace group 1. + var mergedTS pdpb.Timestamp + testutil.Eventually(re, func() bool { + mergedTS, err = suite.requestTSO(re, 333, mcsutils.DefaultKeyspaceGroupID) + if err != nil { + re.ErrorIs(err, errs.ErrKeyspaceGroupIsMerging) + } + return err == nil && tsoutil.CompareTimestamp(&mergedTS, &pdpb.Timestamp{}) > 0 + }, testutil.WithTickInterval(5*time.Second), testutil.WithWaitFor(time.Minute)) + re.Greater(tsoutil.CompareTimestamp(&mergedTS, &ts), 0) +} + +func (suite *tsoKeyspaceGroupManagerTestSuite) TestTSOKeyspaceGroupMergeClient() { + re := suite.Require() + // Create the keyspace group 1 with keyspaces [111, 222, 333]. + handlersutil.MustCreateKeyspaceGroup(re, suite.pdLeaderServer, &handlers.CreateKeyspaceGroupParams{ + KeyspaceGroups: []*endpoint.KeyspaceGroup{ + { + ID: 1, + UserKind: endpoint.Standard.String(), + Members: suite.tsoCluster.GetKeyspaceGroupMember(), + Keyspaces: []uint32{111, 222, 333}, + }, + }, + }) + kg1 := handlersutil.MustLoadKeyspaceGroupByID(re, suite.pdLeaderServer, 1) + re.Equal(uint32(1), kg1.ID) + re.Equal([]uint32{111, 222, 333}, kg1.Keyspaces) + re.False(kg1.IsMerging()) + // Request the TSO for keyspace 222 concurrently via client. + cancel := suite.dispatchClient(re, 222, 1) + // Merge the keyspace group 1 to the default keyspace group. + handlersutil.MustMergeKeyspaceGroup(re, suite.pdLeaderServer, mcsutils.DefaultKeyspaceGroupID, &handlers.MergeKeyspaceGroupsParams{ + MergeList: []uint32{1}, + }) + // Wait for the default keyspace group to finish the merge. + waitFinishMerge(re, suite.pdLeaderServer, mcsutils.DefaultKeyspaceGroupID, []uint32{111, 222, 333}) + // Stop the client. + cancel() +} + +func waitFinishMerge( + re *require.Assertions, + server *tests.TestServer, + mergeTargetID uint32, + keyspaces []uint32, +) { + testutil.Eventually(re, func() bool { + kg := handlersutil.MustLoadKeyspaceGroupByID(re, server, mergeTargetID) + re.Equal(mcsutils.DefaultKeyspaceGroupID, kg.ID) + for _, keyspaceID := range keyspaces { + re.Contains(kg.Keyspaces, keyspaceID) + } + return !kg.IsMergeTarget() + }) +} + +func (suite *tsoKeyspaceGroupManagerTestSuite) TestTSOKeyspaceGroupMergeBeforeInitTSO() { + re := suite.Require() + // Make sure the TSO of keyspace group 1 won't be initialized before it's merged. + re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/tso/failedToSaveTimestamp", `return(true)`)) + // Request the TSO for the default keyspace concurrently via client. + cancel := suite.dispatchClient(re, mcsutils.DefaultKeyspaceID, mcsutils.DefaultKeyspaceGroupID) + // Create the keyspace group 1 with keyspaces [111, 222, 333]. + handlersutil.MustCreateKeyspaceGroup(re, suite.pdLeaderServer, &handlers.CreateKeyspaceGroupParams{ + KeyspaceGroups: []*endpoint.KeyspaceGroup{ + { + ID: 1, + UserKind: endpoint.Standard.String(), + Members: suite.tsoCluster.GetKeyspaceGroupMember(), + Keyspaces: []uint32{111, 222, 333}, + }, + }, + }) + // Merge the keyspace group 1 to the default keyspace group. + handlersutil.MustMergeKeyspaceGroup(re, suite.pdLeaderServer, mcsutils.DefaultKeyspaceGroupID, &handlers.MergeKeyspaceGroupsParams{ + MergeList: []uint32{1}, + }) + // Wait for the default keyspace group to finish the merge. + waitFinishMerge(re, suite.pdLeaderServer, mcsutils.DefaultKeyspaceGroupID, []uint32{111, 222, 333}) + // Stop the client. + cancel() + re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/tso/failedToSaveTimestamp")) +} + +// See https://github.com/tikv/pd/issues/6748 +func TestGetTSOImmediately(t *testing.T) { + re := require.New(t) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/tso/fastPrimaryPriorityCheck", `return(true)`)) + re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/keyspace/acceleratedAllocNodes", `return(true)`)) + re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/tso/fastGroupSplitPatroller", `return(true)`)) + + // Init api server config but not start. + tc, err := tests.NewTestAPICluster(ctx, 1, func(conf *config.Config, _ string) { + conf.Keyspace.PreAlloc = []string{ + "keyspace_a", "keyspace_b", + } + }) + re.NoError(err) + pdAddr := tc.GetConfig().GetClientURL() + + // Start api server and tso server. + err = tc.RunInitialServers() + re.NoError(err) + defer tc.Destroy() + tc.WaitLeader() + leaderServer := tc.GetServer(tc.GetLeader()) + re.NoError(leaderServer.BootstrapCluster()) + + tsoCluster, err := tests.NewTestTSOCluster(ctx, 2, pdAddr) + re.NoError(err) + defer tsoCluster.Destroy() + tsoCluster.WaitForDefaultPrimaryServing(re) + + // First split keyspace group 0 to 1 with keyspace 2. + kgm := leaderServer.GetServer().GetKeyspaceGroupManager() + re.NotNil(kgm) + testutil.Eventually(re, func() bool { + err = kgm.SplitKeyspaceGroupByID(0, 1, []uint32{2}) + return err == nil + }) + + waitFinishSplit(re, leaderServer, 0, 1, []uint32{mcsutils.DefaultKeyspaceID, 1}, []uint32{2}) + + kg0 := handlersutil.MustLoadKeyspaceGroupByID(re, leaderServer, 0) + kg1 := handlersutil.MustLoadKeyspaceGroupByID(re, leaderServer, 1) + re.Equal([]uint32{0, 1}, kg0.Keyspaces) + re.Equal([]uint32{2}, kg1.Keyspaces) + re.False(kg0.IsSplitting()) + re.False(kg1.IsSplitting()) + + // Let group 0 and group 1 have different primary node. + kgm.SetPriorityForKeyspaceGroup(0, kg0.Members[0].Address, 100) + kgm.SetPriorityForKeyspaceGroup(1, kg1.Members[1].Address, 100) + testutil.Eventually(re, func() bool { + p0, _ := kgm.GetKeyspaceGroupPrimaryByID(0) + p1, _ := kgm.GetKeyspaceGroupPrimaryByID(1) + return p0 == kg0.Members[0].Address && p1 == kg1.Members[1].Address + }, testutil.WithWaitFor(5*time.Second), testutil.WithTickInterval(50*time.Millisecond)) + + apiCtx := pd.NewAPIContextV2("keyspace_b") // its keyspace id is 2. + cli, err := pd.NewClientWithAPIContext(ctx, apiCtx, []string{pdAddr}, pd.SecurityOption{}) + re.NoError(err) + _, _, err = cli.GetTS(ctx) + re.NoError(err) + cli.Close() + re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/tso/fastPrimaryPriorityCheck")) + re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/keyspace/acceleratedAllocNodes")) + re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/tso/fastGroupSplitPatroller")) +} + +func (suite *tsoKeyspaceGroupManagerTestSuite) TestKeyspaceGroupMergeIntoDefault() { + re := suite.Require() + re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/keyspace/acceleratedAllocNodes", `return(true)`)) + + var ( + keyspaceGroupNum = keyspace.MaxEtcdTxnOps + keyspaceGroups = make([]*endpoint.KeyspaceGroup, 0, keyspaceGroupNum) + keyspaces = make([]uint32, 0, keyspaceGroupNum) + ) + for i := 1; i <= keyspaceGroupNum; i++ { + keyspaceGroups = append(keyspaceGroups, &endpoint.KeyspaceGroup{ + ID: uint32(i), + UserKind: endpoint.UserKind(rand.Intn(int(endpoint.UserKindCount))).String(), + Keyspaces: []uint32{uint32(i)}, + }) + keyspaces = append(keyspaces, uint32(i)) + if len(keyspaceGroups) < keyspace.MaxEtcdTxnOps/2 && i != keyspaceGroupNum { + continue + } + handlersutil.MustCreateKeyspaceGroup(re, suite.pdLeaderServer, &handlers.CreateKeyspaceGroupParams{ + KeyspaceGroups: keyspaceGroups, + }) + keyspaceGroups = keyspaceGroups[:0] + } + // Check if all the keyspace groups are created. + groups := handlersutil.MustLoadKeyspaceGroups(re, suite.pdLeaderServer, "0", "0") + re.Len(groups, keyspaceGroupNum+1) + // Wait for all the keyspace groups to be served. + svr := suite.tsoCluster.WaitForDefaultPrimaryServing(re) + re.NotNil(svr) + svr = suite.tsoCluster.WaitForPrimaryServing(re, uint32(keyspaceGroupNum), uint32(keyspaceGroupNum)) + re.NotNil(svr) + // Merge all the keyspace groups into the default keyspace group. + handlersutil.MustMergeKeyspaceGroup(re, suite.pdLeaderServer, mcsutils.DefaultKeyspaceGroupID, &handlers.MergeKeyspaceGroupsParams{ + MergeAllIntoDefault: true, + }) + // Wait for all the keyspace groups to be merged. + waitFinishMerge(re, suite.pdLeaderServer, mcsutils.DefaultKeyspaceGroupID, keyspaces) + // Check if all the keyspace groups are merged. + groups = handlersutil.MustLoadKeyspaceGroups(re, suite.pdLeaderServer, "0", "0") + re.Len(groups, 1) + + re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/keyspace/acceleratedAllocNodes")) +} diff --git a/tests/integrations/mcs/tso/proxy_test.go b/tests/integrations/mcs/tso/proxy_test.go new file mode 100644 index 00000000000..625a702ad39 --- /dev/null +++ b/tests/integrations/mcs/tso/proxy_test.go @@ -0,0 +1,501 @@ +// Copyright 2023 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tso + +import ( + "context" + "fmt" + "math/rand" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/pingcap/failpoint" + "github.com/pingcap/kvproto/pkg/pdpb" + "github.com/pingcap/log" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + "github.com/tikv/pd/client/tsoutil" + "github.com/tikv/pd/pkg/utils/testutil" + "github.com/tikv/pd/tests" + "go.uber.org/zap" + "google.golang.org/grpc" +) + +type tsoProxyTestSuite struct { + suite.Suite + ctx context.Context + cancel context.CancelFunc + apiCluster *tests.TestCluster + apiLeader *tests.TestServer + backendEndpoints string + tsoCluster *tests.TestTSOCluster + defaultReq *pdpb.TsoRequest + streams []pdpb.PD_TsoClient + cleanupFuncs []testutil.CleanupFunc +} + +func TestTSOProxyTestSuite(t *testing.T) { + suite.Run(t, new(tsoProxyTestSuite)) +} + +func (s *tsoProxyTestSuite) SetupSuite() { + re := s.Require() + + var err error + s.ctx, s.cancel = context.WithCancel(context.Background()) + // Create an API cluster with 1 server + s.apiCluster, err = tests.NewTestAPICluster(s.ctx, 1) + re.NoError(err) + err = s.apiCluster.RunInitialServers() + re.NoError(err) + leaderName := s.apiCluster.WaitLeader() + s.apiLeader = s.apiCluster.GetServer(leaderName) + s.backendEndpoints = s.apiLeader.GetAddr() + s.NoError(s.apiLeader.BootstrapCluster()) + + // Create a TSO cluster with 2 servers + s.tsoCluster, err = tests.NewTestTSOCluster(s.ctx, 2, s.backendEndpoints) + re.NoError(err) + s.tsoCluster.WaitForDefaultPrimaryServing(re) + + s.defaultReq = &pdpb.TsoRequest{ + Header: &pdpb.RequestHeader{ClusterId: s.apiLeader.GetClusterID()}, + Count: 1, + } + + // Create some TSO client streams with different context. + s.streams, s.cleanupFuncs = createTSOStreams(s.ctx, re, s.backendEndpoints, 200) +} + +func (s *tsoProxyTestSuite) TearDownSuite() { + s.cleanupGRPCStreams(s.cleanupFuncs) + s.tsoCluster.Destroy() + s.apiCluster.Destroy() + s.cancel() +} + +// TestTSOProxyBasic tests the TSO Proxy's basic function to forward TSO requests to TSO microservice. +// It also verifies the correctness of the TSO Proxy's TSO response, such as the count of timestamps +// to retrieve in one TSO request and the monotonicity of the returned timestamps. +func (s *tsoProxyTestSuite) TestTSOProxyBasic() { + s.verifyTSOProxy(s.ctx, s.streams, s.cleanupFuncs, 100, true) +} + +// TestTSOProxyWithLargeCount tests while some grpc streams being cancelled and the others are still +// working, the TSO Proxy can still work correctly. +func (s *tsoProxyTestSuite) TestTSOProxyWorksWithCancellation() { + re := s.Require() + wg := &sync.WaitGroup{} + wg.Add(2) + go func() { + defer wg.Done() + go func() { + defer wg.Done() + for i := 0; i < 3; i++ { + streams, cleanupFuncs := createTSOStreams(s.ctx, re, s.backendEndpoints, 10) + for j := 0; j < 10; j++ { + s.verifyTSOProxy(s.ctx, streams, cleanupFuncs, 10, true) + } + s.cleanupGRPCStreams(cleanupFuncs) + } + }() + for i := 0; i < 10; i++ { + s.verifyTSOProxy(s.ctx, s.streams, s.cleanupFuncs, 10, true) + } + }() + wg.Wait() +} + +// TestTSOProxyStress tests the TSO Proxy can work correctly under the stress. gPRC and TSO failures are allowed, +// but the TSO Proxy should not panic, blocked or deadlocked, and if it returns a timestamp, it should be a valid +// timestamp monotonic increasing. After the stress, the TSO Proxy should still work correctly. +func TestTSOProxyStress(t *testing.T) { + s := new(tsoProxyTestSuite) + s.SetT(&testing.T{}) + s.SetupSuite() + re := s.Require() + + const ( + totalRounds = 4 + clientsIncr = 500 + // The graceful period for TSO Proxy to recover from gPRC and TSO failures. + recoverySLA = 5 * time.Second + ) + streams := make([]pdpb.PD_TsoClient, 0) + cleanupFuncs := make([]testutil.CleanupFunc, 0) + + // Start stress test for 90 seconds to avoid ci-test-job to timeout. + ctxTimeout, cancel := context.WithTimeout(s.ctx, 90*time.Second) + defer cancel() + + // Push load from many concurrent clients in multiple rounds and increase the #client each round. + for i := 0; i < totalRounds; i++ { + log.Info("start a new round of stress test", + zap.Int("round-id", i), zap.Int("clients-count", len(streams)+clientsIncr)) + streamsTemp, cleanupFuncsTemp := + createTSOStreams(s.ctx, re, s.backendEndpoints, clientsIncr) + streams = append(streams, streamsTemp...) + cleanupFuncs = append(cleanupFuncs, cleanupFuncsTemp...) + s.verifyTSOProxy(ctxTimeout, streams, cleanupFuncs, 50, false) + } + s.cleanupGRPCStreams(cleanupFuncs) + log.Info("the stress test completed.") + + // Verify the TSO Proxy can still work correctly after the stress. + testutil.Eventually(re, func() bool { + err := s.verifyTSOProxy(s.ctx, s.streams, s.cleanupFuncs, 1, false) + return err == nil + }, testutil.WithWaitFor(recoverySLA), testutil.WithTickInterval(500*time.Millisecond)) + + s.TearDownSuite() +} + +// TestTSOProxyClientsWithSameContext tests the TSO Proxy can work correctly while the grpc streams +// are created with the same context. +func (s *tsoProxyTestSuite) TestTSOProxyClientsWithSameContext() { + re := s.Require() + const clientCount = 1000 + cleanupFuncs := make([]testutil.CleanupFunc, clientCount) + streams := make([]pdpb.PD_TsoClient, clientCount) + + ctx, cancel := context.WithCancel(s.ctx) + defer cancel() + + for i := 0; i < clientCount; i++ { + conn, err := grpc.Dial(strings.TrimPrefix(s.backendEndpoints, "http://"), grpc.WithInsecure()) + re.NoError(err) + grpcPDClient := pdpb.NewPDClient(conn) + stream, err := grpcPDClient.Tso(ctx) + re.NoError(err) + streams[i] = stream + cleanupFunc := func() { + stream.CloseSend() + conn.Close() + } + cleanupFuncs[i] = cleanupFunc + } + + s.verifyTSOProxy(ctx, streams, cleanupFuncs, 100, true) + s.cleanupGRPCStreams(cleanupFuncs) +} + +// TestTSOProxyRecvFromClientTimeout tests the TSO Proxy can properly close the grpc stream on the server side +// when the client does not send any request to the server for a long time. +func (s *tsoProxyTestSuite) TestTSOProxyRecvFromClientTimeout() { + re := s.Require() + + // Enable the failpoint to make the TSO Proxy's grpc stream timeout on the server side to be 1 second. + re.NoError(failpoint.Enable("github.com/tikv/pd/server/tsoProxyRecvFromClientTimeout", `return(1)`)) + streams, cleanupFuncs := createTSOStreams(s.ctx, re, s.backendEndpoints, 1) + // Sleep 2 seconds to make the TSO Proxy's grpc stream timeout on the server side. + time.Sleep(2 * time.Second) + err := streams[0].Send(s.defaultReq) + re.Error(err) + s.cleanupGRPCStreams(cleanupFuncs) + re.NoError(failpoint.Disable("github.com/tikv/pd/server/tsoProxyRecvFromClientTimeout")) + + // Verify the streams with no fault injection can work correctly. + s.verifyTSOProxy(s.ctx, s.streams, s.cleanupFuncs, 1, true) +} + +// TestTSOProxyFailToSendToClient tests the TSO Proxy can properly close the grpc stream on the server side +// when it fails to send the response to the client. +func (s *tsoProxyTestSuite) TestTSOProxyFailToSendToClient() { + re := s.Require() + + // Enable the failpoint to make the TSO Proxy's grpc stream timeout on the server side to be 1 second. + re.NoError(failpoint.Enable("github.com/tikv/pd/server/tsoProxyFailToSendToClient", `return(true)`)) + streams, cleanupFuncs := createTSOStreams(s.ctx, re, s.backendEndpoints, 1) + err := streams[0].Send(s.defaultReq) + re.NoError(err) + _, err = streams[0].Recv() + re.Error(err) + s.cleanupGRPCStreams(cleanupFuncs) + re.NoError(failpoint.Disable("github.com/tikv/pd/server/tsoProxyFailToSendToClient")) + + s.verifyTSOProxy(s.ctx, s.streams, s.cleanupFuncs, 1, true) +} + +// TestTSOProxySendToTSOTimeout tests the TSO Proxy can properly close the grpc stream on the server side +// when it sends the request to the TSO service and encounters timeout. +func (s *tsoProxyTestSuite) TestTSOProxySendToTSOTimeout() { + re := s.Require() + + // Enable the failpoint to make the TSO Proxy's grpc stream timeout on the server side to be 1 second. + re.NoError(failpoint.Enable("github.com/tikv/pd/server/tsoProxySendToTSOTimeout", `return(true)`)) + streams, cleanupFuncs := createTSOStreams(s.ctx, re, s.backendEndpoints, 1) + err := streams[0].Send(s.defaultReq) + re.NoError(err) + _, err = streams[0].Recv() + re.Error(err) + s.cleanupGRPCStreams(cleanupFuncs) + re.NoError(failpoint.Disable("github.com/tikv/pd/server/tsoProxySendToTSOTimeout")) + + s.verifyTSOProxy(s.ctx, s.streams, s.cleanupFuncs, 1, true) +} + +// TestTSOProxyRecvFromTSOTimeout tests the TSO Proxy can properly close the grpc stream on the server side +// when it receives the response from the TSO service and encounters timeout. +func (s *tsoProxyTestSuite) TestTSOProxyRecvFromTSOTimeout() { + re := s.Require() + + // Enable the failpoint to make the TSO Proxy's grpc stream timeout on the server side to be 1 second. + re.NoError(failpoint.Enable("github.com/tikv/pd/server/tsoProxyRecvFromTSOTimeout", `return(true)`)) + streams, cleanupFuncs := createTSOStreams(s.ctx, re, s.backendEndpoints, 1) + err := streams[0].Send(s.defaultReq) + re.NoError(err) + _, err = streams[0].Recv() + re.Error(err) + s.cleanupGRPCStreams(cleanupFuncs) + re.NoError(failpoint.Disable("github.com/tikv/pd/server/tsoProxyRecvFromTSOTimeout")) + + s.verifyTSOProxy(s.ctx, s.streams, s.cleanupFuncs, 1, true) +} + +func (s *tsoProxyTestSuite) cleanupGRPCStreams(cleanupFuncs []testutil.CleanupFunc) { + for i := 0; i < len(cleanupFuncs); i++ { + if cleanupFuncs[i] != nil { + cleanupFuncs[i]() + cleanupFuncs[i] = nil + } + } +} + +func (s *tsoProxyTestSuite) cleanupGRPCStream( + streams []pdpb.PD_TsoClient, cleanupFuncs []testutil.CleanupFunc, index int, +) { + if cleanupFuncs[index] != nil { + cleanupFuncs[index]() + cleanupFuncs[index] = nil + } + if streams[index] != nil { + streams[index] = nil + } +} + +// verifyTSOProxy verifies the TSO Proxy can work correctly. +// +// 1. If mustReliable == true +// no gPRC or TSO failures, the TSO Proxy should return a valid timestamp monotonic increasing. +// +// 2. If mustReliable == false +// gPRC and TSO failures are allowed, but the TSO Proxy should not panic, blocked or deadlocked. +// If it returns a timestamp, it should be a valid timestamp monotonic increasing. +func (s *tsoProxyTestSuite) verifyTSOProxy( + ctx context.Context, streams []pdpb.PD_TsoClient, + cleanupFuncs []testutil.CleanupFunc, requestsPerClient int, mustReliable bool, +) error { + re := s.Require() + reqs := s.generateRequests(requestsPerClient) + + var respErr atomic.Value + + wg := &sync.WaitGroup{} + for i := 0; i < len(streams); i++ { + if streams[i] == nil { + continue + } + wg.Add(1) + go func(i int) { + defer wg.Done() + lastPhysical, lastLogical := int64(0), int64(0) + for j := 0; j < requestsPerClient; j++ { + select { + case <-ctx.Done(): + respErr.Store(ctx.Err()) + s.cleanupGRPCStream(streams, cleanupFuncs, i) + return + default: + } + + req := reqs[rand.Intn(requestsPerClient)] + err := streams[i].Send(req) + if err != nil && !mustReliable { + respErr.Store(err) + s.cleanupGRPCStream(streams, cleanupFuncs, i) + return + } + re.NoError(err) + resp, err := streams[i].Recv() + if err != nil && !mustReliable { + respErr.Store(err) + s.cleanupGRPCStream(streams, cleanupFuncs, i) + return + } + re.NoError(err) + re.Equal(req.GetCount(), resp.GetCount()) + ts := resp.GetTimestamp() + count := int64(resp.GetCount()) + physical, largestLogic, suffixBits := ts.GetPhysical(), ts.GetLogical(), ts.GetSuffixBits() + firstLogical := tsoutil.AddLogical(largestLogic, -count+1, suffixBits) + re.False(tsoutil.TSLessEqual(physical, firstLogical, lastPhysical, lastLogical)) + } + }(i) + } + wg.Wait() + + if val := respErr.Load(); val != nil { + return val.(error) + } + return nil +} + +func (s *tsoProxyTestSuite) generateRequests(requestsPerClient int) []*pdpb.TsoRequest { + reqs := make([]*pdpb.TsoRequest, requestsPerClient) + for i := 0; i < requestsPerClient; i++ { + reqs[i] = &pdpb.TsoRequest{ + Header: &pdpb.RequestHeader{ClusterId: s.apiLeader.GetClusterID()}, + Count: uint32(i) + 1, // Make sure the count is positive. + } + } + return reqs +} + +// createTSOStreams creates multiple TSO client streams, and each stream uses a different gRPC connection +// to simulate multiple clients. +func createTSOStreams( + ctx context.Context, re *require.Assertions, + backendEndpoints string, clientCount int, +) ([]pdpb.PD_TsoClient, []testutil.CleanupFunc) { + cleanupFuncs := make([]testutil.CleanupFunc, clientCount) + streams := make([]pdpb.PD_TsoClient, clientCount) + + for i := 0; i < clientCount; i++ { + conn, err := grpc.Dial(strings.TrimPrefix(backendEndpoints, "http://"), grpc.WithInsecure()) + re.NoError(err) + grpcPDClient := pdpb.NewPDClient(conn) + cctx, cancel := context.WithCancel(ctx) + stream, err := grpcPDClient.Tso(cctx) + re.NoError(err) + streams[i] = stream + cleanupFunc := func() { + stream.CloseSend() + cancel() + conn.Close() + } + cleanupFuncs[i] = cleanupFunc + } + + return streams, cleanupFuncs +} + +func tsoProxy( + tsoReq *pdpb.TsoRequest, streams []pdpb.PD_TsoClient, + concurrentClient bool, requestsPerClient int, +) error { + if concurrentClient { + wg := &sync.WaitGroup{} + errsReturned := make([]error, len(streams)) + for index, stream := range streams { + streamCopy := stream + wg.Add(1) + go func(index int, streamCopy pdpb.PD_TsoClient) { + defer wg.Done() + for i := 0; i < requestsPerClient; i++ { + if err := streamCopy.Send(tsoReq); err != nil { + errsReturned[index] = err + return + } + if _, err := streamCopy.Recv(); err != nil { + return + } + } + }(index, streamCopy) + } + wg.Wait() + for _, err := range errsReturned { + if err != nil { + return err + } + } + } else { + for _, stream := range streams { + for i := 0; i < requestsPerClient; i++ { + if err := stream.Send(tsoReq); err != nil { + return err + } + if _, err := stream.Recv(); err != nil { + return err + } + } + } + } + return nil +} + +var benmarkTSOProxyTable = []struct { + concurrentClient bool + requestsPerClient int +}{ + {true, 2}, + {true, 10}, + {true, 100}, + {false, 2}, + {false, 10}, + {false, 100}, +} + +// BenchmarkTSOProxy10Clients benchmarks TSO proxy performance with 10 clients. +func BenchmarkTSOProxy10Clients(b *testing.B) { + benchmarkTSOProxyNClients(10, b) +} + +// BenchmarkTSOProxy100Clients benchmarks TSO proxy performance with 100 clients. +func BenchmarkTSOProxy100Clients(b *testing.B) { + benchmarkTSOProxyNClients(100, b) +} + +// BenchmarkTSOProxy1000Clients benchmarks TSO proxy performance with 1000 clients. +func BenchmarkTSOProxy1000Clients(b *testing.B) { + benchmarkTSOProxyNClients(1000, b) +} + +// benchmarkTSOProxyNClients benchmarks TSO proxy performance. +func benchmarkTSOProxyNClients(clientCount int, b *testing.B) { + suite := new(tsoProxyTestSuite) + suite.SetT(&testing.T{}) + suite.SetupSuite() + re := suite.Require() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + streams, cleanupFuncs := createTSOStreams(ctx, re, suite.backendEndpoints, clientCount) + + // Benchmark TSO proxy + b.ResetTimer() + for _, t := range benmarkTSOProxyTable { + var builder strings.Builder + if t.concurrentClient { + builder.WriteString("ConcurrentClients_") + } else { + builder.WriteString("SequentialClients_") + } + b.Run(fmt.Sprintf("%s_%dReqsPerClient", builder.String(), t.requestsPerClient), func(b *testing.B) { + for i := 0; i < b.N; i++ { + err := tsoProxy(suite.defaultReq, streams, t.concurrentClient, t.requestsPerClient) + re.NoError(err) + } + }) + } + b.StopTimer() + + suite.cleanupGRPCStreams(cleanupFuncs) + + suite.TearDownSuite() +} diff --git a/tests/integrations/mcs/tso/server_test.go b/tests/integrations/mcs/tso/server_test.go new file mode 100644 index 00000000000..49eab333066 --- /dev/null +++ b/tests/integrations/mcs/tso/server_test.go @@ -0,0 +1,617 @@ +// Copyright 2023 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tso + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "strconv" + "strings" + "testing" + "time" + + "github.com/pingcap/failpoint" + "github.com/pingcap/kvproto/pkg/metapb" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + pd "github.com/tikv/pd/client" + "github.com/tikv/pd/pkg/core" + "github.com/tikv/pd/pkg/mcs/discovery" + tso "github.com/tikv/pd/pkg/mcs/tso/server" + tsoapi "github.com/tikv/pd/pkg/mcs/tso/server/apis/v1" + "github.com/tikv/pd/pkg/mcs/utils" + "github.com/tikv/pd/pkg/storage/endpoint" + "github.com/tikv/pd/pkg/utils/etcdutil" + "github.com/tikv/pd/pkg/utils/tempurl" + "github.com/tikv/pd/pkg/utils/testutil" + "github.com/tikv/pd/pkg/utils/tsoutil" + "github.com/tikv/pd/tests" + "github.com/tikv/pd/tests/integrations/mcs" + "go.etcd.io/etcd/clientv3" + "go.uber.org/goleak" + "google.golang.org/grpc" +) + +func TestMain(m *testing.M) { + goleak.VerifyTestMain(m, testutil.LeakOptions...) +} + +type tsoServerTestSuite struct { + suite.Suite + ctx context.Context + cancel context.CancelFunc + cluster *tests.TestCluster + pdLeader *tests.TestServer + backendEndpoints string +} + +func TestTSOServerTestSuite(t *testing.T) { + suite.Run(t, new(tsoServerTestSuite)) +} + +func (suite *tsoServerTestSuite) SetupSuite() { + var err error + re := suite.Require() + + suite.ctx, suite.cancel = context.WithCancel(context.Background()) + suite.cluster, err = tests.NewTestAPICluster(suite.ctx, 1) + re.NoError(err) + + err = suite.cluster.RunInitialServers() + re.NoError(err) + + leaderName := suite.cluster.WaitLeader() + suite.pdLeader = suite.cluster.GetServer(leaderName) + suite.backendEndpoints = suite.pdLeader.GetAddr() + suite.NoError(suite.pdLeader.BootstrapCluster()) +} + +func (suite *tsoServerTestSuite) TearDownSuite() { + suite.cluster.Destroy() + suite.cancel() +} + +func (suite *tsoServerTestSuite) TestTSOServerStartAndStopNormally() { + defer func() { + if r := recover(); r != nil { + fmt.Println("Recovered from an unexpected panic", r) + suite.T().Errorf("Expected no panic, but something bad occurred with") + } + }() + + re := suite.Require() + s, cleanup := tests.StartSingleTSOTestServer(suite.ctx, re, suite.backendEndpoints, tempurl.Alloc()) + + defer cleanup() + testutil.Eventually(re, func() bool { + return s.IsServing() + }, testutil.WithWaitFor(5*time.Second), testutil.WithTickInterval(50*time.Millisecond)) + + // Test registered GRPC Service + cc, err := grpc.DialContext(suite.ctx, s.GetAddr(), grpc.WithInsecure()) + re.NoError(err) + cc.Close() + url := s.GetAddr() + tsoapi.APIPathPrefix + { + resetJSON := `{"tso":"121312", "force-use-larger":true}` + re.NoError(err) + resp, err := http.Post(url+"/admin/reset-ts", "application/json", strings.NewReader(resetJSON)) + re.NoError(err) + defer resp.Body.Close() + re.Equal(http.StatusOK, resp.StatusCode) + } + { + resetJSON := `{}` + re.NoError(err) + resp, err := http.Post(url+"/admin/reset-ts", "application/json", strings.NewReader(resetJSON)) + re.NoError(err) + defer resp.Body.Close() + re.Equal(http.StatusBadRequest, resp.StatusCode) + } +} + +func (suite *tsoServerTestSuite) TestParticipantStartWithAdvertiseListenAddr() { + re := suite.Require() + + cfg := tso.NewConfig() + cfg.BackendEndpoints = suite.backendEndpoints + cfg.ListenAddr = tempurl.Alloc() + cfg.AdvertiseListenAddr = tempurl.Alloc() + cfg, err := tso.GenerateConfig(cfg) + re.NoError(err) + + // Setup the logger. + err = tests.InitLogger(cfg) + re.NoError(err) + + s, cleanup, err := tests.NewTSOTestServer(suite.ctx, cfg) + re.NoError(err) + defer cleanup() + testutil.Eventually(re, func() bool { + return s.IsServing() + }, testutil.WithWaitFor(5*time.Second), testutil.WithTickInterval(50*time.Millisecond)) + + member, err := s.GetMember(utils.DefaultKeyspaceID, utils.DefaultKeyspaceGroupID) + re.NoError(err) + re.Equal(fmt.Sprintf("%s-%05d", cfg.AdvertiseListenAddr, utils.DefaultKeyspaceGroupID), member.Name()) +} + +func TestTSOPath(t *testing.T) { + re := require.New(t) + checkTSOPath(re, true /*isAPIServiceMode*/) + re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/mcs/tso/server/skipWaitAPIServiceReady", "return(true)")) + checkTSOPath(re, false /*isAPIServiceMode*/) + re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/mcs/tso/server/skipWaitAPIServiceReady")) +} + +func checkTSOPath(re *require.Assertions, isAPIServiceMode bool) { + var ( + cluster *tests.TestCluster + err error + ) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + if isAPIServiceMode { + cluster, err = tests.NewTestAPICluster(ctx, 1) + } else { + cluster, err = tests.NewTestCluster(ctx, 1) + } + re.NoError(err) + defer cluster.Destroy() + err = cluster.RunInitialServers() + re.NoError(err) + leaderName := cluster.WaitLeader() + pdLeader := cluster.GetServer(leaderName) + re.NoError(pdLeader.BootstrapCluster()) + backendEndpoints := pdLeader.GetAddr() + client := pdLeader.GetEtcdClient() + if isAPIServiceMode { + re.Equal(0, getEtcdTimestampKeyNum(re, client)) + } else { + re.Equal(1, getEtcdTimestampKeyNum(re, client)) + } + + _, cleanup := tests.StartSingleTSOTestServer(ctx, re, backendEndpoints, tempurl.Alloc()) + defer cleanup() + + cli := mcs.SetupClientWithAPIContext(ctx, re, pd.NewAPIContextV2(""), []string{backendEndpoints}) + physical, logical, err := cli.GetTS(ctx) + re.NoError(err) + ts := tsoutil.ComposeTS(physical, logical) + re.NotEmpty(ts) + // After we request the tso server, etcd still has only one key related to the timestamp. + re.Equal(1, getEtcdTimestampKeyNum(re, client)) +} + +func getEtcdTimestampKeyNum(re *require.Assertions, client *clientv3.Client) int { + resp, err := etcdutil.EtcdKVGet(client, "/", clientv3.WithPrefix()) + re.NoError(err) + var count int + for _, kv := range resp.Kvs { + key := strings.TrimSpace(string(kv.Key)) + if !strings.HasSuffix(key, endpoint.TimestampKey) { + continue + } + count++ + } + return count +} + +func TestWaitAPIServiceReady(t *testing.T) { + re := require.New(t) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + startCluster := func(isAPIServiceMode bool) (cluster *tests.TestCluster, backendEndpoints string) { + var err error + if isAPIServiceMode { + cluster, err = tests.NewTestAPICluster(ctx, 1) + } else { + cluster, err = tests.NewTestCluster(ctx, 1) + } + re.NoError(err) + err = cluster.RunInitialServers() + re.NoError(err) + leaderName := cluster.WaitLeader() + pdLeader := cluster.GetServer(leaderName) + return cluster, pdLeader.GetAddr() + } + + // tso server cannot be started because the pd server is not ready as api service. + cluster, backendEndpoints := startCluster(false /*isAPIServiceMode*/) + sctx, scancel := context.WithTimeout(ctx, time.Second*10) + defer scancel() + s, _, err := tests.StartSingleTSOTestServerWithoutCheck(sctx, re, backendEndpoints, tempurl.Alloc()) + re.Error(err) + re.Nil(s) + cluster.Destroy() + + // tso server can be started because the pd server is ready as api service. + cluster, backendEndpoints = startCluster(true /*isAPIServiceMode*/) + sctx, scancel = context.WithTimeout(ctx, time.Second*10) + defer scancel() + s, cleanup, err := tests.StartSingleTSOTestServerWithoutCheck(sctx, re, backendEndpoints, tempurl.Alloc()) + re.NoError(err) + defer cluster.Destroy() + defer cleanup() + + for i := 0; i < 12; i++ { + select { + case <-time.After(time.Second): + case <-sctx.Done(): + return + } + if s != nil && s.IsServing() { + break + } + } +} + +type APIServerForwardTestSuite struct { + suite.Suite + ctx context.Context + cancel context.CancelFunc + cluster *tests.TestCluster + pdLeader *tests.TestServer + backendEndpoints string + pdClient pd.Client +} + +func TestAPIServerForwardTestSuite(t *testing.T) { + suite.Run(t, new(APIServerForwardTestSuite)) +} + +func (suite *APIServerForwardTestSuite) SetupTest() { + var err error + re := suite.Require() + suite.ctx, suite.cancel = context.WithCancel(context.Background()) + suite.cluster, err = tests.NewTestAPICluster(suite.ctx, 3) + re.NoError(err) + + err = suite.cluster.RunInitialServers() + re.NoError(err) + + leaderName := suite.cluster.WaitLeader() + suite.pdLeader = suite.cluster.GetServer(leaderName) + suite.backendEndpoints = suite.pdLeader.GetAddr() + suite.NoError(suite.pdLeader.BootstrapCluster()) + suite.addRegions() + + suite.NoError(failpoint.Enable("github.com/tikv/pd/client/usePDServiceMode", "return(true)")) + suite.pdClient, err = pd.NewClientWithContext(context.Background(), + []string{suite.backendEndpoints}, pd.SecurityOption{}, pd.WithMaxErrorRetry(1)) + suite.NoError(err) +} + +func (suite *APIServerForwardTestSuite) TearDownTest() { + suite.pdClient.Close() + + etcdClient := suite.pdLeader.GetEtcdClient() + clusterID := strconv.FormatUint(suite.pdLeader.GetClusterID(), 10) + endpoints, err := discovery.Discover(etcdClient, clusterID, utils.TSOServiceName) + suite.NoError(err) + if len(endpoints) != 0 { + endpoints, err = discovery.Discover(etcdClient, clusterID, utils.TSOServiceName) + suite.NoError(err) + suite.Empty(endpoints) + } + suite.cluster.Destroy() + suite.cancel() + suite.NoError(failpoint.Disable("github.com/tikv/pd/client/usePDServiceMode")) +} + +func (suite *APIServerForwardTestSuite) TestForwardTSORelated() { + // Unable to use the tso-related interface without tso server + suite.checkUnavailableTSO() + tc, err := tests.NewTestTSOCluster(suite.ctx, 1, suite.backendEndpoints) + suite.NoError(err) + defer tc.Destroy() + tc.WaitForDefaultPrimaryServing(suite.Require()) + suite.checkAvailableTSO() +} + +func (suite *APIServerForwardTestSuite) TestForwardTSOWhenPrimaryChanged() { + re := suite.Require() + + tc, err := tests.NewTestTSOCluster(suite.ctx, 2, suite.backendEndpoints) + re.NoError(err) + defer tc.Destroy() + tc.WaitForDefaultPrimaryServing(re) + + // can use the tso-related interface with old primary + oldPrimary, exist := suite.pdLeader.GetServer().GetServicePrimaryAddr(suite.ctx, utils.TSOServiceName) + re.True(exist) + suite.checkAvailableTSO() + + // can use the tso-related interface with new primary + tc.DestroyServer(oldPrimary) + time.Sleep(time.Duration(utils.DefaultLeaderLease) * time.Second) // wait for leader lease timeout + tc.WaitForDefaultPrimaryServing(re) + primary, exist := suite.pdLeader.GetServer().GetServicePrimaryAddr(suite.ctx, utils.TSOServiceName) + re.True(exist) + re.NotEqual(oldPrimary, primary) + suite.checkAvailableTSO() + + // can use the tso-related interface with old primary again + tc.AddServer(oldPrimary) + suite.checkAvailableTSO() + for addr := range tc.GetServers() { + if addr != oldPrimary { + tc.DestroyServer(addr) + } + } + tc.WaitForDefaultPrimaryServing(re) + time.Sleep(time.Duration(utils.DefaultLeaderLease) * time.Second) // wait for leader lease timeout + primary, exist = suite.pdLeader.GetServer().GetServicePrimaryAddr(suite.ctx, utils.TSOServiceName) + re.True(exist) + re.Equal(oldPrimary, primary) + suite.checkAvailableTSO() +} + +func (suite *APIServerForwardTestSuite) TestResignTSOPrimaryForward() { + // TODO: test random kill primary with 3 nodes + re := suite.Require() + + tc, err := tests.NewTestTSOCluster(suite.ctx, 2, suite.backendEndpoints) + re.NoError(err) + defer tc.Destroy() + tc.WaitForDefaultPrimaryServing(re) + + for j := 0; j < 10; j++ { + tc.ResignPrimary(utils.DefaultKeyspaceID, utils.DefaultKeyspaceGroupID) + tc.WaitForDefaultPrimaryServing(re) + var err error + for i := 0; i < 3; i++ { // try 3 times + _, _, err = suite.pdClient.GetTS(suite.ctx) + if err == nil { + break + } + time.Sleep(100 * time.Millisecond) + } + suite.NoError(err) + suite.checkAvailableTSO() + } +} + +func (suite *APIServerForwardTestSuite) TestResignAPIPrimaryForward() { + re := suite.Require() + + tc, err := tests.NewTestTSOCluster(suite.ctx, 2, suite.backendEndpoints) + re.NoError(err) + defer tc.Destroy() + tc.WaitForDefaultPrimaryServing(re) + + for j := 0; j < 10; j++ { + suite.pdLeader.ResignLeader() + suite.pdLeader = suite.cluster.GetServer(suite.cluster.WaitLeader()) + suite.backendEndpoints = suite.pdLeader.GetAddr() + _, _, err = suite.pdClient.GetTS(suite.ctx) + suite.NoError(err) + } +} + +func (suite *APIServerForwardTestSuite) TestForwardTSOUnexpectedToFollower1() { + suite.checkForwardTSOUnexpectedToFollower(func() { + // unary call will retry internally + // try to update gc safe point + min, err := suite.pdClient.UpdateServiceGCSafePoint(context.Background(), "a", 1000, 1) + suite.NoError(err) + suite.Equal(uint64(0), min) + }) +} + +func (suite *APIServerForwardTestSuite) TestForwardTSOUnexpectedToFollower2() { + suite.checkForwardTSOUnexpectedToFollower(func() { + // unary call will retry internally + // try to set external ts + ts, err := suite.pdClient.GetExternalTimestamp(suite.ctx) + suite.NoError(err) + err = suite.pdClient.SetExternalTimestamp(suite.ctx, ts+1) + suite.NoError(err) + }) +} + +func (suite *APIServerForwardTestSuite) TestForwardTSOUnexpectedToFollower3() { + suite.checkForwardTSOUnexpectedToFollower(func() { + _, _, err := suite.pdClient.GetTS(suite.ctx) + suite.Error(err) + }) +} + +func (suite *APIServerForwardTestSuite) checkForwardTSOUnexpectedToFollower(checkTSO func()) { + re := suite.Require() + tc, err := tests.NewTestTSOCluster(suite.ctx, 2, suite.backendEndpoints) + re.NoError(err) + tc.WaitForDefaultPrimaryServing(re) + + // get follower's address + servers := tc.GetServers() + oldPrimary := tc.GetPrimaryServer(utils.DefaultKeyspaceID, utils.DefaultKeyspaceGroupID).GetAddr() + var follower string + for addr := range servers { + if addr != oldPrimary { + follower = addr + break + } + } + re.NotEmpty(follower) + + // write follower's address to cache to simulate cache is not updated. + suite.pdLeader.GetServer().SetServicePrimaryAddr(utils.TSOServiceName, follower) + errorAddr, ok := suite.pdLeader.GetServer().GetServicePrimaryAddr(suite.ctx, utils.TSOServiceName) + suite.True(ok) + suite.Equal(follower, errorAddr) + + // test tso request + checkTSO() + + // test tso request will success after cache is updated + suite.checkAvailableTSO() + newPrimary, exist2 := suite.pdLeader.GetServer().GetServicePrimaryAddr(suite.ctx, utils.TSOServiceName) + suite.True(exist2) + suite.NotEqual(errorAddr, newPrimary) + suite.Equal(oldPrimary, newPrimary) + tc.Destroy() +} + +func (suite *APIServerForwardTestSuite) addRegions() { + leader := suite.cluster.GetServer(suite.cluster.WaitLeader()) + rc := leader.GetServer().GetRaftCluster() + for i := 0; i < 3; i++ { + region := &metapb.Region{ + Id: uint64(i*4 + 1), + Peers: []*metapb.Peer{{Id: uint64(i*4 + 2), StoreId: uint64(i*4 + 3)}}, + StartKey: []byte{byte(i)}, + EndKey: []byte{byte(i + 1)}, + } + rc.HandleRegionHeartbeat(core.NewRegionInfo(region, region.Peers[0])) + } +} + +func (suite *APIServerForwardTestSuite) checkUnavailableTSO() { + _, _, err := suite.pdClient.GetTS(suite.ctx) + suite.Error(err) + // try to update gc safe point + _, err = suite.pdClient.UpdateServiceGCSafePoint(suite.ctx, "a", 1000, 1) + suite.Error(err) + // try to set external ts + err = suite.pdClient.SetExternalTimestamp(suite.ctx, 1000) + suite.Error(err) +} + +func (suite *APIServerForwardTestSuite) checkAvailableTSO() { + re := suite.Require() + mcs.WaitForTSOServiceAvailable(suite.ctx, re, suite.pdClient) + // try to get ts + _, _, err := suite.pdClient.GetTS(suite.ctx) + re.NoError(err) + // try to update gc safe point + min, err := suite.pdClient.UpdateServiceGCSafePoint(context.Background(), "a", 1000, 1) + re.NoError(err) + re.Equal(uint64(0), min) + // try to set external ts + ts, err := suite.pdClient.GetExternalTimestamp(suite.ctx) + re.NoError(err) + err = suite.pdClient.SetExternalTimestamp(suite.ctx, ts+1) + re.NoError(err) +} + +type CommonTestSuite struct { + suite.Suite + ctx context.Context + cancel context.CancelFunc + cluster *tests.TestCluster + tsoCluster *tests.TestTSOCluster + pdLeader *tests.TestServer + // tsoDefaultPrimaryServer is the primary server of the default keyspace group + tsoDefaultPrimaryServer *tso.Server + backendEndpoints string +} + +func TestCommonTestSuite(t *testing.T) { + suite.Run(t, new(CommonTestSuite)) +} + +func (suite *CommonTestSuite) SetupSuite() { + var err error + re := suite.Require() + suite.ctx, suite.cancel = context.WithCancel(context.Background()) + suite.cluster, err = tests.NewTestAPICluster(suite.ctx, 1) + re.NoError(err) + + err = suite.cluster.RunInitialServers() + re.NoError(err) + + leaderName := suite.cluster.WaitLeader() + suite.pdLeader = suite.cluster.GetServer(leaderName) + suite.backendEndpoints = suite.pdLeader.GetAddr() + suite.NoError(suite.pdLeader.BootstrapCluster()) + + suite.tsoCluster, err = tests.NewTestTSOCluster(suite.ctx, 1, suite.backendEndpoints) + suite.NoError(err) + suite.tsoCluster.WaitForDefaultPrimaryServing(re) + suite.tsoDefaultPrimaryServer = suite.tsoCluster.GetPrimaryServer(utils.DefaultKeyspaceID, utils.DefaultKeyspaceGroupID) +} + +func (suite *CommonTestSuite) TearDownSuite() { + suite.tsoCluster.Destroy() + etcdClient := suite.pdLeader.GetEtcdClient() + clusterID := strconv.FormatUint(suite.pdLeader.GetClusterID(), 10) + endpoints, err := discovery.Discover(etcdClient, clusterID, utils.TSOServiceName) + suite.NoError(err) + if len(endpoints) != 0 { + endpoints, err = discovery.Discover(etcdClient, clusterID, utils.TSOServiceName) + suite.NoError(err) + suite.Empty(endpoints) + } + suite.cluster.Destroy() + suite.cancel() +} + +func (suite *CommonTestSuite) TestAdvertiseAddr() { + re := suite.Require() + + conf := suite.tsoDefaultPrimaryServer.GetConfig() + re.Equal(conf.GetListenAddr(), conf.GetAdvertiseListenAddr()) +} + +func (suite *CommonTestSuite) TestMetrics() { + re := suite.Require() + + resp, err := http.Get(suite.tsoDefaultPrimaryServer.GetConfig().GetAdvertiseListenAddr() + "/metrics") + re.NoError(err) + defer resp.Body.Close() + re.Equal(http.StatusOK, resp.StatusCode) + respBytes, err := io.ReadAll(resp.Body) + re.NoError(err) + re.Contains(string(respBytes), "tso_server_info") +} + +func (suite *CommonTestSuite) TestBootstrapDefaultKeyspaceGroup() { + re := suite.Require() + + // check the default keyspace group + check := func() { + resp, err := http.Get(suite.pdLeader.GetServer().GetConfig().AdvertiseClientUrls + "/pd/api/v2/tso/keyspace-groups") + re.NoError(err) + defer resp.Body.Close() + re.Equal(http.StatusOK, resp.StatusCode) + respString, err := io.ReadAll(resp.Body) + re.NoError(err) + var kgs []*endpoint.KeyspaceGroup + re.NoError(json.Unmarshal(respString, &kgs)) + re.Len(kgs, 1) + re.Equal(utils.DefaultKeyspaceGroupID, kgs[0].ID) + re.Equal(endpoint.Basic.String(), kgs[0].UserKind) + re.Empty(kgs[0].SplitState) + re.Empty(kgs[0].Members) + re.Empty(kgs[0].KeyspaceLookupTable) + } + check() + + s, err := suite.cluster.JoinAPIServer(suite.ctx) + re.NoError(err) + re.NoError(s.Run()) + + // transfer leader to the new server + suite.pdLeader.ResignLeader() + suite.pdLeader = suite.cluster.GetServer(suite.cluster.WaitLeader()) + check() + suite.pdLeader.ResignLeader() + suite.pdLeader = suite.cluster.GetServer(suite.cluster.WaitLeader()) +} diff --git a/tests/mcs/Makefile b/tests/integrations/tso/Makefile similarity index 53% rename from tests/mcs/Makefile rename to tests/integrations/tso/Makefile index 86dc5f58bd2..e353f686fe7 100644 --- a/tests/mcs/Makefile +++ b/tests/integrations/tso/Makefile @@ -12,34 +12,38 @@ # See the License for the specific language governing permissions and # limitations under the License. -GO_TOOLS_BIN_PATH := $(shell pwd)/../../.tools/bin +ROOT_PATH := ../../.. +GO_TOOLS_BIN_PATH := $(ROOT_PATH)/.tools/bin PATH := $(GO_TOOLS_BIN_PATH):$(PATH) SHELL := env PATH='$(PATH)' GOBIN='$(GO_TOOLS_BIN_PATH)' $(shell which bash) static: install-tools + @ echo "gofmt ..." @ gofmt -s -l -d . 2>&1 | awk '{ print } END { if (NR > 0) { exit 1 } }' - @ golangci-lint run ./... - @ revive -formatter friendly -config ../../revive.toml . + @ echo "golangci-lint ..." + @ golangci-lint run -c $(ROOT_PATH)/.golangci.yml --verbose ./... --allow-parallel-runners + @ echo "revive ..." + @ revive -formatter friendly -config $(ROOT_PATH)/revive.toml ./... tidy: @ go mod tidy git diff go.mod go.sum | cat git diff --quiet go.mod go.sum -test: enable-codegen - CGO_ENABLED=1 go test ./... -tags deadlock -race -cover || { $(MAKE) disable-codegen && exit 1; } - $(MAKE) disable-codegen +test: failpoint-enable + CGO_ENABLED=1 go test ./... -v -tags deadlock -race -cover || { $(MAKE) failpoint-disable && exit 1; } + $(MAKE) failpoint-disable -ci-test-job: enable-codegen - CGO_ENABLED=1 go test ./... -tags deadlock -race -covermode=atomic -coverprofile=covprofile -coverpkg=../../... github.com/tikv/pd/tests/mcs +ci-test-job: + CGO_ENABLED=1 go test ./... -v -tags deadlock -race -covermode=atomic -coverprofile=covprofile -coverpkg=$(ROOT_PATH)/... github.com/tikv/pd/tests/integrations/tso install-tools: - cd ../../ && $(MAKE) install-tools + cd $(ROOT_PATH) && $(MAKE) install-tools -enable-codegen: - cd ../../ && $(MAKE) failpoint-enable +failpoint-enable: + cd $(ROOT_PATH) && $(MAKE) failpoint-enable go mod tidy -disable-codegen: - cd ../../ && $(MAKE) failpoint-disable +failpoint-disable: + cd $(ROOT_PATH) && $(MAKE) failpoint-disable go mod tidy diff --git a/tests/integrations/tso/client_test.go b/tests/integrations/tso/client_test.go new file mode 100644 index 00000000000..d7f1bd3b5d2 --- /dev/null +++ b/tests/integrations/tso/client_test.go @@ -0,0 +1,530 @@ +// Copyright 2023 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tso + +import ( + "context" + "fmt" + "math" + "math/rand" + "strings" + "sync" + "testing" + "time" + + "github.com/pingcap/failpoint" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + pd "github.com/tikv/pd/client" + "github.com/tikv/pd/client/testutil" + bs "github.com/tikv/pd/pkg/basicserver" + mcsutils "github.com/tikv/pd/pkg/mcs/utils" + "github.com/tikv/pd/pkg/slice" + "github.com/tikv/pd/pkg/storage/endpoint" + "github.com/tikv/pd/pkg/utils/tempurl" + "github.com/tikv/pd/pkg/utils/tsoutil" + "github.com/tikv/pd/server/apiv2/handlers" + "github.com/tikv/pd/tests" + "github.com/tikv/pd/tests/integrations/mcs" + handlersutil "github.com/tikv/pd/tests/server/apiv2/handlers" +) + +var r = rand.New(rand.NewSource(time.Now().UnixNano())) + +type tsoClientTestSuite struct { + suite.Suite + legacy bool + + ctx context.Context + cancel context.CancelFunc + // The PD cluster. + cluster *tests.TestCluster + // pdLeaderServer is the leader server of the PD cluster. + pdLeaderServer *tests.TestServer + // The TSO service in microservice mode. + tsoCluster *tests.TestTSOCluster + + keyspaceGroups []struct { + keyspaceGroupID uint32 + keyspaceIDs []uint32 + } + + backendEndpoints string + keyspaceIDs []uint32 + clients []pd.Client +} + +func TestLegacyTSOClient(t *testing.T) { + suite.Run(t, &tsoClientTestSuite{ + legacy: true, + }) +} + +func TestMicroserviceTSOClient(t *testing.T) { + suite.Run(t, &tsoClientTestSuite{ + legacy: false, + }) +} + +func (suite *tsoClientTestSuite) SetupSuite() { + re := suite.Require() + + var err error + suite.ctx, suite.cancel = context.WithCancel(context.Background()) + if suite.legacy { + suite.cluster, err = tests.NewTestCluster(suite.ctx, serverCount) + } else { + suite.cluster, err = tests.NewTestAPICluster(suite.ctx, serverCount) + } + re.NoError(err) + err = suite.cluster.RunInitialServers() + re.NoError(err) + leaderName := suite.cluster.WaitLeader() + suite.pdLeaderServer = suite.cluster.GetServer(leaderName) + re.NoError(suite.pdLeaderServer.BootstrapCluster()) + suite.backendEndpoints = suite.pdLeaderServer.GetAddr() + suite.keyspaceIDs = make([]uint32, 0) + + if suite.legacy { + client, err := pd.NewClientWithContext(suite.ctx, strings.Split(suite.backendEndpoints, ","), pd.SecurityOption{}) + re.NoError(err) + innerClient, ok := client.(interface{ GetServiceDiscovery() pd.ServiceDiscovery }) + re.True(ok) + re.Equal(mcsutils.NullKeyspaceID, innerClient.GetServiceDiscovery().GetKeyspaceID()) + re.Equal(mcsutils.DefaultKeyspaceGroupID, innerClient.GetServiceDiscovery().GetKeyspaceGroupID()) + mcs.WaitForTSOServiceAvailable(suite.ctx, re, client) + suite.clients = make([]pd.Client, 0) + suite.clients = append(suite.clients, client) + } else { + suite.tsoCluster, err = tests.NewTestTSOCluster(suite.ctx, 3, suite.backendEndpoints) + re.NoError(err) + + suite.keyspaceGroups = []struct { + keyspaceGroupID uint32 + keyspaceIDs []uint32 + }{ + {0, []uint32{mcsutils.DefaultKeyspaceID, 10}}, + {1, []uint32{1, 11}}, + {2, []uint32{2}}, + } + + for _, keyspaceGroup := range suite.keyspaceGroups { + suite.keyspaceIDs = append(suite.keyspaceIDs, keyspaceGroup.keyspaceIDs...) + } + + for _, param := range suite.keyspaceGroups { + if param.keyspaceGroupID == 0 { + // we have already created default keyspace group, so we can skip it. + // keyspace 10 isn't assigned to any keyspace group, so they will be + // served by default keyspace group. + continue + } + handlersutil.MustCreateKeyspaceGroup(re, suite.pdLeaderServer, &handlers.CreateKeyspaceGroupParams{ + KeyspaceGroups: []*endpoint.KeyspaceGroup{ + { + ID: param.keyspaceGroupID, + UserKind: endpoint.Standard.String(), + Members: suite.tsoCluster.GetKeyspaceGroupMember(), + Keyspaces: param.keyspaceIDs, + }, + }, + }) + } + + suite.waitForAllKeyspaceGroupsInServing(re) + } +} + +func (suite *tsoClientTestSuite) waitForAllKeyspaceGroupsInServing(re *require.Assertions) { + // The tso servers are loading keyspace groups asynchronously. Make sure all keyspace groups + // are available for serving tso requests from corresponding keyspaces by querying + // IsKeyspaceServing(keyspaceID, the Desired KeyspaceGroupID). if use default keyspace group id + // in the query, it will always return true as the keyspace will be served by default keyspace + // group before the keyspace groups are loaded. + testutil.Eventually(re, func() bool { + for _, keyspaceGroup := range suite.keyspaceGroups { + for _, keyspaceID := range keyspaceGroup.keyspaceIDs { + served := false + for _, server := range suite.tsoCluster.GetServers() { + if server.IsKeyspaceServing(keyspaceID, keyspaceGroup.keyspaceGroupID) { + served = true + break + } + } + if !served { + return false + } + } + } + return true + }, testutil.WithWaitFor(5*time.Second), testutil.WithTickInterval(50*time.Millisecond)) + + // Create clients and make sure they all have discovered the tso service. + suite.clients = mcs.WaitForMultiKeyspacesTSOAvailable( + suite.ctx, re, suite.keyspaceIDs, strings.Split(suite.backendEndpoints, ",")) + re.Equal(len(suite.keyspaceIDs), len(suite.clients)) +} + +func (suite *tsoClientTestSuite) TearDownSuite() { + suite.cancel() + if !suite.legacy { + suite.tsoCluster.Destroy() + } + suite.cluster.Destroy() +} + +func (suite *tsoClientTestSuite) TestGetTS() { + var wg sync.WaitGroup + wg.Add(tsoRequestConcurrencyNumber * len(suite.clients)) + for i := 0; i < tsoRequestConcurrencyNumber; i++ { + for _, client := range suite.clients { + go func(client pd.Client) { + defer wg.Done() + var lastTS uint64 + for j := 0; j < tsoRequestRound; j++ { + physical, logical, err := client.GetTS(suite.ctx) + suite.NoError(err) + ts := tsoutil.ComposeTS(physical, logical) + suite.Less(lastTS, ts) + lastTS = ts + } + }(client) + } + } + wg.Wait() +} + +func (suite *tsoClientTestSuite) TestGetTSAsync() { + var wg sync.WaitGroup + wg.Add(tsoRequestConcurrencyNumber * len(suite.clients)) + for i := 0; i < tsoRequestConcurrencyNumber; i++ { + for _, client := range suite.clients { + go func(client pd.Client) { + defer wg.Done() + tsFutures := make([]pd.TSFuture, tsoRequestRound) + for j := range tsFutures { + tsFutures[j] = client.GetTSAsync(suite.ctx) + } + var lastTS uint64 = math.MaxUint64 + for j := len(tsFutures) - 1; j >= 0; j-- { + physical, logical, err := tsFutures[j].Wait() + suite.NoError(err) + ts := tsoutil.ComposeTS(physical, logical) + suite.Greater(lastTS, ts) + lastTS = ts + } + }(client) + } + } + wg.Wait() +} + +func (suite *tsoClientTestSuite) TestDiscoverTSOServiceWithLegacyPath() { + re := suite.Require() + keyspaceID := uint32(1000000) + // Make sure this keyspace ID is not in use somewhere. + re.False(slice.Contains(suite.keyspaceIDs, keyspaceID)) + failpointValue := fmt.Sprintf(`return(%d)`, keyspaceID) + // Simulate the case that the server has lower version than the client and returns no tso addrs + // in the GetClusterInfo RPC. + re.NoError(failpoint.Enable("github.com/tikv/pd/client/serverReturnsNoTSOAddrs", `return(true)`)) + re.NoError(failpoint.Enable("github.com/tikv/pd/client/unexpectedCallOfFindGroupByKeyspaceID", failpointValue)) + defer func() { + re.NoError(failpoint.Disable("github.com/tikv/pd/client/serverReturnsNoTSOAddrs")) + re.NoError(failpoint.Disable("github.com/tikv/pd/client/unexpectedCallOfFindGroupByKeyspaceID")) + }() + + ctx, cancel := context.WithCancel(suite.ctx) + defer cancel() + client := mcs.SetupClientWithKeyspaceID( + ctx, re, keyspaceID, strings.Split(suite.backendEndpoints, ",")) + var lastTS uint64 + for j := 0; j < tsoRequestRound; j++ { + physical, logical, err := client.GetTS(ctx) + suite.NoError(err) + ts := tsoutil.ComposeTS(physical, logical) + suite.Less(lastTS, ts) + lastTS = ts + } +} + +// TestGetMinTS tests the correctness of GetMinTS. +func (suite *tsoClientTestSuite) TestGetMinTS() { + re := suite.Require() + suite.waitForAllKeyspaceGroupsInServing(re) + + var wg sync.WaitGroup + wg.Add(tsoRequestConcurrencyNumber * len(suite.clients)) + for i := 0; i < tsoRequestConcurrencyNumber; i++ { + for _, client := range suite.clients { + go func(client pd.Client) { + defer wg.Done() + var lastMinTS uint64 + for j := 0; j < tsoRequestRound; j++ { + physical, logical, err := client.GetMinTS(suite.ctx) + re.NoError(err) + minTS := tsoutil.ComposeTS(physical, logical) + re.Less(lastMinTS, minTS) + lastMinTS = minTS + + // Now we check whether the returned ts is the minimum one + // among all keyspace groups, i.e., the returned ts is + // less than the new timestamps of all keyspace groups. + for _, client := range suite.clients { + physical, logical, err := client.GetTS(suite.ctx) + re.NoError(err) + ts := tsoutil.ComposeTS(physical, logical) + re.Less(minTS, ts) + } + } + }(client) + } + } + wg.Wait() +} + +// More details can be found in this issue: https://github.com/tikv/pd/issues/4884 +func (suite *tsoClientTestSuite) TestUpdateAfterResetTSO() { + re := suite.Require() + ctx, cancel := context.WithCancel(suite.ctx) + defer cancel() + + for i := 0; i < len(suite.clients); i++ { + client := suite.clients[i] + testutil.Eventually(re, func() bool { + _, _, err := client.GetTS(ctx) + return err == nil + }) + // Resign leader to trigger the TSO resetting. + re.NoError(failpoint.Enable("github.com/tikv/pd/server/updateAfterResetTSO", "return(true)")) + oldLeaderName := suite.cluster.WaitLeader() + err := suite.cluster.GetServer(oldLeaderName).ResignLeader() + re.NoError(err) + re.NoError(failpoint.Disable("github.com/tikv/pd/server/updateAfterResetTSO")) + newLeaderName := suite.cluster.WaitLeader() + re.NotEqual(oldLeaderName, newLeaderName) + // Request a new TSO. + testutil.Eventually(re, func() bool { + _, _, err := client.GetTS(ctx) + return err == nil + }) + // Transfer leader back. + re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/tso/delaySyncTimestamp", `return(true)`)) + err = suite.cluster.GetServer(newLeaderName).ResignLeader() + re.NoError(err) + // Should NOT panic here. + testutil.Eventually(re, func() bool { + _, _, err := client.GetTS(ctx) + return err == nil + }) + re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/tso/delaySyncTimestamp")) + } +} + +func (suite *tsoClientTestSuite) TestRandomResignLeader() { + re := suite.Require() + re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/tso/fastUpdatePhysicalInterval", "return(true)")) + + parallelAct := func() { + // After https://github.com/tikv/pd/issues/6376 is fixed, we can use a smaller number here. + // currently, the time to discover tso service is usually a little longer than 1s, compared + // to the previous time taken < 1s. + n := r.Intn(2) + 3 + time.Sleep(time.Duration(n) * time.Second) + if !suite.legacy { + wg := sync.WaitGroup{} + // Select the first keyspace from all keyspace groups. We need to make sure the selected + // keyspaces are from different keyspace groups, otherwise multiple goroutines below could + // try to resign the primary of the same keyspace group and cause race condition. + keyspaceIDs := make([]uint32, 0) + for _, keyspaceGroup := range suite.keyspaceGroups { + if len(keyspaceGroup.keyspaceIDs) > 0 { + keyspaceIDs = append(keyspaceIDs, keyspaceGroup.keyspaceIDs[0]) + } + } + wg.Add(len(keyspaceIDs)) + for _, keyspaceID := range keyspaceIDs { + go func(keyspaceID uint32) { + defer wg.Done() + err := suite.tsoCluster.ResignPrimary(keyspaceID, mcsutils.DefaultKeyspaceGroupID) + re.NoError(err) + suite.tsoCluster.WaitForPrimaryServing(re, keyspaceID, 0) + }(keyspaceID) + } + wg.Wait() + } else { + err := suite.cluster.ResignLeader() + re.NoError(err) + suite.cluster.WaitLeader() + } + time.Sleep(time.Duration(n) * time.Second) + } + + mcs.CheckMultiKeyspacesTSO(suite.ctx, re, suite.clients, parallelAct) + re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/tso/fastUpdatePhysicalInterval")) +} + +func (suite *tsoClientTestSuite) TestRandomShutdown() { + re := suite.Require() + re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/tso/fastUpdatePhysicalInterval", "return(true)")) + + parallelAct := func() { + // After https://github.com/tikv/pd/issues/6376 is fixed, we can use a smaller number here. + // currently, the time to discover tso service is usually a little longer than 1s, compared + // to the previous time taken < 1s. + n := r.Intn(2) + 3 + time.Sleep(time.Duration(n) * time.Second) + if !suite.legacy { + suite.tsoCluster.WaitForDefaultPrimaryServing(re).Close() + } else { + suite.cluster.GetServer(suite.cluster.GetLeader()).GetServer().Close() + } + time.Sleep(time.Duration(n) * time.Second) + } + + mcs.CheckMultiKeyspacesTSO(suite.ctx, re, suite.clients, parallelAct) + suite.TearDownSuite() + suite.SetupSuite() + re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/tso/fastUpdatePhysicalInterval")) +} + +// When we upgrade the PD cluster, there may be a period of time that the old and new PDs are running at the same time. +func TestMixedTSODeployment(t *testing.T) { + re := require.New(t) + + re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/tso/fastUpdatePhysicalInterval", "return(true)")) + re.NoError(failpoint.Enable("github.com/tikv/pd/client/skipUpdateServiceMode", "return(true)")) + re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/mcs/tso/server/skipWaitAPIServiceReady", "return(true)")) + defer func() { + re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/tso/fastUpdatePhysicalInterval")) + re.NoError(failpoint.Disable("github.com/tikv/pd/client/skipUpdateServiceMode")) + re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/mcs/tso/server/skipWaitAPIServiceReady")) + }() + + ctx, cancel := context.WithCancel(context.Background()) + cluster, err := tests.NewTestCluster(ctx, 1) + re.NoError(err) + defer cancel() + defer cluster.Destroy() + + err = cluster.RunInitialServers() + re.NoError(err) + + leaderServer := cluster.GetServer(cluster.WaitLeader()) + backendEndpoints := leaderServer.GetAddr() + + apiSvr, err := cluster.JoinAPIServer(ctx) + re.NoError(err) + err = apiSvr.Run() + re.NoError(err) + + s, cleanup := tests.StartSingleTSOTestServer(ctx, re, backendEndpoints, tempurl.Alloc()) + defer cleanup() + tests.WaitForPrimaryServing(re, map[string]bs.Server{s.GetAddr(): s}) + + ctx1, cancel1 := context.WithCancel(context.Background()) + var wg sync.WaitGroup + checkTSO(ctx1, re, &wg, backendEndpoints, pd.WithAllowTSOFallback() /* It's expected that the timestamp fallback happens here */) + wg.Add(1) + go func() { + defer wg.Done() + for i := 0; i < 2; i++ { + n := r.Intn(2) + 1 + time.Sleep(time.Duration(n) * time.Second) + leaderServer.ResignLeader() + leaderServer = cluster.GetServer(cluster.WaitLeader()) + } + cancel1() + }() + wg.Wait() +} + +// TestUpgradingAPIandTSOClusters tests the scenario that after we restart the API cluster +// then restart the TSO cluster, the TSO service can still serve TSO requests normally. +func TestUpgradingAPIandTSOClusters(t *testing.T) { + re := require.New(t) + ctx, cancel := context.WithCancel(context.Background()) + + // Create an API cluster which has 3 servers + apiCluster, err := tests.NewTestAPICluster(ctx, 3) + re.NoError(err) + err = apiCluster.RunInitialServers() + re.NoError(err) + leaderName := apiCluster.WaitLeader() + pdLeader := apiCluster.GetServer(leaderName) + backendEndpoints := pdLeader.GetAddr() + + // Create a pd client in PD mode to let the API leader to forward requests to the TSO cluster. + re.NoError(failpoint.Enable("github.com/tikv/pd/client/usePDServiceMode", "return(true)")) + pdClient, err := pd.NewClientWithContext(context.Background(), + []string{backendEndpoints}, pd.SecurityOption{}, pd.WithMaxErrorRetry(1)) + re.NoError(err) + + // Create a TSO cluster which has 2 servers + tsoCluster, err := tests.NewTestTSOCluster(ctx, 2, backendEndpoints) + re.NoError(err) + tsoCluster.WaitForDefaultPrimaryServing(re) + // The TSO service should be eventually healthy + mcs.WaitForTSOServiceAvailable(ctx, re, pdClient) + + // Restart the API cluster + apiCluster, err = tests.RestartTestAPICluster(ctx, apiCluster) + re.NoError(err) + // The TSO service should be eventually healthy + mcs.WaitForTSOServiceAvailable(ctx, re, pdClient) + + // Restart the TSO cluster + tsoCluster, err = tests.RestartTestTSOCluster(ctx, tsoCluster) + re.NoError(err) + // The TSO service should be eventually healthy + mcs.WaitForTSOServiceAvailable(ctx, re, pdClient) + + tsoCluster.Destroy() + apiCluster.Destroy() + cancel() + re.NoError(failpoint.Disable("github.com/tikv/pd/client/usePDServiceMode")) +} + +func checkTSO( + ctx context.Context, re *require.Assertions, wg *sync.WaitGroup, + backendEndpoints string, opts ...pd.ClientOption, +) { + wg.Add(tsoRequestConcurrencyNumber) + for i := 0; i < tsoRequestConcurrencyNumber; i++ { + go func() { + defer wg.Done() + cli := mcs.SetupClientWithAPIContext(ctx, re, pd.NewAPIContextV1(), strings.Split(backendEndpoints, ","), opts...) + defer cli.Close() + var ts, lastTS uint64 + for { + select { + case <-ctx.Done(): + // Make sure the lastTS is not empty + re.NotEmpty(lastTS) + return + default: + } + physical, logical, err := cli.GetTS(ctx) + // omit the error check since there are many kinds of errors + if err != nil { + continue + } + ts = tsoutil.ComposeTS(physical, logical) + re.Less(lastTS, ts) + lastTS = ts + } + }() + } +} diff --git a/tests/integrations/tso/consistency_test.go b/tests/integrations/tso/consistency_test.go new file mode 100644 index 00000000000..1d35e8bf5e2 --- /dev/null +++ b/tests/integrations/tso/consistency_test.go @@ -0,0 +1,205 @@ +// Copyright 2021 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tso + +import ( + "context" + "sync" + "testing" + "time" + + "github.com/pingcap/failpoint" + "github.com/pingcap/kvproto/pkg/pdpb" + "github.com/pingcap/kvproto/pkg/tsopb" + "github.com/stretchr/testify/suite" + tso "github.com/tikv/pd/pkg/mcs/tso/server" + tsopkg "github.com/tikv/pd/pkg/tso" + "github.com/tikv/pd/pkg/utils/tempurl" + pd "github.com/tikv/pd/pkg/utils/testutil" + "github.com/tikv/pd/pkg/utils/tsoutil" + "github.com/tikv/pd/tests" + "google.golang.org/grpc" +) + +type tsoConsistencyTestSuite struct { + suite.Suite + legacy bool + + ctx context.Context + cancel context.CancelFunc + + // The PD cluster. + cluster *tests.TestCluster + // pdLeaderServer is the leader server of the PD cluster. + pdLeaderServer *tests.TestServer + // tsoServer is the TSO service provider. + tsoServer *tso.Server + tsoServerCleanup func() + tsoClientConn *grpc.ClientConn + + pdClient pdpb.PDClient + tsoClient tsopb.TSOClient +} + +func TestLegacyTSOConsistency(t *testing.T) { + suite.Run(t, &tsoConsistencyTestSuite{ + legacy: true, + }) +} + +func TestMicroserviceTSOConsistency(t *testing.T) { + suite.Run(t, &tsoConsistencyTestSuite{ + legacy: false, + }) +} + +func (suite *tsoConsistencyTestSuite) SetupSuite() { + re := suite.Require() + + var err error + suite.ctx, suite.cancel = context.WithCancel(context.Background()) + if suite.legacy { + suite.cluster, err = tests.NewTestCluster(suite.ctx, serverCount) + } else { + suite.cluster, err = tests.NewTestAPICluster(suite.ctx, serverCount) + } + re.NoError(err) + err = suite.cluster.RunInitialServers() + re.NoError(err) + leaderName := suite.cluster.WaitLeader() + suite.pdLeaderServer = suite.cluster.GetServer(leaderName) + backendEndpoints := suite.pdLeaderServer.GetAddr() + if suite.legacy { + suite.pdClient = pd.MustNewGrpcClient(re, backendEndpoints) + } else { + suite.tsoServer, suite.tsoServerCleanup = tests.StartSingleTSOTestServer(suite.ctx, re, backendEndpoints, tempurl.Alloc()) + suite.tsoClientConn, suite.tsoClient = tso.MustNewGrpcClient(re, suite.tsoServer.GetAddr()) + } +} + +func (suite *tsoConsistencyTestSuite) TearDownSuite() { + suite.cancel() + if !suite.legacy { + suite.tsoClientConn.Close() + suite.tsoServerCleanup() + } + suite.cluster.Destroy() +} + +func (suite *tsoConsistencyTestSuite) getClusterID() uint64 { + if suite.legacy { + return suite.pdLeaderServer.GetServer().ClusterID() + } + return suite.tsoServer.ClusterID() +} + +func (suite *tsoConsistencyTestSuite) request(ctx context.Context, count uint32) *pdpb.Timestamp { + re := suite.Require() + clusterID := suite.getClusterID() + if suite.legacy { + req := &pdpb.TsoRequest{ + Header: &pdpb.RequestHeader{ClusterId: clusterID}, + DcLocation: tsopkg.GlobalDCLocation, + Count: count, + } + tsoClient, err := suite.pdClient.Tso(ctx) + re.NoError(err) + defer tsoClient.CloseSend() + re.NoError(tsoClient.Send(req)) + resp, err := tsoClient.Recv() + re.NoError(err) + return checkAndReturnTimestampResponse(re, resp) + } + req := &tsopb.TsoRequest{ + Header: &tsopb.RequestHeader{ClusterId: clusterID}, + DcLocation: tsopkg.GlobalDCLocation, + Count: count, + } + tsoClient, err := suite.tsoClient.Tso(ctx) + re.NoError(err) + defer tsoClient.CloseSend() + re.NoError(tsoClient.Send(req)) + resp, err := tsoClient.Recv() + re.NoError(err) + return checkAndReturnTimestampResponse(re, resp) +} + +func (suite *tsoConsistencyTestSuite) TestRequestTSOConcurrently() { + suite.requestTSOConcurrently() + // Test Global TSO after the leader change + suite.pdLeaderServer.GetServer().GetMember().ResetLeader() + suite.cluster.WaitLeader() + suite.requestTSOConcurrently() +} + +func (suite *tsoConsistencyTestSuite) requestTSOConcurrently() { + ctx, cancel := context.WithCancel(suite.ctx) + defer cancel() + + var wg sync.WaitGroup + wg.Add(tsoRequestConcurrencyNumber) + for i := 0; i < tsoRequestConcurrencyNumber; i++ { + go func() { + defer wg.Done() + last := &pdpb.Timestamp{ + Physical: 0, + Logical: 0, + } + var ts *pdpb.Timestamp + for j := 0; j < tsoRequestRound; j++ { + ts = suite.request(ctx, tsoCount) + // Check whether the TSO fallbacks + suite.Equal(1, tsoutil.CompareTimestamp(ts, last)) + last = ts + time.Sleep(10 * time.Millisecond) + } + }() + } + wg.Wait() +} + +func (suite *tsoConsistencyTestSuite) TestFallbackTSOConsistency() { + re := suite.Require() + + // Re-create the cluster to enable the failpoints. + suite.TearDownSuite() + re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/tso/fallBackSync", `return(true)`)) + re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/tso/fallBackUpdate", `return(true)`)) + suite.SetupSuite() + re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/tso/fallBackSync")) + re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/tso/fallBackUpdate")) + + ctx, cancel := context.WithCancel(suite.ctx) + defer cancel() + var wg sync.WaitGroup + wg.Add(tsoRequestConcurrencyNumber) + for i := 0; i < tsoRequestConcurrencyNumber; i++ { + go func() { + defer wg.Done() + last := &pdpb.Timestamp{ + Physical: 0, + Logical: 0, + } + var ts *pdpb.Timestamp + for j := 0; j < tsoRequestRound; j++ { + ts = suite.request(ctx, tsoCount) + re.Equal(1, tsoutil.CompareTimestamp(ts, last)) + last = ts + time.Sleep(10 * time.Millisecond) + } + }() + } + wg.Wait() +} diff --git a/tests/integrations/tso/go.mod b/tests/integrations/tso/go.mod new file mode 100644 index 00000000000..c76efe9271b --- /dev/null +++ b/tests/integrations/tso/go.mod @@ -0,0 +1,185 @@ +module github.com/tikv/pd/tests/integrations/tso + +go 1.20 + +replace ( + github.com/tikv/pd => ../../../ + github.com/tikv/pd/client => ../../../client + github.com/tikv/pd/tests/integrations/mcs => ../mcs +) + +// reset grpc and protobuf deps in order to import client and server at the same time +replace google.golang.org/grpc v1.54.0 => google.golang.org/grpc v1.26.0 + +require ( + github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c + github.com/pingcap/kvproto v0.0.0-20230727073445-53e1f8730c30 + github.com/stretchr/testify v1.8.4 + github.com/tikv/pd v0.0.0-00010101000000-000000000000 + github.com/tikv/pd/client v0.0.0-00010101000000-000000000000 + github.com/tikv/pd/tests/integrations/mcs v0.0.0-00010101000000-000000000000 + google.golang.org/grpc v1.54.0 +) + +require ( + github.com/BurntSushi/toml v0.3.1 // indirect + github.com/KyleBanks/depth v1.2.1 // indirect + github.com/Masterminds/semver v1.5.0 // indirect + github.com/PuerkitoBio/purell v1.1.1 // indirect + github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect + github.com/ReneKroon/ttlcache/v2 v2.3.0 // indirect + github.com/VividCortex/mysqlerr v1.0.0 // indirect + github.com/Xeoncross/go-aesctr-with-hmac v0.0.0-20200623134604-12b17a7ff502 // indirect + github.com/aws/aws-sdk-go-v2 v1.17.7 // indirect + github.com/aws/aws-sdk-go-v2/config v1.18.19 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.13.18 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.1 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.31 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.25 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.3.32 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.25 // indirect + github.com/aws/aws-sdk-go-v2/service/kms v1.20.8 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.12.6 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.6 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.18.7 // indirect + github.com/aws/smithy-go v1.13.5 // indirect + github.com/benbjohnson/clock v1.3.0 // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/bitly/go-simplejson v0.5.0 // indirect + github.com/breeswish/gin-jwt/v2 v2.6.4-jwt-patch // indirect + github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 // indirect + github.com/cenkalti/backoff/v4 v4.0.2 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/coreos/go-semver v0.3.0 // indirect + github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f // indirect + github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/docker/go-units v0.4.0 // indirect + github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4 // indirect + github.com/elliotchance/pie/v2 v2.1.0 // indirect + github.com/fogleman/gg v1.3.0 // indirect + github.com/gin-contrib/cors v1.4.0 // indirect + github.com/gin-contrib/gzip v0.0.1 // indirect + github.com/gin-contrib/pprof v1.4.0 // indirect + github.com/gin-contrib/sse v0.1.0 // indirect + github.com/gin-gonic/gin v1.8.1 // indirect + github.com/go-ole/go-ole v1.2.6 // indirect + github.com/go-openapi/jsonpointer v0.19.5 // indirect + github.com/go-openapi/jsonreference v0.19.6 // indirect + github.com/go-openapi/spec v0.20.4 // indirect + github.com/go-openapi/swag v0.19.15 // indirect + github.com/go-playground/locales v0.14.0 // indirect + github.com/go-playground/universal-translator v0.18.0 // indirect + github.com/go-playground/validator/v10 v10.10.0 // indirect + github.com/go-resty/resty/v2 v2.6.0 // indirect + github.com/go-sql-driver/mysql v1.7.0 // indirect + github.com/goccy/go-graphviz v0.0.9 // indirect + github.com/goccy/go-json v0.9.7 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang-jwt/jwt v3.2.1+incompatible // indirect + github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 // indirect + github.com/golang/protobuf v1.5.3 // indirect + github.com/golang/snappy v0.0.4 // indirect + github.com/google/btree v1.1.2 // indirect + github.com/google/pprof v0.0.0-20211122183932-1daafda22083 // indirect + github.com/google/uuid v1.3.0 // indirect + github.com/gorilla/mux v1.7.4 // indirect + github.com/gorilla/websocket v1.4.2 // indirect + github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4 // indirect + github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect + github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect + github.com/gtank/cryptopasta v0.0.0-20170601214702-1f550f6f2f69 // indirect + github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d // indirect + github.com/inconshreveable/mousetrap v1.0.0 // indirect + github.com/jinzhu/inflection v1.0.0 // indirect + github.com/jinzhu/now v1.1.5 // indirect + github.com/joho/godotenv v1.4.0 // indirect + github.com/jonboulle/clockwork v0.2.2 // indirect + github.com/joomcode/errorx v1.0.1 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/konsorten/go-windows-terminal-sequences v1.0.3 // indirect + github.com/leodido/go-urn v1.2.1 // indirect + github.com/lufia/plan9stats v0.0.0-20230326075908-cb1d2100619a // indirect + github.com/mailru/easyjson v0.7.6 // indirect + github.com/mattn/go-isatty v0.0.14 // indirect + github.com/mattn/go-sqlite3 v1.14.15 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect + github.com/minio/sio v0.3.0 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/oleiade/reflections v1.0.1 // indirect + github.com/opentracing/opentracing-go v1.2.0 // indirect + github.com/pelletier/go-toml/v2 v2.0.1 // indirect + github.com/petermattis/goid v0.0.0-20211229010228-4d14c490ee36 // indirect + github.com/phf/go-queue v0.0.0-20170504031614-9abe38d0371d // indirect + github.com/pingcap/errcode v0.3.0 // indirect + github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c // indirect + github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 // indirect + github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21 // indirect + github.com/pingcap/tidb-dashboard v0.0.0-20230705095454-5e220f970f27 // indirect + github.com/pingcap/tipb v0.0.0-20220718022156-3e2483c20a9e // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/power-devops/perfstat v0.0.0-20221212215047-62379fc7944b // indirect + github.com/prometheus/client_golang v1.11.1 // indirect + github.com/prometheus/client_model v0.2.0 // indirect + github.com/prometheus/common v0.26.0 // indirect + github.com/prometheus/procfs v0.6.0 // indirect + github.com/rs/cors v1.7.0 // indirect + github.com/samber/lo v1.37.0 // indirect + github.com/sasha-s/go-deadlock v0.2.0 // indirect + github.com/shirou/gopsutil/v3 v3.23.3 // indirect + github.com/shoenig/go-m1cpu v0.1.5 // indirect + github.com/shurcooL/httpgzip v0.0.0-20190720172056-320755c1c1b0 // indirect + github.com/sirupsen/logrus v1.6.0 // indirect + github.com/smallnest/chanx v0.0.0-20221229104322-eb4c998d2072 // indirect + github.com/soheilhy/cmux v0.1.4 // indirect + github.com/spf13/cobra v1.0.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect + github.com/stretchr/objx v0.5.0 // indirect + github.com/swaggo/files v0.0.0-20210815190702-a29dd2bc99b2 // indirect + github.com/swaggo/http-swagger v1.2.6 // indirect + github.com/swaggo/swag v1.8.3 // indirect + github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965 // indirect + github.com/tklauser/go-sysconf v0.3.11 // indirect + github.com/tklauser/numcpus v0.6.0 // indirect + github.com/tmc/grpc-websocket-proxy v0.0.0-20200427203606-3cfed13b9966 // indirect + github.com/ugorji/go/codec v1.2.7 // indirect + github.com/unrolled/render v1.0.1 // indirect + github.com/urfave/negroni v0.3.0 // indirect + github.com/vmihailenco/msgpack/v5 v5.3.5 // indirect + github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect + github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect + github.com/yusufpapurcu/wmi v1.2.2 // indirect + go.etcd.io/bbolt v1.3.6 // indirect + go.etcd.io/etcd v0.5.0-alpha.5.0.20220915004622-85b640cee793 // indirect + go.uber.org/atomic v1.10.0 // indirect + go.uber.org/dig v1.9.0 // indirect + go.uber.org/fx v1.12.0 // indirect + go.uber.org/goleak v1.1.12 // indirect + go.uber.org/multierr v1.11.0 // indirect + go.uber.org/zap v1.24.0 // indirect + golang.org/x/crypto v0.1.0 // indirect + golang.org/x/exp v0.0.0-20230711005742-c3f37128e5a4 // indirect + golang.org/x/image v0.5.0 // indirect + golang.org/x/net v0.10.0 // indirect + golang.org/x/oauth2 v0.4.0 // indirect + golang.org/x/sync v0.1.0 // indirect + golang.org/x/sys v0.8.0 // indirect + golang.org/x/text v0.9.0 // indirect + golang.org/x/time v0.1.0 // indirect + golang.org/x/tools v0.6.0 // indirect + google.golang.org/appengine v1.6.7 // indirect + google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect + google.golang.org/protobuf v1.30.0 // indirect + gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + gorm.io/datatypes v1.1.0 // indirect + gorm.io/driver/mysql v1.4.5 // indirect + gorm.io/driver/sqlite v1.4.3 // indirect + gorm.io/gorm v1.24.3 // indirect + moul.io/zapgorm2 v1.1.0 // indirect + sigs.k8s.io/yaml v1.2.0 // indirect +) diff --git a/tests/mcs/go.sum b/tests/integrations/tso/go.sum similarity index 85% rename from tests/mcs/go.sum rename to tests/integrations/tso/go.sum index dd2b20b4b4b..b04353eb0b6 100644 --- a/tests/mcs/go.sum +++ b/tests/integrations/tso/go.sum @@ -7,19 +7,17 @@ github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6Xge github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww= github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/ReneKroon/ttlcache/v2 v2.3.0 h1:qZnUjRKIrbKHH6vF5T7Y9Izn5ObfTZfyYpGhvz2BKPo= github.com/ReneKroon/ttlcache/v2 v2.3.0/go.mod h1:zbo6Pv/28e21Z8CzzqgYRArQYGYtjONRxaAKGxzQvG4= -github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUWq3EgK3CesDbo8upS2Vm9/P3FtgI+Jk= -github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/VividCortex/mysqlerr v1.0.0 h1:5pZ2TZA+YnzPgzBfiUWGqWmKDVNBdrkf9g+DNe1Tiq8= github.com/VividCortex/mysqlerr v1.0.0/go.mod h1:xERx8E4tBhLvpjzdUyQiSfUxeMcATEQrflDAfXsqcAE= github.com/Xeoncross/go-aesctr-with-hmac v0.0.0-20200623134604-12b17a7ff502 h1:L8IbaI/W6h5Cwgh0n4zGeZpVK78r/jBf9ASurHo9+/o= github.com/Xeoncross/go-aesctr-with-hmac v0.0.0-20200623134604-12b17a7ff502/go.mod h1:pmnBM9bxWSiHvC/gSWunUIyDvGn33EkP2CUjxFKtTTM= +github.com/agiledragon/gomonkey/v2 v2.3.1/go.mod h1:ap1AmDzcVOAz1YpeJ3TCzIgstoaWLA6jbbgxfB4w2iY= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -30,10 +28,35 @@ github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kd github.com/appleboy/gofight/v2 v2.1.2 h1:VOy3jow4vIK8BRQJoC/I9muxyYlJ2yb9ht2hZoS3rf4= github.com/appleboy/gofight/v2 v2.1.2/go.mod h1:frW+U1QZEdDgixycTj4CygQ48yLTUhplt43+Wczp3rw= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= -github.com/aws/aws-sdk-go v1.35.3 h1:r0puXncSaAfRt7Btml2swUo74Kao+vKhO3VLjwDjK54= -github.com/aws/aws-sdk-go v1.35.3/go.mod h1:H7NKnBqNVzoTJpGfLrQkkD+ytBA93eiDYi/+8rV9s48= -github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= +github.com/aws/aws-sdk-go-v2 v1.17.7 h1:CLSjnhJSTSogvqUGhIC6LqFKATMRexcxLZ0i/Nzk9Eg= +github.com/aws/aws-sdk-go-v2 v1.17.7/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= +github.com/aws/aws-sdk-go-v2/config v1.18.19 h1:AqFK6zFNtq4i1EYu+eC7lcKHYnZagMn6SW171la0bGw= +github.com/aws/aws-sdk-go-v2/config v1.18.19/go.mod h1:XvTmGMY8d52ougvakOv1RpiTLPz9dlG/OQHsKU/cMmY= +github.com/aws/aws-sdk-go-v2/credentials v1.13.18 h1:EQMdtHwz0ILTW1hoP+EwuWhwCG1hD6l3+RWFQABET4c= +github.com/aws/aws-sdk-go-v2/credentials v1.13.18/go.mod h1:vnwlwjIe+3XJPBYKu1et30ZPABG3VaXJYr8ryohpIyM= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.1 h1:gt57MN3liKiyGopcqgNzJb2+d9MJaKT/q1OksHNXVE4= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.1/go.mod h1:lfUx8puBRdM5lVVMQlwt2v+ofiG/X6Ms+dy0UkG/kXw= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.31 h1:sJLYcS+eZn5EeNINGHSCRAwUJMFVqklwkH36Vbyai7M= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.31/go.mod h1:QT0BqUvX1Bh2ABdTGnjqEjvjzrCfIniM9Sc8zn9Yndo= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.25 h1:1mnRASEKnkqsntcxHaysxwgVoUUp5dkiB+l3llKnqyg= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.25/go.mod h1:zBHOPwhBc3FlQjQJE/D3IfPWiWaQmT06Vq9aNukDo0k= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.32 h1:p5luUImdIqywn6JpQsW3tq5GNOxKmOnEpybzPx+d1lk= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.32/go.mod h1:XGhIBZDEgfqmFIugclZ6FU7v75nHhBDtzuB4xB/tEi4= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.25 h1:5LHn8JQ0qvjD9L9JhMtylnkcw7j05GDZqM9Oin6hpr0= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.25/go.mod h1:/95IA+0lMnzW6XzqYJRpjjsAbKEORVeO0anQqjd2CNU= +github.com/aws/aws-sdk-go-v2/service/kms v1.20.8 h1:R5f4VOFi3ScTe7TtePyxLqEhNqTJIAxL57MzrXFNs6I= +github.com/aws/aws-sdk-go-v2/service/kms v1.20.8/go.mod h1:OtP3pBOgmJM+acQyQcQXtQHets3yJoVuanCx2T5M7v4= +github.com/aws/aws-sdk-go-v2/service/sso v1.12.6 h1:5V7DWLBd7wTELVz5bPpwzYy/sikk0gsgZfj40X+l5OI= +github.com/aws/aws-sdk-go-v2/service/sso v1.12.6/go.mod h1:Y1VOmit/Fn6Tz1uFAeCO6Q7M2fmfXSCLeL5INVYsLuY= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.6 h1:B8cauxOH1W1v7rd8RdI/MWnoR4Ze0wIHWrb90qczxj4= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.6/go.mod h1:Lh/bc9XUf8CfOY6Jp5aIkQtN+j1mc+nExc+KXj9jx2s= +github.com/aws/aws-sdk-go-v2/service/sts v1.18.7 h1:bWNgNdRko2x6gqa0blfATqAZKZokPIeM1vfmQt2pnvM= +github.com/aws/aws-sdk-go-v2/service/sts v1.18.7/go.mod h1:JuTnSoeePXmMVe9G8NcjjwgOKEfZ4cOjMuT2IBT/2eI= +github.com/aws/smithy-go v1.13.5 h1:hgz0X/DX0dGqTYpGALqXJoRKRj5oQ7150i5FdTePzO8= +github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= +github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= +github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -51,8 +74,8 @@ github.com/cenkalti/backoff/v4 v4.0.2/go.mod h1:eEew/i+1Q6OrCDZh3WiXYv3+nJwBASZ8 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= -github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= @@ -74,6 +97,7 @@ github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbp github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/corona10/goimagehash v1.0.2 h1:pUfB0LnsJASMPGEZLj7tGY251vF+qLGqOgEP4rUs6kA= github.com/corona10/goimagehash v1.0.2/go.mod h1:/l9umBhvcHQXVtQO1V6Gp1yD20STawkhRnnX0D1bvVI= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= @@ -88,8 +112,6 @@ github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4 h1:qk/FSDDxo05w github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385 h1:clC1lXBpe2kTj2VHdaIu9ajZQe4kcEY9j0NsnDDBZ3o= github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM= -github.com/elastic/gosigar v0.14.2 h1:Dg80n8cr90OZ7x+bAax/QjoW/XqTI11RmA79ZwIm9/4= -github.com/elastic/gosigar v0.14.2/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs= github.com/elliotchance/pie/v2 v2.1.0 h1:KEVAAzxYxTyFs4hvebFZVzBdEo3YeMzl2HYDWn+P3F4= github.com/elliotchance/pie/v2 v2.1.0/go.mod h1:18t0dgGFH006g4eVdDtWfgFZPQEgl10IoEO8YWEq3Og= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= @@ -105,38 +127,30 @@ github.com/gin-contrib/cors v1.4.0 h1:oJ6gwtUl3lqV0WEIwM/LxPF1QZ5qe2lGWdY2+bz7y0 github.com/gin-contrib/cors v1.4.0/go.mod h1:bs9pNM0x/UsmHPBWT2xZz9ROh8xYjYkiURUfmBoMlcs= github.com/gin-contrib/gzip v0.0.1 h1:ezvKOL6jH+jlzdHNE4h9h8q8uMpDQjyl0NN0Jd7jozc= github.com/gin-contrib/gzip v0.0.1/go.mod h1:fGBJBCdt6qCZuCAOwWuFhBB4OOq9EFqlo5dEaFhhu5w= +github.com/gin-contrib/pprof v1.4.0 h1:XxiBSf5jWZ5i16lNOPbMTVdgHBdhfGRD5PZ1LWazzvg= +github.com/gin-contrib/pprof v1.4.0/go.mod h1:RrehPJasUVBPK6yTUwOl8/NP6i0vbUgmxtis+Z5KE90= github.com/gin-contrib/sse v0.0.0-20170109093832-22d885f9ecc7/go.mod h1:VJ0WA2NBN22VlZ2dKZQPAPnyWw5XTlK1KymzLKsr59s= -github.com/gin-contrib/sse v0.0.0-20190301062529-5545eab6dad3/go.mod h1:VJ0WA2NBN22VlZ2dKZQPAPnyWw5XTlK1KymzLKsr59s= github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= github.com/gin-gonic/gin v1.3.0/go.mod h1:7cKuhb5qV2ggCFctp2fJQ+ErvciLZrIeoOSOm6mUr7Y= -github.com/gin-gonic/gin v1.4.0/go.mod h1:OW2EZn3DO8Ln9oIKOvM++LBO+5UPHJJDH72/q/3rZdM= github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M= github.com/gin-gonic/gin v1.8.1 h1:4+fr/el88TOO3ewCmQr8cx/CtZ/umlIRIs5M4NTNjf8= github.com/gin-gonic/gin v1.8.1/go.mod h1:ji8BvRH1azfM+SYow9zQ6SZMvR8qOMZHmsCuWR9tTTk= -github.com/go-chi/chi v4.0.2+incompatible h1:maB6vn6FqCxrpz4FqWdh4+lwpyZIQS7YEAUcHlgXVRs= -github.com/go-chi/chi v4.0.2+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= -github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= -github.com/go-openapi/jsonpointer v0.17.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonreference v0.17.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= -github.com/go-openapi/jsonreference v0.19.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= github.com/go-openapi/jsonreference v0.19.6 h1:UBIxjkht+AWIgYzCDSv2GN+E/togfwXUJFRTWhl2Jjs= github.com/go-openapi/jsonreference v0.19.6/go.mod h1:diGHMEHg2IqXZGKxqyvWdfWU/aim5Dprw5bqpKkTvns= -github.com/go-openapi/spec v0.19.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= github.com/go-openapi/spec v0.20.4 h1:O8hJrt0UMnhHcluhIdUgCLRWyM2x7QkBXRvOs7m+O1M= github.com/go-openapi/spec v0.20.4/go.mod h1:faYFR1CvsJZ0mNsmsphTMSoRrNV3TEDoAM7FOEWeq8I= -github.com/go-openapi/swag v0.17.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.19.15 h1:D2NRCBzS9/pEY3gP9Nl8aDqGUcPFrwG2p+CNFrLyrCM= github.com/go-openapi/swag v0.19.15/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= @@ -153,7 +167,6 @@ github.com/go-playground/validator/v10 v10.10.0 h1:I7mrTYv78z8k8VXa/qJlOlEXn/nBh github.com/go-playground/validator/v10 v10.10.0/go.mod h1:74x4gJWsvQexRdW8Pn3dXSGrTK4nAUsbPlLADvpJkos= github.com/go-resty/resty/v2 v2.6.0 h1:joIR5PNLM2EFqqESUjCMGXrWmXNHEU9CEiK813oKYS4= github.com/go-resty/resty/v2 v2.6.0/go.mod h1:PwvJS6hvaPkjtjNg9ph+VrSD92bi5Zq73w/BIH7cC3Q= -github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-sql-driver/mysql v1.7.0 h1:ueSltNNllEqE3qcWBTD0iQd3IpL/6U+mJxLkazJ7YPc= github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= @@ -173,8 +186,8 @@ github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 h1:au07oEsX2xN0kt github.com/golang-sql/sqlexp v0.1.0 h1:ZCD6MBpcuOVfGVqsEmY5/4FtYiKz6tSyUv9LPEDei6A= github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 h1:DACJavvAHhabrF08vX0COfcOBJRhZ8lUbR+ZWIs0Y5g= github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/glog v1.1.0 h1:/d3pCKDPWNnvIWe0vVUpNP32qc8U3PDVxySP/y360qE= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef h1:veQD95Isof8w9/WXiA+pa3tz3fJXkt5B7QaRBrM62gk= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -193,8 +206,8 @@ github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvq github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= -github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= @@ -208,6 +221,7 @@ github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -218,7 +232,7 @@ github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/mux v1.7.4 h1:VuZ8uybHlWmqV03+zRzdwKL4tUnIp1MAQtp1mIFE1bc= github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= @@ -258,9 +272,7 @@ github.com/jinzhu/now v1.1.2/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/ github.com/jinzhu/now v1.1.4/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= -github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= -github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/joho/godotenv v1.4.0 h1:3l4+N6zfMWnkbPEXKng2o2/MR5mSwTrBih4ZEkkz1lg= github.com/joho/godotenv v1.4.0/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= @@ -281,6 +293,7 @@ github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/ github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= @@ -301,18 +314,16 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w= github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= -github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= +github.com/lufia/plan9stats v0.0.0-20230326075908-cb1d2100619a h1:N9zuLhTvBSRt0gWSiJswwQ2HqDmtX/ZCDJURnKUt1Ik= +github.com/lufia/plan9stats v0.0.0-20230326075908-cb1d2100619a/go.mod h1:JKx41uQRwqlTZabZc+kILPrO/3jlKnQ2Z8b7YiVw5cE= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA= github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= @@ -350,6 +361,11 @@ github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1Cpa github.com/onsi/gomega v1.20.1 h1:PA/3qinGoukvymdIDV8pii6tiZgC8kbmJO6Z5+b002Q= github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= +github.com/otiai10/copy v1.7.0/go.mod h1:rmRl6QPdJj6EiUqXQ/4Nn2lLXoNQjFCQbbNrxgc/t3U= +github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95/go.mod h1:9qAhocn7zKJG+0mI8eUu6xqkFDYS2kb2saOteoSB3cE= +github.com/otiai10/curr v1.0.0/go.mod h1:LskTG5wDwr8Rs+nNQ+1LlxRjAtTZZjtJW4rMXl6j4vs= +github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT91xUo= +github.com/otiai10/mint v1.3.3/go.mod h1:/yxELlJQ0ufhjUwhshSj+wFjZ78CnZ48/1wtmBH1OTc= github.com/pascaldekloe/name v0.0.0-20180628100202-0fd16699aae1/go.mod h1:eD5JxqMiuNYyFNmyY9rkJ/slN8y59oEu4Ei7F8OoKWQ= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml/v2 v2.0.1 h1:8e3L2cCQzLFi2CR4g7vGFuFxX7Jl1kKX8gW+iV0GUKU= @@ -358,8 +374,6 @@ github.com/petermattis/goid v0.0.0-20211229010228-4d14c490ee36 h1:64bxqeTEN0/xoE github.com/petermattis/goid v0.0.0-20211229010228-4d14c490ee36/go.mod h1:pxMtw7cyUw6B2bRH0ZBANSPg+AoSud1I1iyJHI69jH4= github.com/phf/go-queue v0.0.0-20170504031614-9abe38d0371d h1:U+PMnTlV2tu7RuMK5etusZG3Cf+rpow5hqQByeCzJ2g= github.com/phf/go-queue v0.0.0-20170504031614-9abe38d0371d/go.mod h1:lXfE4PvvTW5xOjO6Mba8zDPyw8M93B6AQ7frTGnMlA8= -github.com/pingcap/check v0.0.0-20190102082844-67f458068fc8/go.mod h1:B1+S9LNcuMyLH/4HMTViQOJevkGiik3wW2AN9zb2fNQ= -github.com/pingcap/check v0.0.0-20191107115940-caf2b9e6ccf4/go.mod h1:PYMCGwN0JHjoqGr3HrZoD+b8Tgx8bKnArhSq8YVzUMc= github.com/pingcap/check v0.0.0-20191216031241-8a5a85928f12 h1:rfD9v3+ppLPzoQBgZev0qYCpegrwyFx/BUpkApEiKdY= github.com/pingcap/errcode v0.3.0 h1:IF6LC/4+b1KNwrMlr2rBTUrojFPMexXBcDWZSpNwxjg= github.com/pingcap/errcode v0.3.0/go.mod h1:4b2X8xSqxIroj/IZ9MX/VGZhAwc11wB9wRIzHvz6SeM= @@ -368,19 +382,18 @@ github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTw github.com/pingcap/errors v0.11.5-0.20190809092503-95897b64e011/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c h1:xpW9bvK+HuuTmyFqUwr+jcCvpVkK7sumiz+ko5H9eq4= github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c/go.mod h1:X2r9ueLEUZgtx2cIogM0v4Zj5uvvzhuuiu7Pn8HzMPg= -github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00 h1:C3N3itkduZXDZFh4N3vQ5HEtld3S+Y+StULhWVvumU0= -github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00/go.mod h1:4qGtCB0QK0wBzKtFEGDhxXnSnbQApw1gc9siScUl8ew= +github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c h1:CgbKAHto5CQgWM9fSBIvaxsJHuGP0uM74HXtv3MyyGQ= +github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c/go.mod h1:4qGtCB0QK0wBzKtFEGDhxXnSnbQApw1gc9siScUl8ew= github.com/pingcap/kvproto v0.0.0-20191211054548-3c6b38ea5107/go.mod h1:WWLmULLO7l8IOcQG+t+ItJ3fEcrL5FxF0Wu+HrMy26w= -github.com/pingcap/kvproto v0.0.0-20230228041042-1e9aca94bab6 h1:bgLRG7gPJCq6aduA65ZV7xWQBThTcuarBB9VdfAzV4g= -github.com/pingcap/kvproto v0.0.0-20230228041042-1e9aca94bab6/go.mod h1:KUrW1FGoznGMMTssYBu0czfAhn6vQcIrHyZoSC6T990= -github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8= +github.com/pingcap/kvproto v0.0.0-20230727073445-53e1f8730c30 h1:EvqKcDT7ceGLW0mXqM8Cp5Z8DfgQRnwj2YTnlCLj2QI= +github.com/pingcap/kvproto v0.0.0-20230727073445-53e1f8730c30/go.mod h1:r0q/CFcwvyeRhKtoqzmWMBebrtpIziQQ9vR+JKh1knc= github.com/pingcap/log v0.0.0-20210625125904-98ed8e2eb1c7/go.mod h1:8AanEdAHATuRurdGxZXBz0At+9avep+ub7U1AGYLIMM= github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 h1:HR/ylkkLmGdSSDaD8IDP+SZrdhV1Kibl9KrHxJ9eciw= github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4= -github.com/pingcap/sysutil v0.0.0-20211208032423-041a72e5860d h1:k3/APKZjXOyJrFy8VyYwRlZhMelpD3qBLJNsw3bPl/g= -github.com/pingcap/sysutil v0.0.0-20211208032423-041a72e5860d/go.mod h1:7j18ezaWTao2LHOyMlsc2Dg1vW+mDY9dEbPzVyOlaeM= -github.com/pingcap/tidb-dashboard v0.0.0-20230209052558-a58fc2a7e924 h1:49x3JR5zEYqjVqONKV9r/nrv0Rh5QU8ivIhktoLvP4g= -github.com/pingcap/tidb-dashboard v0.0.0-20230209052558-a58fc2a7e924/go.mod h1:OUzFMMVjR1GKlf4LWLqza9QNKjCrYJ7stVn/3PN0djM= +github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21 h1:QV6jqlfOkh8hqvEAgwBZa+4bSgO0EeKC7s5c6Luam2I= +github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21/go.mod h1:QYnjfA95ZaMefyl1NO8oPtKeb8pYUdnDVhQgf+qdpjM= +github.com/pingcap/tidb-dashboard v0.0.0-20230705095454-5e220f970f27 h1:K9lZMYuDuAiR5kOjFESwJ8KfSb4ui5zX6vZGbUp58uk= +github.com/pingcap/tidb-dashboard v0.0.0-20230705095454-5e220f970f27/go.mod h1:OUzFMMVjR1GKlf4LWLqza9QNKjCrYJ7stVn/3PN0djM= github.com/pingcap/tipb v0.0.0-20220718022156-3e2483c20a9e h1:FBaTXU8C3xgt/drM58VHxojHo/QoG1oPsgWTGvaSpO4= github.com/pingcap/tipb v0.0.0-20220718022156-3e2483c20a9e/go.mod h1:A7mrd7WHBl1o63LE2bIBGEJMTNWXqhgmYiOvMLxozfs= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= @@ -390,8 +403,9 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw= github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= +github.com/power-devops/perfstat v0.0.0-20221212215047-62379fc7944b h1:0LFwY6Q3gMACTjAbMZBjXAqTOzOwFaj2Ld6cjeQ7Rig= +github.com/power-devops/perfstat v0.0.0-20221212215047-62379fc7944b/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= @@ -430,10 +444,14 @@ github.com/samber/lo v1.37.0/go.mod h1:9vaz2O4o8oOnK23pd2TrXufcbdbJIa3b6cstBWKpo github.com/sasha-s/go-deadlock v0.2.0 h1:lMqc+fUb7RrFS3gQLtoQsJ7/6TV/pAIFvBsqX73DK8Y= github.com/sasha-s/go-deadlock v0.2.0/go.mod h1:StQn567HiB1fF2yJ44N9au7wOhrPS3iZqiDbRupzT10= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= -github.com/shirou/gopsutil v3.21.3+incompatible h1:uenXGGa8ESCQq+dbgtl916dmg6PSAz2cXov0uORQ9v8= -github.com/shirou/gopsutil v3.21.3+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= -github.com/shirou/gopsutil/v3 v3.22.12 h1:oG0ns6poeUSxf78JtOsfygNWuEHYYz8hnnNg7P04TJs= -github.com/shirou/gopsutil/v3 v3.22.12/go.mod h1:Xd7P1kwZcp5VW52+9XsirIKd/BROzbb2wdX3Kqlz9uI= +github.com/shirou/gopsutil/v3 v3.21.12/go.mod h1:BToYZVTlSVlfazpDDYFnsVZLaoRG+g8ufT6fPQLdJzA= +github.com/shirou/gopsutil/v3 v3.23.3 h1:Syt5vVZXUDXPEXpIBt5ziWsJ4LdSAAxF4l/xZeQgSEE= +github.com/shirou/gopsutil/v3 v3.23.3/go.mod h1:lSBNN6t3+D6W5e5nXTxc8KIMMVxAcS+6IJlffjRRlMU= +github.com/shoenig/go-m1cpu v0.1.4/go.mod h1:Wwvst4LR89UxjeFtLRMrpgRiyY4xPsejnVZym39dbAQ= +github.com/shoenig/go-m1cpu v0.1.5 h1:LF57Z/Fpb/WdGLjt2HZilNnmZOxg/q2bSKTQhgbrLrQ= +github.com/shoenig/go-m1cpu v0.1.5/go.mod h1:Wwvst4LR89UxjeFtLRMrpgRiyY4xPsejnVZym39dbAQ= +github.com/shoenig/test v0.6.3 h1:GVXWJFk9PiOjN0KoJ7VrJGH6uLPnqxR7/fe3HUPfE0c= +github.com/shoenig/test v0.6.3/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k= github.com/shurcooL/httpgzip v0.0.0-20190720172056-320755c1c1b0 h1:mj/nMDAwTBiaCqMEs4cYCqF7pO6Np7vhy1D1wcQGz+E= github.com/shurcooL/httpgzip v0.0.0-20190720172056-320755c1c1b0/go.mod h1:919LwcH0M7/W4fcZ0/jy0qGght1GIhqyS/EgWGH2j5Q= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= @@ -443,6 +461,8 @@ github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/smallnest/chanx v0.0.0-20221229104322-eb4c998d2072 h1:Txo4SXVJq/OgEjwgkWoxkMoTjGlcrgsQE/XSghjmu0w= github.com/smallnest/chanx v0.0.0-20221229104322-eb4c998d2072/go.mod h1:+4nWMF0+CqEcU74SnX2NxaGqZ8zX4pcQ8Jcs77DbX5A= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.4 h1:0HKaf1o97UwFjHH9o5XsHUOF+tqmdA7KEzXLpiyaw0E= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= @@ -470,15 +490,14 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= -github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/swaggo/files v0.0.0-20190704085106-630677cd5c14 h1:PyYN9JH5jY9j6av01SpfRMb+1DWg/i3MbGOKPxJ2wjM= -github.com/swaggo/files v0.0.0-20190704085106-630677cd5c14/go.mod h1:gxQT6pBGRuIGunNf/+tSOB5OHvguWi8Tbt82WOkf35E= -github.com/swaggo/gin-swagger v1.2.0/go.mod h1:qlH2+W7zXGZkczuL+r2nEBR2JTT+/lX05Nn6vPhc7OI= -github.com/swaggo/http-swagger v0.0.0-20200308142732-58ac5e232fba h1:lUPlXKqgbqT2SVg2Y+eT9mu5wbqMnG+i/+Q9nK7C0Rs= -github.com/swaggo/http-swagger v0.0.0-20200308142732-58ac5e232fba/go.mod h1:O1lAbCgAAX/KZ80LM/OXwtWFI/5TvZlwxSg8Cq08PV0= -github.com/swaggo/swag v1.5.1/go.mod h1:1Bl9F/ZBpVWh22nY0zmYyASPO1lI/zIwRDrpZU+tv8Y= -github.com/swaggo/swag v1.6.3/go.mod h1:wcc83tB4Mb2aNiL/HP4MFeQdpHUrca+Rp/DRNgWAUio= +github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/swaggo/files v0.0.0-20210815190702-a29dd2bc99b2 h1:+iNTcqQJy0OZ5jk6a5NLib47eqXK8uYcPX+O4+cBpEM= +github.com/swaggo/files v0.0.0-20210815190702-a29dd2bc99b2/go.mod h1:lKJPbtWzJ9JhsTN1k1gZgleJWY/cqq0psdoMmaThG3w= +github.com/swaggo/http-swagger v1.2.6 h1:ihTjChUoSRMpFMjWw+0AkL1Ti4r6v8pCgVYLmQVRlRw= +github.com/swaggo/http-swagger v1.2.6/go.mod h1:CcoICgY3yVDk2u1LQUCMHbAj0fjlxIX+873psXlIKNA= +github.com/swaggo/swag v1.7.9/go.mod h1:gZ+TJ2w/Ve1RwQsA2IRoSOTidHz6DX+PIG8GWvbnoLU= github.com/swaggo/swag v1.8.3 h1:3pZSSCQ//gAH88lfmxM3Cd1+JCsxV8Md6f36b9hrZ5s= github.com/swaggo/swag v1.8.3/go.mod h1:jMLeXOOmYyjk8PvHTsXBdrubsNd9gUJTTCzL5iBnseg= github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965 h1:1oFLiOyVl+W7bnBzGhf7BbIv9loSFQcieWWYIjLqcAw= @@ -489,27 +508,26 @@ github.com/tidwall/match v1.0.1 h1:PnKP62LPNxHKTwvHHZZzdOAOCtsJTjo6dZLCwpKm5xc= github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E= github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= -github.com/tklauser/go-sysconf v0.3.4/go.mod h1:Cl2c8ZRWfHD5IrfHo9VN+FX9kCFjIOyVklgXycLB6ek= +github.com/tklauser/go-sysconf v0.3.9/go.mod h1:11DU/5sG7UexIrp/O6g35hrWzu0JxlwQ3LSFUzyeuhs= github.com/tklauser/go-sysconf v0.3.11 h1:89WgdJhk5SNwJfu+GKyYveZ4IaJ7xAkecBo+KdJV0CM= github.com/tklauser/go-sysconf v0.3.11/go.mod h1:GqXfhXY3kiPa0nAXPDIQIWzJbMCB7AmcWpGR8lSZfqI= -github.com/tklauser/numcpus v0.2.1/go.mod h1:9aU+wOc6WjUIZEwWMP62PL/41d65P+iks1gBkr4QyP8= +github.com/tklauser/numcpus v0.3.0/go.mod h1:yFGUr7TUHQRAhyqBcEg0Ge34zDBAsIvJJcyE6boqnA8= github.com/tklauser/numcpus v0.6.0 h1:kebhY2Qt+3U6RNK7UqpYNA+tJ23IBEGKkB7JQBfDYms= github.com/tklauser/numcpus v0.6.0/go.mod h1:FEZLMke0lhOUG6w2JadTzp0a+Nl8PF/GFkQ5UVIcaL4= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20200427203606-3cfed13b9966 h1:j6JEOq5QWFker+d7mFQYOhjTZonQ7YkLTHm56dbn+yM= github.com/tmc/grpc-websocket-proxy v0.0.0-20200427203606-3cfed13b9966/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= -github.com/ugorji/go v1.1.5-pre/go.mod h1:FwP/aQVg39TXzItUBMwnWp9T9gPQnXw4Poh4/oBQZ/0= github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= github.com/ugorji/go v1.2.7/go.mod h1:nF9osbDWLy6bDVv/Rtoh6QgnvNDpmCalQV5urGCCS6M= github.com/ugorji/go/codec v0.0.0-20181022190402-e5e69e061d4f/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= -github.com/ugorji/go/codec v1.1.5-pre/go.mod h1:tULtS6Gy1AE1yCENaw4Vb//HLH5njI2tfCQDUqRd8fI= github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= github.com/ugorji/go/codec v1.2.7 h1:YPXUKf7fYbp/y8xloBqZOw2qaVggbfwMlI8WM3wZUJ0= github.com/ugorji/go/codec v1.2.7/go.mod h1:WGN1fab3R1fzQlVQTkfxVtIBhWDRqOviHU95kRgeqEY= github.com/unrolled/render v1.0.1 h1:VDDnQQVfBMsOsp3VaCJszSO0nkBIVEYoPWeRThk9spY= github.com/unrolled/render v1.0.1/go.mod h1:gN9T0NhL4Bfbwu8ann7Ry/TGHYfosul+J0obPf6NBdM= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= +github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= github.com/urfave/negroni v0.3.0 h1:PaXOb61mWeZJxc1Ji2xJjpVg9QfPo0rrB+lHyBxGNSU= github.com/urfave/negroni v0.3.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4= github.com/vmihailenco/msgpack/v5 v5.3.5 h1:5gO0H1iULLWGhs2H5tbAHIZTV8/cYafcFOr9znI5mJU= @@ -522,6 +540,8 @@ github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1: github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg= github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= @@ -543,7 +563,6 @@ go.uber.org/fx v1.12.0 h1:+1+3Cz9M0dFMPy9SW9XUIUHye8bnPUm7q7DroNGWYG4= go.uber.org/fx v1.12.0/go.mod h1:egT3Kyg1JFYQkvKLZ3EsykxkNrZxgXS+gKoKo7abERY= go.uber.org/goleak v0.10.0/go.mod h1:VCZuO8V8mFPlL0F5J5GK1rtHV3DrFcQ1R8ryq7FK0aI= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= -go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= @@ -551,16 +570,17 @@ go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+ go.uber.org/multierr v1.4.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= -go.uber.org/multierr v1.7.0 h1:zaiO/rmgFjbmCXdSYJWQcdvOCsthmdaHfr3Gm2Kx4Ec= go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.12.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= -go.uber.org/zap v1.20.0 h1:N4oPlghZwYG55MlU6LXk/Zp00FVNE9X9wrYO8CEs4lc= -go.uber.org/zap v1.20.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw= +go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= +go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -569,14 +589,16 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200204104054-c9f3fb736b72/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.1.0 h1:MDRAIl0xIo9Io2xV565hzXHw3zVseKrJKodhohM5CjU= golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20230108222341-4b8118a2686a h1:tlXy25amD5A7gOfbXdqCGN5k8ESEed/Ee1E5RcrYnqU= -golang.org/x/exp v0.0.0-20230108222341-4b8118a2686a/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= -golang.org/x/image v0.0.0-20200119044424-58c23975cae1 h1:5h3ngYt7+vXCDZCup/HkCQgW5XwmSvR/nA2JmJ0RErg= +golang.org/x/exp v0.0.0-20230711005742-c3f37128e5a4 h1:QLureRX3moex6NVu/Lr4MGakp9FdA7sBHGBmvRW7NaM= +golang.org/x/exp v0.0.0-20230711005742-c3f37128e5a4/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= golang.org/x/image v0.0.0-20200119044424-58c23975cae1/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.5.0 h1:5JMiNunQeQw++mMOz48/ISeNu3Iweh/JaZU8ZLqHRrI= +golang.org/x/image v0.5.0/go.mod h1:FVC7BI/5Ym8R25iw5OLsgshdUBbT1h5jZTpA+mvAdZ4= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= @@ -588,7 +610,8 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.6.0 h1:b9gGHsz9/HhJ3HF5DHQytPpuwocVTChQJK3AvoLRD5I= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.11.0 h1:bUO06HqtnRcc/7l71XBe4WcqTZ+3AH1J59zWDDwLKgU= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -599,13 +622,10 @@ golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= -golang.org/x/net v0.0.0-20190611141213-3f473d35a33a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= @@ -614,14 +634,16 @@ golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM= +golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.2.0 h1:sZfSu1wtKLGlWI4ZZayP0ck9Y73K1ynO6gqzTdBVdPU= -golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783 h1:nt+Q6cXKz4MosCSpnbMtqiQ8Oz0pxTef2B4Vca2lvfk= -golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= +golang.org/x/oauth2 v0.4.0 h1:NF0gk8LVPg1Ml7SSbGyySuoxdsXitj7TvgvuRxIMc/M= +golang.org/x/oauth2 v0.4.0/go.mod h1:RznEsdpjGAINPTOF0UH/t+xJ75L18YO3Ho6Pyn+uRec= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -630,9 +652,9 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20180810173357-98c5dad5d1a0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -640,10 +662,8 @@ golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181228144115-9a3f9b0469bb/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190610200419-93c9922d18ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -655,7 +675,6 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210217105451-b926d437f341/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -664,19 +683,27 @@ golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210816074244-15123e1e1f71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211013075003-97ac67df715c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ= -golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.4.0 h1:BrVqGRd7+k1DiOgtnFvAkoQEWQvBc25ouMJM6429SFg= -golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.1.0 h1:xYY+Bajn2a7VBmTM5GikTmnK8ZuX8YgnQCqZpbBNtmA= @@ -686,15 +713,13 @@ golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524210228-3d17549cdc6b/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190606050223-4d9ae51c2468/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190611222205-d73e1c7e250b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191030062658-86caa796c7ab/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191107010934-f79515f33823/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191114200427-caa0b0f7d508/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -704,8 +729,10 @@ golang.org/x/tools v0.0.0-20201125231158-b5590deeca9b/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210112230658-8b4aab62c064/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.2.0 h1:G6AHpWxTMGY1KyEYoAQ5WTtIekUUvDNjan3ugu60JvE= -golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA= +golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -718,8 +745,8 @@ google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoA google.golang.org/genproto v0.0.0-20181004005441-af9cb2a35e7f/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20221202195650-67e5cbc046fd h1:OjndDrsik+Gt+e6fs45z9AxiewiKyLKYpA45W5Kpkks= -google.golang.org/genproto v0.0.0-20221202195650-67e5cbc046fd/go.mod h1:cTsE614GARnxrLsqKREzmNYJACSWWpAWdNMwnD7c2BE= +google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 h1:KpwkzHKEF7B9Zxg18WzOa7djJ+Ha5DzthMyZYQfEn2A= +google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU= google.golang.org/grpc v0.0.0-20180607172857-7a6a684ca69e/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -739,8 +766,8 @@ google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= -google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= +google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -754,8 +781,9 @@ gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= gopkg.in/go-playground/validator.v8 v8.18.2/go.mod h1:RX2a/7Ha8BgOhfk7j780h4/u/RRjR0eouCJSH80/M2Y= -gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= +gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= +gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= diff --git a/tests/integrations/tso/server_test.go b/tests/integrations/tso/server_test.go new file mode 100644 index 00000000000..518335442f4 --- /dev/null +++ b/tests/integrations/tso/server_test.go @@ -0,0 +1,174 @@ +// Copyright 2020 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tso + +import ( + "context" + "sync" + "testing" + "time" + + "github.com/pingcap/kvproto/pkg/pdpb" + "github.com/pingcap/kvproto/pkg/tsopb" + "github.com/stretchr/testify/suite" + tso "github.com/tikv/pd/pkg/mcs/tso/server" + tsopkg "github.com/tikv/pd/pkg/tso" + "github.com/tikv/pd/pkg/utils/tempurl" + pd "github.com/tikv/pd/pkg/utils/testutil" + "github.com/tikv/pd/tests" + "google.golang.org/grpc" +) + +type tsoServerTestSuite struct { + suite.Suite + legacy bool + + ctx context.Context + cancel context.CancelFunc + + // The PD cluster. + cluster *tests.TestCluster + // pdLeaderServer is the leader server of the PD cluster. + pdLeaderServer *tests.TestServer + // tsoServer is the TSO service provider. + tsoServer *tso.Server + tsoServerCleanup func() + tsoClientConn *grpc.ClientConn + + pdClient pdpb.PDClient + tsoClient tsopb.TSOClient +} + +func TestLegacyTSOServer(t *testing.T) { + suite.Run(t, &tsoServerTestSuite{ + legacy: true, + }) +} + +func TestMicroserviceTSOServer(t *testing.T) { + suite.Run(t, &tsoServerTestSuite{ + legacy: false, + }) +} + +func (suite *tsoServerTestSuite) SetupSuite() { + re := suite.Require() + + var err error + suite.ctx, suite.cancel = context.WithCancel(context.Background()) + if suite.legacy { + suite.cluster, err = tests.NewTestCluster(suite.ctx, serverCount) + } else { + suite.cluster, err = tests.NewTestAPICluster(suite.ctx, serverCount) + } + re.NoError(err) + err = suite.cluster.RunInitialServers() + re.NoError(err) + leaderName := suite.cluster.WaitLeader() + suite.pdLeaderServer = suite.cluster.GetServer(leaderName) + backendEndpoints := suite.pdLeaderServer.GetAddr() + if suite.legacy { + suite.pdClient = pd.MustNewGrpcClient(re, backendEndpoints) + } else { + suite.tsoServer, suite.tsoServerCleanup = tests.StartSingleTSOTestServer(suite.ctx, re, backendEndpoints, tempurl.Alloc()) + suite.tsoClientConn, suite.tsoClient = tso.MustNewGrpcClient(re, suite.tsoServer.GetAddr()) + } +} + +func (suite *tsoServerTestSuite) TearDownSuite() { + suite.cancel() + if !suite.legacy { + suite.tsoClientConn.Close() + suite.tsoServerCleanup() + } + suite.cluster.Destroy() +} + +func (suite *tsoServerTestSuite) getClusterID() uint64 { + if suite.legacy { + return suite.pdLeaderServer.GetServer().ClusterID() + } + return suite.tsoServer.ClusterID() +} + +func (suite *tsoServerTestSuite) resetTS(ts uint64, ignoreSmaller, skipUpperBoundCheck bool) { + var err error + if suite.legacy { + err = suite.pdLeaderServer.GetServer().GetHandler().ResetTS(ts, ignoreSmaller, skipUpperBoundCheck, 0) + } else { + err = suite.tsoServer.GetHandler().ResetTS(ts, ignoreSmaller, skipUpperBoundCheck, 0) + } + // Only this error is acceptable. + if err != nil { + suite.Require().ErrorContains(err, "is smaller than now") + } +} + +func (suite *tsoServerTestSuite) request(ctx context.Context, count uint32) (err error) { + re := suite.Require() + clusterID := suite.getClusterID() + if suite.legacy { + req := &pdpb.TsoRequest{ + Header: &pdpb.RequestHeader{ClusterId: clusterID}, + DcLocation: tsopkg.GlobalDCLocation, + Count: count, + } + tsoClient, err := suite.pdClient.Tso(ctx) + re.NoError(err) + defer tsoClient.CloseSend() + re.NoError(tsoClient.Send(req)) + _, err = tsoClient.Recv() + return err + } + req := &tsopb.TsoRequest{ + Header: &tsopb.RequestHeader{ClusterId: clusterID}, + DcLocation: tsopkg.GlobalDCLocation, + Count: count, + } + tsoClient, err := suite.tsoClient.Tso(ctx) + re.NoError(err) + defer tsoClient.CloseSend() + re.NoError(tsoClient.Send(req)) + _, err = tsoClient.Recv() + return err +} + +func (suite *tsoServerTestSuite) TestConcurrentlyReset() { + var wg sync.WaitGroup + wg.Add(2) + now := time.Now() + for i := 0; i < 2; i++ { + go func() { + defer wg.Done() + for j := 0; j <= 100; j++ { + // Get a copy of now then call base.add, because now is shared by all goroutines + // and now.add() will add to itself which isn't atomic and multi-goroutine safe. + base := now + physical := base.Add(time.Duration(2*j)*time.Minute).UnixNano() / int64(time.Millisecond) + ts := uint64(physical << 18) + suite.resetTS(ts, false, false) + } + }() + } + wg.Wait() +} + +func (suite *tsoServerTestSuite) TestZeroTSOCount() { + re := suite.Require() + ctx, cancel := context.WithCancel(suite.ctx) + defer cancel() + + re.ErrorContains(suite.request(ctx, 0), "tso count should be positive") +} diff --git a/tests/integrations/tso/testutil.go b/tests/integrations/tso/testutil.go new file mode 100644 index 00000000000..2a4e5eabd90 --- /dev/null +++ b/tests/integrations/tso/testutil.go @@ -0,0 +1,40 @@ +// Copyright 2023 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tso + +import ( + "github.com/pingcap/kvproto/pkg/pdpb" + "github.com/stretchr/testify/require" +) + +const ( + serverCount = 3 + tsoRequestConcurrencyNumber = 5 + tsoRequestRound = 300 + tsoCount = 10 +) + +type tsoResponse interface { + GetCount() uint32 + GetTimestamp() *pdpb.Timestamp +} + +func checkAndReturnTimestampResponse[T tsoResponse](re *require.Assertions, resp T) *pdpb.Timestamp { + re.Equal(uint32(tsoCount), resp.GetCount()) + timestamp := resp.GetTimestamp() + re.Greater(timestamp.GetPhysical(), int64(0)) + re.GreaterOrEqual(uint32(timestamp.GetLogical())>>timestamp.GetSuffixBits(), uint32(tsoCount)) + return timestamp +} diff --git a/tests/mcs/resource_manager/resource_manager_test.go b/tests/mcs/resource_manager/resource_manager_test.go deleted file mode 100644 index 1ab91391196..00000000000 --- a/tests/mcs/resource_manager/resource_manager_test.go +++ /dev/null @@ -1,736 +0,0 @@ -// Copyright 2022 TiKV Project Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package resourcemanager_test - -import ( - "context" - "encoding/json" - "fmt" - "io" - "net/http" - "strconv" - "strings" - "testing" - "time" - - "github.com/pingcap/failpoint" - rmpb "github.com/pingcap/kvproto/pkg/resource_manager" - "github.com/stretchr/testify/suite" - pd "github.com/tikv/pd/client" - "github.com/tikv/pd/client/resource_group/controller" - "github.com/tikv/pd/pkg/mcs/resource_manager/server" - "github.com/tikv/pd/pkg/utils/testutil" - "github.com/tikv/pd/tests" - "go.uber.org/goleak" - - // Register Service - _ "github.com/tikv/pd/pkg/mcs/registry" - _ "github.com/tikv/pd/pkg/mcs/resource_manager/server/install" -) - -func TestMain(m *testing.M) { - goleak.VerifyTestMain(m, testutil.LeakOptions...) -} - -type resourceManagerClientTestSuite struct { - suite.Suite - ctx context.Context - clean context.CancelFunc - cluster *tests.TestCluster - client pd.Client - initGroups []*rmpb.ResourceGroup -} - -func TestResourceManagerClientTestSuite(t *testing.T) { - suite.Run(t, new(resourceManagerClientTestSuite)) -} - -func (suite *resourceManagerClientTestSuite) SetupSuite() { - var err error - re := suite.Require() - - suite.ctx, suite.clean = context.WithCancel(context.Background()) - - suite.cluster, err = tests.NewTestCluster(suite.ctx, 2) - re.NoError(err) - - err = suite.cluster.RunInitialServers() - re.NoError(err) - - suite.client, err = pd.NewClientWithContext(suite.ctx, suite.cluster.GetConfig().GetClientURLs(), pd.SecurityOption{}) - re.NoError(err) - leader := suite.cluster.GetServer(suite.cluster.WaitLeader()) - suite.waitLeader(suite.client, leader.GetAddr()) - - suite.initGroups = []*rmpb.ResourceGroup{ - { - Name: "test1", - Mode: rmpb.GroupMode_RUMode, - RUSettings: &rmpb.GroupRequestUnitSettings{ - RU: &rmpb.TokenBucket{ - Settings: &rmpb.TokenLimitSettings{ - FillRate: 10000, - }, - Tokens: 100000, - }, - }, - }, - { - Name: "test2", - Mode: rmpb.GroupMode_RUMode, - RUSettings: &rmpb.GroupRequestUnitSettings{ - RU: &rmpb.TokenBucket{ - Settings: &rmpb.TokenLimitSettings{ - FillRate: 20000, - BurstLimit: -1, - }, - Tokens: 100000, - }, - }, - }, - } -} - -func (suite *resourceManagerClientTestSuite) waitLeader(cli pd.Client, leaderAddr string) { - innerCli, ok := cli.(interface{ GetServiceDiscovery() pd.ServiceDiscovery }) - suite.True(ok) - suite.NotNil(innerCli) - testutil.Eventually(suite.Require(), func() bool { - innerCli.GetServiceDiscovery().ScheduleCheckMemberChanged() - return innerCli.GetServiceDiscovery().GetServingAddr() == leaderAddr - }) -} - -func (suite *resourceManagerClientTestSuite) TearDownSuite() { - suite.client.Close() - suite.cluster.Destroy() - suite.clean() -} - -func (suite *resourceManagerClientTestSuite) cleanupResourceGroups() { - cli := suite.client - groups, err := cli.ListResourceGroups(suite.ctx) - suite.NoError(err) - for _, group := range groups { - deleteResp, err := cli.DeleteResourceGroup(suite.ctx, group.GetName()) - suite.NoError(err) - suite.Contains(deleteResp, "Success!") - } -} - -func (suite *resourceManagerClientTestSuite) resignAndWaitLeader() { - suite.NoError(suite.cluster.ResignLeader()) - newLeader := suite.cluster.GetServer(suite.cluster.WaitLeader()) - suite.NotNil(newLeader) - suite.waitLeader(suite.client, newLeader.GetAddr()) -} - -func (suite *resourceManagerClientTestSuite) TestWatchResourceGroup() { - re := suite.Require() - cli := suite.client - group := &rmpb.ResourceGroup{ - Name: "test", - Mode: rmpb.GroupMode_RUMode, - RUSettings: &rmpb.GroupRequestUnitSettings{ - RU: &rmpb.TokenBucket{ - Settings: &rmpb.TokenLimitSettings{ - FillRate: 10000, - }, - Tokens: 100000, - }, - }, - } - // Mock get revision by listing - for i := 0; i < 3; i++ { - group.Name += strconv.Itoa(i) - resp, err := cli.AddResourceGroup(suite.ctx, group) - group.Name = "test" - re.NoError(err) - re.Contains(resp, "Success!") - } - lresp, err := cli.ListResourceGroups(suite.ctx) - re.NoError(err) - re.Equal(len(lresp), 3) - // Start watcher - watchChan, err := suite.client.WatchResourceGroup(suite.ctx, int64(0)) - suite.NoError(err) - // Mock add resource groups - for i := 3; i < 9; i++ { - group.Name = "test" + strconv.Itoa(i) - resp, err := cli.AddResourceGroup(suite.ctx, group) - re.NoError(err) - re.Contains(resp, "Success!") - } - // Mock modify resource groups - modifySettings := func(gs *rmpb.ResourceGroup) { - gs.RUSettings = &rmpb.GroupRequestUnitSettings{ - RU: &rmpb.TokenBucket{ - Settings: &rmpb.TokenLimitSettings{ - FillRate: 20000, - }, - }, - } - } - for i := 0; i < 9; i++ { - group.Name = "test" + strconv.Itoa(i) - modifySettings(group) - resp, err := cli.ModifyResourceGroup(suite.ctx, group) - re.NoError(err) - re.Contains(resp, "Success!") - } - // Mock delete resource groups - suite.cleanupResourceGroups() - // Check watch result - i := 0 - for { - select { - case <-time.After(time.Second): - return - case res := <-watchChan: - if i < 6 { - for _, r := range res { - suite.Equal(uint64(10000), r.RUSettings.RU.Settings.FillRate) - i++ - } - } else { // after modify - for _, r := range res { - suite.Equal(uint64(20000), r.RUSettings.RU.Settings.FillRate) - i++ - } - } - } - } -} - -const buffDuration = time.Millisecond * 300 - -type testRequestInfo struct { - isWrite bool - writeBytes uint64 -} - -func (ti *testRequestInfo) IsWrite() bool { - return ti.isWrite -} - -func (ti *testRequestInfo) WriteBytes() uint64 { - return ti.writeBytes -} - -type testResponseInfo struct { - cpu time.Duration - readBytes uint64 - succeed bool -} - -func (tri *testResponseInfo) ReadBytes() uint64 { - return tri.readBytes -} - -func (tri *testResponseInfo) KVCPU() time.Duration { - return tri.cpu -} - -func (tri *testResponseInfo) Succeed() bool { - return tri.succeed -} - -type tokenConsumptionPerSecond struct { - rruTokensAtATime float64 - wruTokensAtATime float64 - times int - waitDuration time.Duration -} - -func (t tokenConsumptionPerSecond) makeReadRequest() *testRequestInfo { - return &testRequestInfo{ - isWrite: false, - writeBytes: 0, - } -} - -func (t tokenConsumptionPerSecond) makeWriteRequest() *testRequestInfo { - return &testRequestInfo{ - isWrite: true, - writeBytes: uint64(t.wruTokensAtATime - 1), - } -} - -func (t tokenConsumptionPerSecond) makeReadResponse() *testResponseInfo { - return &testResponseInfo{ - readBytes: uint64((t.rruTokensAtATime - 1) / 2), - cpu: time.Duration(t.rruTokensAtATime/2) * time.Millisecond, - } -} - -func (t tokenConsumptionPerSecond) makeWriteResponse() *testResponseInfo { - return &testResponseInfo{ - readBytes: 0, - cpu: time.Duration(0), - succeed: true, - } -} - -func (suite *resourceManagerClientTestSuite) TestResourceGroupController() { - re := suite.Require() - cli := suite.client - - for _, group := range suite.initGroups { - resp, err := cli.AddResourceGroup(suite.ctx, group) - re.NoError(err) - re.Contains(resp, "Success!") - } - - cfg := &controller.RequestUnitConfig{ - ReadBaseCost: 1, - ReadCostPerByte: 1, - WriteBaseCost: 1, - WriteCostPerByte: 1, - CPUMsCost: 1, - } - - controller, _ := controller.NewResourceGroupController(suite.ctx, 1, cli, cfg, controller.EnableSingleGroupByKeyspace()) - controller.Start(suite.ctx) - - testCases := []struct { - resourceGroupName string - tcs []tokenConsumptionPerSecond - len int - }{ - { - resourceGroupName: suite.initGroups[0].Name, - len: 8, - tcs: []tokenConsumptionPerSecond{ - {rruTokensAtATime: 50, wruTokensAtATime: 20, times: 100, waitDuration: 0}, - {rruTokensAtATime: 50, wruTokensAtATime: 100, times: 100, waitDuration: 0}, - {rruTokensAtATime: 50, wruTokensAtATime: 100, times: 100, waitDuration: 0}, - {rruTokensAtATime: 20, wruTokensAtATime: 40, times: 250, waitDuration: 0}, - {rruTokensAtATime: 25, wruTokensAtATime: 50, times: 200, waitDuration: 0}, - {rruTokensAtATime: 30, wruTokensAtATime: 60, times: 165, waitDuration: 0}, - {rruTokensAtATime: 40, wruTokensAtATime: 80, times: 125, waitDuration: 0}, - {rruTokensAtATime: 50, wruTokensAtATime: 100, times: 100, waitDuration: 0}, - }, - }, - } - tricker := time.NewTicker(time.Second) - defer tricker.Stop() - i := 0 - for { - v := false - <-tricker.C - for _, cas := range testCases { - if i >= cas.len { - continue - } - v = true - sum := time.Duration(0) - for j := 0; j < cas.tcs[i].times; j++ { - rreq := cas.tcs[i].makeReadRequest() - wreq := cas.tcs[i].makeWriteRequest() - rres := cas.tcs[i].makeReadResponse() - wres := cas.tcs[i].makeWriteResponse() - startTime := time.Now() - controller.OnRequestWait(suite.ctx, cas.resourceGroupName, rreq) - controller.OnRequestWait(suite.ctx, cas.resourceGroupName, wreq) - sum += time.Since(startTime) - controller.OnResponse(suite.ctx, cas.resourceGroupName, rreq, rres) - controller.OnResponse(suite.ctx, cas.resourceGroupName, wreq, wres) - time.Sleep(1000 * time.Microsecond) - } - re.LessOrEqual(sum, buffDuration+cas.tcs[i].waitDuration) - } - i++ - if !v { - break - } - } - suite.cleanupResourceGroups() - controller.Stop() -} - -func (suite *resourceManagerClientTestSuite) TestAcquireTokenBucket() { - re := suite.Require() - cli := suite.client - - groups := make([]*rmpb.ResourceGroup, 0) - groups = append(groups, suite.initGroups...) - for _, group := range groups { - resp, err := cli.AddResourceGroup(suite.ctx, group) - re.NoError(err) - re.Contains(resp, "Success!") - } - reqs := &rmpb.TokenBucketsRequest{ - Requests: make([]*rmpb.TokenBucketRequest, 0), - TargetRequestPeriodMs: uint64(time.Second * 10 / time.Millisecond), - } - re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/mcs/resource_manager/server/fastPersist", `return(true)`)) - suite.resignAndWaitLeader() - groups = append(groups, &rmpb.ResourceGroup{Name: "test3"}) - for i := 0; i < 3; i++ { - for _, group := range groups { - requests := make([]*rmpb.RequestUnitItem, 0) - requests = append(requests, &rmpb.RequestUnitItem{ - Type: rmpb.RequestUnitType_RU, - Value: 30000, - }) - req := &rmpb.TokenBucketRequest{ - ResourceGroupName: group.Name, - Request: &rmpb.TokenBucketRequest_RuItems{ - RuItems: &rmpb.TokenBucketRequest_RequestRU{ - RequestRU: requests, - }, - }, - } - reqs.Requests = append(reqs.Requests, req) - } - aresp, err := cli.AcquireTokenBuckets(suite.ctx, reqs) - re.NoError(err) - for _, resp := range aresp { - re.Len(resp.GrantedRUTokens, 1) - re.Equal(resp.GrantedRUTokens[0].GrantedTokens.Tokens, float64(30000.)) - if resp.ResourceGroupName == "test2" { - re.Equal(int64(-1), resp.GrantedRUTokens[0].GrantedTokens.GetSettings().GetBurstLimit()) - } - } - gresp, err := cli.GetResourceGroup(suite.ctx, groups[0].GetName()) - re.NoError(err) - re.Less(gresp.RUSettings.RU.Tokens, groups[0].RUSettings.RU.Tokens) - - checkFunc := func(g1 *rmpb.ResourceGroup, g2 *rmpb.ResourceGroup) { - re.Equal(g1.GetName(), g2.GetName()) - re.Equal(g1.GetMode(), g2.GetMode()) - re.Equal(g1.GetRUSettings().RU.Settings.FillRate, g2.GetRUSettings().RU.Settings.FillRate) - // now we don't persistent tokens in running state, so tokens is original. - re.Less(g1.GetRUSettings().RU.Tokens, g2.GetRUSettings().RU.Tokens) - re.NoError(err) - } - time.Sleep(250 * time.Millisecond) - // to test persistent - suite.resignAndWaitLeader() - gresp, err = cli.GetResourceGroup(suite.ctx, groups[0].GetName()) - re.NoError(err) - checkFunc(gresp, groups[0]) - } - re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/mcs/resource_manager/server/fastPersist")) - suite.cleanupResourceGroups() -} - -func (suite *resourceManagerClientTestSuite) TestBasicResourceGroupCURD() { - re := suite.Require() - cli := suite.client - testCasesSet1 := []struct { - name string - mode rmpb.GroupMode - addSuccess bool - modifySuccess bool - expectMarshal string - modifySettings func(*rmpb.ResourceGroup) - }{ - {"test1", rmpb.GroupMode_RUMode, true, true, - `{"name":"test1","mode":1,"r_u_settings":{"r_u":{"settings":{"fill_rate":10000},"state":{"initialized":false}}}}`, - func(gs *rmpb.ResourceGroup) { - gs.RUSettings = &rmpb.GroupRequestUnitSettings{ - RU: &rmpb.TokenBucket{ - Settings: &rmpb.TokenLimitSettings{ - FillRate: 10000, - }, - }, - } - }, - }, - - {"test2", rmpb.GroupMode_RUMode, true, true, - `{"name":"test2","mode":1,"r_u_settings":{"r_u":{"settings":{"fill_rate":20000},"state":{"initialized":false}}}}`, - func(gs *rmpb.ResourceGroup) { - gs.RUSettings = &rmpb.GroupRequestUnitSettings{ - RU: &rmpb.TokenBucket{ - Settings: &rmpb.TokenLimitSettings{ - FillRate: 20000, - }, - }, - } - }, - }, - {"test2", rmpb.GroupMode_RUMode, false, true, - `{"name":"test2","mode":1,"r_u_settings":{"r_u":{"settings":{"fill_rate":30000,"burst_limit":-1},"state":{"initialized":false}}}}`, - func(gs *rmpb.ResourceGroup) { - gs.RUSettings = &rmpb.GroupRequestUnitSettings{ - RU: &rmpb.TokenBucket{ - Settings: &rmpb.TokenLimitSettings{ - FillRate: 30000, - BurstLimit: -1, - }, - }, - } - }, - }, - {"test3", rmpb.GroupMode_RawMode, true, false, - `{"name":"test3","mode":2}`, - func(gs *rmpb.ResourceGroup) { - gs.RUSettings = &rmpb.GroupRequestUnitSettings{ - RU: &rmpb.TokenBucket{ - Settings: &rmpb.TokenLimitSettings{ - FillRate: 10000, - }, - }, - } - }, - }, - {"test3", rmpb.GroupMode_RawMode, false, true, - `{"name":"test3","mode":2,"raw_resource_settings":{"cpu":{"settings":{"fill_rate":1000000},"state":{"initialized":false}},"io_read_bandwidth":{"state":{"initialized":false}},"io_write_bandwidth":{"state":{"initialized":false}}}}`, - func(gs *rmpb.ResourceGroup) { - gs.RawResourceSettings = &rmpb.GroupRawResourceSettings{ - Cpu: &rmpb.TokenBucket{ - Settings: &rmpb.TokenLimitSettings{ - FillRate: 1000000, - }, - }, - } - }, - }, - } - - checkErr := func(err error, success bool) { - if success { - re.NoError(err) - } else { - re.Error(err) - } - } - - finalNum := 0 - // Test Resource Group CURD via gRPC - for i, tcase := range testCasesSet1 { - group := &rmpb.ResourceGroup{ - Name: tcase.name, - Mode: tcase.mode, - } - // Create Resource Group - resp, err := cli.AddResourceGroup(suite.ctx, group) - checkErr(err, tcase.addSuccess) - if tcase.addSuccess { - finalNum++ - re.Contains(resp, "Success!") - } - - // Modify Resource Group - tcase.modifySettings(group) - mresp, err := cli.ModifyResourceGroup(suite.ctx, group) - checkErr(err, tcase.modifySuccess) - if tcase.modifySuccess { - re.Contains(mresp, "Success!") - } - - // Get Resource Group - gresp, err := cli.GetResourceGroup(suite.ctx, tcase.name) - re.NoError(err) - re.Equal(tcase.name, gresp.Name) - if tcase.modifySuccess { - re.Equal(group, gresp) - } - - // Last one, Check list and delete all resource groups - if i == len(testCasesSet1)-1 { - // List Resource Groups - lresp, err := cli.ListResourceGroups(suite.ctx) - re.NoError(err) - re.Equal(finalNum, len(lresp)) - - for _, g := range lresp { - // Delete Resource Group - dresp, err := cli.DeleteResourceGroup(suite.ctx, g.Name) - re.NoError(err) - re.Contains(dresp, "Success!") - _, err = cli.GetResourceGroup(suite.ctx, g.Name) - re.EqualError(err, fmt.Sprintf("get resource group %v failed, rpc error: code = Unknown desc = resource group not found", g.Name)) - } - - // to test the deletion of persistence - suite.resignAndWaitLeader() - // List Resource Group - lresp, err = cli.ListResourceGroups(suite.ctx) - re.NoError(err) - re.Equal(0, len(lresp)) - } - } - - // Test Resource Group CURD via HTTP - finalNum = 0 - getAddr := func(i int) string { - server := suite.cluster.GetServer(suite.cluster.GetLeader()) - if i%2 == 1 { - server = suite.cluster.GetServer(suite.cluster.GetFollower()) - } - return server.GetAddr() - } - for i, tcase := range testCasesSet1 { - // Create Resource Group - group := &rmpb.ResourceGroup{ - Name: tcase.name, - Mode: tcase.mode, - } - createJSON, err := json.Marshal(group) - re.NoError(err) - resp, err := http.Post(getAddr(i)+"/resource-manager/api/v1/config/group", "application/json", strings.NewReader(string(createJSON))) - re.NoError(err) - defer resp.Body.Close() - if tcase.addSuccess { - re.Equal(http.StatusOK, resp.StatusCode) - finalNum++ - } else { - re.Equal(http.StatusInternalServerError, resp.StatusCode) - } - - // Modify Resource Group - tcase.modifySettings(group) - modifyJSON, err := json.Marshal(group) - re.NoError(err) - req, err := http.NewRequest(http.MethodPut, getAddr(i+1)+"/resource-manager/api/v1/config/group", strings.NewReader(string(modifyJSON))) - re.NoError(err) - req.Header.Set("Content-Type", "application/json") - resp, err = http.DefaultClient.Do(req) - re.NoError(err) - defer resp.Body.Close() - if tcase.modifySuccess { - re.Equal(http.StatusOK, resp.StatusCode) - } else { - re.Equal(http.StatusInternalServerError, resp.StatusCode) - } - - // Get Resource Group - resp, err = http.Get(getAddr(i) + "/resource-manager/api/v1/config/group/" + tcase.name) - re.NoError(err) - defer resp.Body.Close() - re.Equal(http.StatusOK, resp.StatusCode) - respString, err := io.ReadAll(resp.Body) - re.NoError(err) - re.Contains(string(respString), tcase.name) - if tcase.modifySuccess { - re.Equal(string(respString), tcase.expectMarshal) - } - - // Last one, Check list and delete all resource groups - if i == len(testCasesSet1)-1 { - resp, err := http.Get(getAddr(i) + "/resource-manager/api/v1/config/groups") - re.NoError(err) - defer resp.Body.Close() - re.Equal(http.StatusOK, resp.StatusCode) - respString, err := io.ReadAll(resp.Body) - re.NoError(err) - groups := make([]*server.ResourceGroup, 0) - json.Unmarshal(respString, &groups) - re.Equal(finalNum, len(groups)) - - // Delete all resource groups - for _, g := range groups { - req, err := http.NewRequest(http.MethodDelete, getAddr(i+1)+"/resource-manager/api/v1/config/group/"+g.Name, nil) - re.NoError(err) - resp, err := http.DefaultClient.Do(req) - re.NoError(err) - defer resp.Body.Close() - re.Equal(http.StatusOK, resp.StatusCode) - respString, err := io.ReadAll(resp.Body) - re.NoError(err) - re.Contains(string(respString), "Success!") - } - - // verify again - resp1, err := http.Get(getAddr(i) + "/resource-manager/api/v1/config/groups") - re.NoError(err) - defer resp1.Body.Close() - re.Equal(http.StatusOK, resp1.StatusCode) - respString1, err := io.ReadAll(resp1.Body) - re.NoError(err) - groups1 := make([]server.ResourceGroup, 0) - json.Unmarshal(respString1, &groups1) - re.Equal(0, len(groups1)) - } - } -} - -func (suite *resourceManagerClientTestSuite) TestResourceManagerClientFailover() { - re := suite.Require() - cli := suite.client - - group := &rmpb.ResourceGroup{ - Name: "test3", - Mode: rmpb.GroupMode_RUMode, - RUSettings: &rmpb.GroupRequestUnitSettings{ - RU: &rmpb.TokenBucket{ - Settings: &rmpb.TokenLimitSettings{ - FillRate: 10000, - }, - Tokens: 100000, - }, - }, - } - addResp, err := cli.AddResourceGroup(suite.ctx, group) - re.NoError(err) - re.Contains(addResp, "Success!") - getResp, err := cli.GetResourceGroup(suite.ctx, group.GetName()) - re.NoError(err) - re.NotNil(getResp) - re.Equal(*group, *getResp) - - // Change the leader after each time we modify the resource group. - for i := 0; i < 4; i++ { - group.RUSettings.RU.Settings.FillRate += uint64(i) - modifyResp, err := cli.ModifyResourceGroup(suite.ctx, group) - re.NoError(err) - re.Contains(modifyResp, "Success!") - suite.resignAndWaitLeader() - getResp, err = cli.GetResourceGroup(suite.ctx, group.GetName()) - re.NoError(err) - re.NotNil(getResp) - re.Equal(group.RUSettings.RU.Settings.FillRate, getResp.RUSettings.RU.Settings.FillRate) - } - - // Cleanup the resource group. - suite.cleanupResourceGroups() -} - -func (suite *resourceManagerClientTestSuite) TestLoadRequestUnitConfig() { - re := suite.Require() - cli := suite.client - // Test load from resource manager. - ctr, err := controller.NewResourceGroupController(suite.ctx, 1, cli, nil) - re.NoError(err) - config := ctr.GetConfig() - re.NotNil(config) - expectedConfig := controller.DefaultConfig() - re.Equal(expectedConfig.ReadBaseCost, config.ReadBaseCost) - re.Equal(expectedConfig.ReadBytesCost, config.ReadBytesCost) - re.Equal(expectedConfig.WriteBaseCost, config.WriteBaseCost) - re.Equal(expectedConfig.WriteBytesCost, config.WriteBytesCost) - re.Equal(expectedConfig.CPUMsCost, config.CPUMsCost) - // Test init from given config. - ruConfig := &controller.RequestUnitConfig{ - ReadBaseCost: 1, - ReadCostPerByte: 2, - WriteBaseCost: 3, - WriteCostPerByte: 4, - CPUMsCost: 5, - } - ctr, err = controller.NewResourceGroupController(suite.ctx, 1, cli, ruConfig) - re.NoError(err) - config = ctr.GetConfig() - re.NotNil(config) - expectedConfig = controller.GenerateConfig(ruConfig) - re.Equal(expectedConfig.ReadBaseCost, config.ReadBaseCost) - re.Equal(expectedConfig.ReadBytesCost, config.ReadBytesCost) - re.Equal(expectedConfig.WriteBaseCost, config.WriteBaseCost) - re.Equal(expectedConfig.WriteBytesCost, config.WriteBytesCost) - re.Equal(expectedConfig.CPUMsCost, config.CPUMsCost) -} diff --git a/tests/mcs/tso/server_test.go b/tests/mcs/tso/server_test.go deleted file mode 100644 index 5325cd573ce..00000000000 --- a/tests/mcs/tso/server_test.go +++ /dev/null @@ -1,176 +0,0 @@ -// Copyright 2023 TiKV Project Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package tso - -import ( - "context" - "fmt" - "net/http" - "strings" - "testing" - "time" - - "github.com/stretchr/testify/require" - "github.com/stretchr/testify/suite" - "github.com/tikv/pd/pkg/mcs/discovery" - tsosvr "github.com/tikv/pd/pkg/mcs/tso/server" - tsoapi "github.com/tikv/pd/pkg/mcs/tso/server/apis/v1" - "github.com/tikv/pd/pkg/utils/etcdutil" - "github.com/tikv/pd/pkg/utils/testutil" - "github.com/tikv/pd/pkg/utils/tsoutil" - "github.com/tikv/pd/tests" - "go.etcd.io/etcd/clientv3" - "go.uber.org/goleak" - "google.golang.org/grpc" -) - -func TestMain(m *testing.M) { - goleak.VerifyTestMain(m, testutil.LeakOptions...) -} - -type tsoServerTestSuite struct { - suite.Suite - ctx context.Context - cancel context.CancelFunc - cluster *tests.TestCluster - pdLeader *tests.TestServer - backendEndpoints string - tsosvrs map[string]*tsosvr.Server -} - -func TestTSOServerTestSuite(t *testing.T) { - suite.Run(t, new(tsoServerTestSuite)) -} - -func (suite *tsoServerTestSuite) SetupSuite() { - var err error - re := suite.Require() - - suite.ctx, suite.cancel = context.WithCancel(context.Background()) - suite.cluster, err = tests.NewTestCluster(suite.ctx, 1) - re.NoError(err) - - err = suite.cluster.RunInitialServers() - re.NoError(err) - - leaderName := suite.cluster.WaitLeader() - suite.pdLeader = suite.cluster.GetServer(leaderName) - suite.backendEndpoints = suite.pdLeader.GetAddr() -} - -func (suite *tsoServerTestSuite) TearDownSuite() { - for _, s := range suite.tsosvrs { - s.Close() - testutil.CleanServer(s.GetConfig().DataDir) - } - suite.cluster.Destroy() - suite.cancel() -} - -func (suite *tsoServerTestSuite) TestTSOServerStartAndStopNormally() { - defer func() { - if r := recover(); r != nil { - fmt.Println("Recovered from an unexpected panic", r) - suite.T().Errorf("Expected no panic, but something bad occurred with") - } - }() - - re := suite.Require() - s, cleanup, err := startSingleTSOTestServer(suite.ctx, re, suite.backendEndpoints) - re.NoError(err) - - defer cleanup() - testutil.Eventually(re, func() bool { - return s.IsServing() - }, testutil.WithWaitFor(5*time.Second), testutil.WithTickInterval(50*time.Millisecond)) - - // Test registered GRPC Service - cc, err := grpc.DialContext(suite.ctx, s.GetListenURL().Host, grpc.WithInsecure()) - re.NoError(err) - cc.Close() - url := s.GetConfig().ListenAddr + tsoapi.APIPathPrefix - { - resetJSON := `{"tso":"121312", "force-use-larger":true}` - re.NoError(err) - resp, err := http.Post(url+"/admin/reset-ts", "application/json", strings.NewReader(resetJSON)) - re.NoError(err) - defer resp.Body.Close() - re.Equal(http.StatusOK, resp.StatusCode) - } - { - resetJSON := `{}` - re.NoError(err) - resp, err := http.Post(url+"/admin/reset-ts", "application/json", strings.NewReader(resetJSON)) - re.NoError(err) - defer resp.Body.Close() - re.Equal(http.StatusBadRequest, resp.StatusCode) - } -} - -func (suite *tsoServerTestSuite) TestTSOServerRegister() { - re := suite.Require() - s, cleanup, err := startSingleTSOTestServer(suite.ctx, re, suite.backendEndpoints) - re.NoError(err) - - serviceName := "tso" - client := suite.pdLeader.GetEtcdClient() - endpoints, err := discovery.Discover(client, serviceName) - re.NoError(err) - re.Equal(s.GetConfig().ListenAddr, endpoints[0]) - - // test API server discovery - exist, addr, err := suite.pdLeader.GetServer().GetServicePrimaryAddr(suite.ctx, serviceName) - re.NoError(err) - re.True(exist) - re.Equal(s.GetConfig().ListenAddr, addr) - - cleanup() - endpoints, err = discovery.Discover(client, serviceName) - re.NoError(err) - re.Empty(endpoints) -} - -func (suite *tsoServerTestSuite) TestTSOPath() { - re := suite.Require() - - client := suite.pdLeader.GetEtcdClient() - re.Equal(1, getEtcdTimestampKeyNum(re, client)) - - _, cleanup, err := startSingleTSOTestServer(suite.ctx, re, suite.backendEndpoints) - re.NoError(err) - defer cleanup() - - cli := setupCli(re, suite.ctx, []string{suite.backendEndpoints}) - physical, logical, err := cli.GetTS(suite.ctx) - re.NoError(err) - ts := tsoutil.ComposeTS(physical, logical) - re.NotEmpty(ts) - // After we request the tso server, etcd still has only one key related to the timestamp. - re.Equal(1, getEtcdTimestampKeyNum(re, client)) -} - -func getEtcdTimestampKeyNum(re *require.Assertions, client *clientv3.Client) int { - resp, err := etcdutil.EtcdKVGet(client, "/", clientv3.WithPrefix()) - re.NoError(err) - var count int - for _, kv := range resp.Kvs { - key := strings.TrimSpace(string(kv.Key)) - if !strings.HasSuffix(key, "timestamp") { - continue - } - count++ - } - return count -} diff --git a/tests/mcs/tso/testutil.go b/tests/mcs/tso/testutil.go deleted file mode 100644 index 5b07db1bcc9..00000000000 --- a/tests/mcs/tso/testutil.go +++ /dev/null @@ -1,89 +0,0 @@ -// Copyright 2023 TiKV Project Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package tso - -import ( - "context" - "os" - "time" - - "github.com/pingcap/log" - "github.com/spf13/cobra" - "github.com/stretchr/testify/require" - pd "github.com/tikv/pd/client" - "github.com/tikv/pd/client/testutil" - tsosvr "github.com/tikv/pd/pkg/mcs/tso/server" - "github.com/tikv/pd/pkg/utils/logutil" - "github.com/tikv/pd/pkg/utils/tempurl" -) - -// CleanupFunc closes test tso server(s) and deletes any files left behind. -type CleanupFunc func() - -// newTSOTestServer creates a tso server for testing. -func newTSOTestServer(ctx context.Context, re *require.Assertions, cfg *tsosvr.Config) (*tsosvr.Server, CleanupFunc, error) { - // New zap logger - err := logutil.SetupLogger(cfg.Log, &cfg.Logger, &cfg.LogProps, cfg.Security.RedactInfoLog) - re.NoError(err) - log.ReplaceGlobals(cfg.Logger, cfg.LogProps) - // Flushing any buffered log entries - defer log.Sync() - - s := tsosvr.CreateServer(ctx, cfg) - if err = s.Run(); err != nil { - return nil, nil, err - } - - cleanup := func() { - s.Close() - os.RemoveAll(cfg.DataDir) - } - return s, cleanup, nil -} - -// newTSOTestDefaultConfig is only for test to create one pd. -// Because PD client also needs this, so export here. -func newTSOTestDefaultConfig() (*tsosvr.Config, error) { - cmd := &cobra.Command{ - Use: "tso", - Short: "Run the tso service", - } - cfg := tsosvr.NewConfig() - flagSet := cmd.Flags() - return cfg, cfg.Parse(flagSet) -} - -// startTSOTestServer creates and starts a tso server with default config for testing. -func startSingleTSOTestServer(ctx context.Context, re *require.Assertions, backendEndpoints string) (*tsosvr.Server, CleanupFunc, error) { - cfg, err := newTSOTestDefaultConfig() - re.NoError(err) - cfg.BackendEndpoints = backendEndpoints - cfg.ListenAddr = tempurl.Alloc() - - s, cleanup, err := newTSOTestServer(ctx, re, cfg) - re.NoError(err) - testutil.Eventually(re, func() bool { - return s.IsServing() - }, testutil.WithWaitFor(5*time.Second), testutil.WithTickInterval(50*time.Millisecond)) - - return s, cleanup, err -} - -func setupCli(re *require.Assertions, ctx context.Context, endpoints []string, opts ...pd.ClientOption) pd.Client { - // TODO: we use keyspace 0 as the default keyspace for now, which mightn't need change in the future - cli, err := pd.NewTSOClientWithContext(ctx, 0, endpoints, pd.SecurityOption{}, opts...) - re.NoError(err) - return cli -} diff --git a/tests/mcs/tso/tso_service_test.go b/tests/mcs/tso/tso_service_test.go deleted file mode 100644 index 819fda4072f..00000000000 --- a/tests/mcs/tso/tso_service_test.go +++ /dev/null @@ -1,102 +0,0 @@ -// Copyright 2023 TiKV Project Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package tso - -import ( - "context" - "strings" - "sync" - "testing" - - "github.com/stretchr/testify/suite" - tsosvr "github.com/tikv/pd/pkg/mcs/tso/server" - "github.com/tikv/pd/pkg/utils/tsoutil" - "github.com/tikv/pd/tests" -) - -const ( - tsoRequestConcurrencyNumber = 1 - tsoRequestRound = 30 -) - -type tsoServiceTestSuite struct { - suite.Suite - ctx context.Context - cancel context.CancelFunc - cluster *tests.TestCluster - pdLeader *tests.TestServer - backendEndpoints string - tsoSvr1 *tsosvr.Server - tsoSvrCleanup1 CleanupFunc -} - -func TestTSOServiceTestSuite(t *testing.T) { - suite.Run(t, new(tsoServiceTestSuite)) -} - -func (suite *tsoServiceTestSuite) SetupSuite() { - var err error - re := suite.Require() - - suite.ctx, suite.cancel = context.WithCancel(context.Background()) - suite.cluster, err = tests.NewTestCluster(suite.ctx, 1) - re.NoError(err) - - err = suite.cluster.RunInitialServers() - re.NoError(err) - - leaderName := suite.cluster.WaitLeader() - suite.pdLeader = suite.cluster.GetServer(leaderName) - suite.backendEndpoints = suite.pdLeader.GetAddr() - - suite.tsoSvr1, suite.tsoSvrCleanup1, err = startSingleTSOTestServer(suite.ctx, re, suite.backendEndpoints) - re.NoError(err) -} - -func (suite *tsoServiceTestSuite) TearDownSuite() { - suite.tsoSvrCleanup1() - suite.cluster.Destroy() - suite.cancel() -} - -func (suite *tsoServiceTestSuite) TestTSOServerRegister() { - re := suite.Require() - - endpoints := strings.Split(suite.backendEndpoints, ",") - cli1 := setupCli(re, suite.ctx, endpoints) - cli2 := setupCli(re, suite.ctx, endpoints) - - var wg sync.WaitGroup - wg.Add(tsoRequestConcurrencyNumber) - for i := 0; i < tsoRequestConcurrencyNumber; i++ { - go func() { - defer wg.Done() - var lastTS uint64 - for i := 0; i < tsoRequestRound; i++ { - physical, logical, err := cli1.GetTS(context.Background()) - re.NoError(err) - ts := tsoutil.ComposeTS(physical, logical) - re.Less(lastTS, ts) - lastTS = ts - physical, logical, err = cli2.GetTS(context.Background()) - re.NoError(err) - ts = tsoutil.ComposeTS(physical, logical) - re.Less(lastTS, ts) - lastTS = ts - } - }() - } - wg.Wait() -} diff --git a/tests/pdctl/config/config_test.go b/tests/pdctl/config/config_test.go index cd9b94b2c60..087ec2ca39e 100644 --- a/tests/pdctl/config/config_test.go +++ b/tests/pdctl/config/config_test.go @@ -25,6 +25,7 @@ import ( "github.com/coreos/go-semver/semver" "github.com/pingcap/kvproto/pkg/metapb" "github.com/stretchr/testify/require" + sc "github.com/tikv/pd/pkg/schedule/config" "github.com/tikv/pd/pkg/schedule/placement" "github.com/tikv/pd/pkg/utils/typeutil" "github.com/tikv/pd/server/config" @@ -36,10 +37,10 @@ import ( type testCase struct { name string value interface{} - read func(scheduleConfig *config.ScheduleConfig) interface{} + read func(scheduleConfig *sc.ScheduleConfig) interface{} } -func (t *testCase) judge(re *require.Assertions, scheduleConfigs ...*config.ScheduleConfig) { +func (t *testCase) judge(re *require.Assertions, scheduleConfigs ...*sc.ScheduleConfig) { value := t.value for _, scheduleConfig := range scheduleConfigs { re.NotNil(scheduleConfig) @@ -87,7 +88,6 @@ func TestConfig(t *testing.T) { scheduleConfig.EnableMakeUpReplica = false scheduleConfig.EnableRemoveExtraReplica = false scheduleConfig.EnableLocationReplacement = false - scheduleConfig.StoreLimitMode = "" re.Equal(uint64(0), scheduleConfig.MaxMergeRegionKeys) // The result of config show doesn't be 0. scheduleConfig.MaxMergeRegionKeys = scheduleConfig.GetMaxMergeRegionKeys() @@ -113,7 +113,7 @@ func TestConfig(t *testing.T) { args = []string{"-u", pdAddr, "config", "show", "schedule"} output, err = pdctl.ExecuteCommand(cmd, args...) re.NoError(err) - scheduleCfg := config.ScheduleConfig{} + scheduleCfg := sc.ScheduleConfig{} re.NoError(json.Unmarshal(output, &scheduleCfg)) scheduleConfig = svr.GetScheduleConfig() scheduleConfig.MaxMergeRegionKeys = scheduleConfig.GetMaxMergeRegionKeys() @@ -136,11 +136,21 @@ func TestConfig(t *testing.T) { re.Equal(20*10000, int(svr.GetScheduleConfig().MaxMergeRegionKeys)) re.Equal(20*10000, int(svr.GetScheduleConfig().GetMaxMergeRegionKeys())) + // set store limit v2 + args = []string{"-u", pdAddr, "config", "set", "store-limit-version", "v2"} + _, err = pdctl.ExecuteCommand(cmd, args...) + re.NoError(err) + re.Equal("v2", svr.GetScheduleConfig().StoreLimitVersion) + args = []string{"-u", pdAddr, "config", "set", "store-limit-version", "v1"} + _, err = pdctl.ExecuteCommand(cmd, args...) + re.NoError(err) + re.Equal("v1", svr.GetScheduleConfig().StoreLimitVersion) + // config show replication args = []string{"-u", pdAddr, "config", "show", "replication"} output, err = pdctl.ExecuteCommand(cmd, args...) re.NoError(err) - replicationCfg := config.ReplicationConfig{} + replicationCfg := sc.ReplicationConfig{} re.NoError(json.Unmarshal(output, &replicationCfg)) re.Equal(svr.GetReplicationConfig(), &replicationCfg) @@ -212,20 +222,20 @@ func TestConfig(t *testing.T) { // test config read and write testCases := []testCase{ - {"leader-schedule-limit", uint64(64), func(scheduleConfig *config.ScheduleConfig) interface{} { + {"leader-schedule-limit", uint64(64), func(scheduleConfig *sc.ScheduleConfig) interface{} { return scheduleConfig.LeaderScheduleLimit - }}, {"hot-region-schedule-limit", uint64(64), func(scheduleConfig *config.ScheduleConfig) interface{} { + }}, {"hot-region-schedule-limit", uint64(64), func(scheduleConfig *sc.ScheduleConfig) interface{} { return scheduleConfig.HotRegionScheduleLimit - }}, {"hot-region-cache-hits-threshold", uint64(5), func(scheduleConfig *config.ScheduleConfig) interface{} { + }}, {"hot-region-cache-hits-threshold", uint64(5), func(scheduleConfig *sc.ScheduleConfig) interface{} { return scheduleConfig.HotRegionCacheHitsThreshold - }}, {"enable-remove-down-replica", false, func(scheduleConfig *config.ScheduleConfig) interface{} { + }}, {"enable-remove-down-replica", false, func(scheduleConfig *sc.ScheduleConfig) interface{} { return scheduleConfig.EnableRemoveDownReplica }}, - {"enable-debug-metrics", true, func(scheduleConfig *config.ScheduleConfig) interface{} { + {"enable-debug-metrics", true, func(scheduleConfig *sc.ScheduleConfig) interface{} { return scheduleConfig.EnableDebugMetrics }}, // set again - {"enable-debug-metrics", true, func(scheduleConfig *config.ScheduleConfig) interface{} { + {"enable-debug-metrics", true, func(scheduleConfig *sc.ScheduleConfig) interface{} { return scheduleConfig.EnableDebugMetrics }}, } @@ -411,6 +421,9 @@ func TestPlacementRuleGroups(t *testing.T) { output, err = pdctl.ExecuteCommand(cmd, "-u", pdAddr, "config", "placement-rules", "rule-group", "set", "group2", "100", "false") re.NoError(err) re.Contains(string(output), "Success!") + output, err = pdctl.ExecuteCommand(cmd, "-u", pdAddr, "config", "placement-rules", "rule-group", "set", "group3", "200", "false") + re.NoError(err) + re.Contains(string(output), "Success!") // show all var groups []placement.RuleGroup @@ -420,17 +433,26 @@ func TestPlacementRuleGroups(t *testing.T) { re.Equal([]placement.RuleGroup{ {ID: "pd", Index: 42, Override: true}, {ID: "group2", Index: 100, Override: false}, + {ID: "group3", Index: 200, Override: false}, }, groups) // delete output, err = pdctl.ExecuteCommand(cmd, "-u", pdAddr, "config", "placement-rules", "rule-group", "delete", "group2") re.NoError(err) - re.Contains(string(output), "Success!") + re.Contains(string(output), "Delete group and rules successfully.") // show again output, err = pdctl.ExecuteCommand(cmd, "-u", pdAddr, "config", "placement-rules", "rule-group", "show", "group2") re.NoError(err) re.Contains(string(output), "404") + + // delete using regex + _, err = pdctl.ExecuteCommand(cmd, "-u", pdAddr, "config", "placement-rules", "rule-group", "delete", "--regexp", ".*3") + re.NoError(err) + + _, err = pdctl.ExecuteCommand(cmd, "-u", pdAddr, "config", "placement-rules", "rule-group", "show", "group3") + re.NoError(err) + re.Contains(string(output), "404") } func TestPlacementRuleBundle(t *testing.T) { @@ -657,7 +679,7 @@ func TestUpdateDefaultReplicaConfig(t *testing.T) { args := []string{"-u", pdAddr, "config", "show", "replication"} output, err := pdctl.ExecuteCommand(cmd, args...) re.NoError(err) - replicationCfg := config.ReplicationConfig{} + replicationCfg := sc.ReplicationConfig{} re.NoError(json.Unmarshal(output, &replicationCfg)) re.Equal(expect, replicationCfg.MaxReplicas) } @@ -666,7 +688,7 @@ func TestUpdateDefaultReplicaConfig(t *testing.T) { args := []string{"-u", pdAddr, "config", "show", "replication"} output, err := pdctl.ExecuteCommand(cmd, args...) re.NoError(err) - replicationCfg := config.ReplicationConfig{} + replicationCfg := sc.ReplicationConfig{} re.NoError(json.Unmarshal(output, &replicationCfg)) re.Len(replicationCfg.LocationLabels, expect) } diff --git a/tests/pdctl/helper.go b/tests/pdctl/helper.go index 2b958a6d41f..d7d6a858497 100644 --- a/tests/pdctl/helper.go +++ b/tests/pdctl/helper.go @@ -20,6 +20,7 @@ import ( "fmt" "sort" + "github.com/docker/go-units" "github.com/pingcap/kvproto/pkg/metapb" "github.com/pingcap/kvproto/pkg/pdpb" "github.com/spf13/cobra" @@ -35,7 +36,7 @@ import ( // ExecuteCommand is used for test purpose. func ExecuteCommand(root *cobra.Command, args ...string) (output []byte, err error) { buf := new(bytes.Buffer) - root.SetOutput(buf) + root.SetOut(buf) root.SetArgs(args) err = root.Execute() return buf.Bytes(), err @@ -101,6 +102,14 @@ func MustPutStore(re *require.Assertions, svr *server.Server, store *metapb.Stor Store: store, }) re.NoError(err) + + storeInfo := grpcServer.GetRaftCluster().GetStore(store.GetId()) + newStore := storeInfo.Clone(core.SetStoreStats(&pdpb.StoreStats{ + Capacity: uint64(10 * units.GiB), + UsedSize: uint64(9 * units.GiB), + Available: uint64(1 * units.GiB), + })) + grpcServer.GetRaftCluster().GetBasicCluster().PutStore(newStore) } // MustPutRegion is used for test purpose. @@ -121,3 +130,18 @@ func MustPutRegion(re *require.Assertions, cluster *tests.TestCluster, regionID, re.NoError(err) return r } + +// MustReportBuckets is used for test purpose. +func MustReportBuckets(re *require.Assertions, cluster *tests.TestCluster, regionID uint64, start, end []byte, stats *metapb.BucketStats) *metapb.Buckets { + buckets := &metapb.Buckets{ + RegionId: regionID, + Version: 1, + Keys: [][]byte{start, end}, + Stats: stats, + // report buckets interval is 10s + PeriodInMs: 10000, + } + err := cluster.HandleReportBuckets(buckets) + re.NoError(err) + return buckets +} diff --git a/tests/pdctl/hot/hot_test.go b/tests/pdctl/hot/hot_test.go index 594c14f707f..2dfa89acb52 100644 --- a/tests/pdctl/hot/hot_test.go +++ b/tests/pdctl/hot/hot_test.go @@ -257,6 +257,39 @@ func TestHotWithStoreID(t *testing.T) { re.Equal(1, hotRegion.AsLeader[2].Count) re.Equal(float64(200000000), hotRegion.AsLeader[1].TotalBytesRate) re.Equal(float64(100000000), hotRegion.AsLeader[2].TotalBytesRate) + + stats := &metapb.BucketStats{ + ReadBytes: []uint64{10 * units.MiB}, + ReadKeys: []uint64{11 * units.MiB}, + ReadQps: []uint64{0}, + WriteKeys: []uint64{12 * units.MiB}, + WriteBytes: []uint64{13 * units.MiB}, + WriteQps: []uint64{0}, + } + buckets := pdctl.MustReportBuckets(re, cluster, 1, []byte("a"), []byte("b"), stats) + args = []string{"-u", pdAddr, "hot", "buckets", "1"} + output, err = pdctl.ExecuteCommand(cmd, args...) + re.NoError(err) + hotBuckets := api.HotBucketsResponse{} + re.NoError(json.Unmarshal(output, &hotBuckets)) + re.Len(hotBuckets, 1) + re.Len(hotBuckets[1], 1) + item := hotBuckets[1][0] + re.Equal(core.HexRegionKeyStr(buckets.GetKeys()[0]), item.StartKey) + re.Equal(core.HexRegionKeyStr(buckets.GetKeys()[1]), item.EndKey) + re.Equal(1, item.HotDegree) + interval := buckets.GetPeriodInMs() / 1000 + re.Equal(buckets.GetStats().ReadBytes[0]/interval, item.ReadBytes) + re.Equal(buckets.GetStats().ReadKeys[0]/interval, item.ReadKeys) + re.Equal(buckets.GetStats().WriteBytes[0]/interval, item.WriteBytes) + re.Equal(buckets.GetStats().WriteKeys[0]/interval, item.WriteKeys) + + args = []string{"-u", pdAddr, "hot", "buckets", "2"} + output, err = pdctl.ExecuteCommand(cmd, args...) + re.NoError(err) + hotBuckets = api.HotBucketsResponse{} + re.NoError(json.Unmarshal(output, &hotBuckets)) + re.Nil(hotBuckets[2]) } func TestHistoryHotRegions(t *testing.T) { diff --git a/tests/pdctl/keyspace/keyspace_group_test.go b/tests/pdctl/keyspace/keyspace_group_test.go new file mode 100644 index 00000000000..1d0c8132c13 --- /dev/null +++ b/tests/pdctl/keyspace/keyspace_group_test.go @@ -0,0 +1,585 @@ +// Copyright 2023 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package keyspace_test + +import ( + "context" + "encoding/json" + "fmt" + "strconv" + "strings" + "testing" + + "github.com/pingcap/failpoint" + "github.com/stretchr/testify/require" + "github.com/tikv/pd/pkg/mcs/utils" + "github.com/tikv/pd/pkg/storage/endpoint" + "github.com/tikv/pd/pkg/utils/testutil" + "github.com/tikv/pd/server/apiv2/handlers" + "github.com/tikv/pd/server/config" + "github.com/tikv/pd/tests" + "github.com/tikv/pd/tests/pdctl" + handlersutil "github.com/tikv/pd/tests/server/apiv2/handlers" + pdctlCmd "github.com/tikv/pd/tools/pd-ctl/pdctl" +) + +func TestKeyspaceGroup(t *testing.T) { + re := require.New(t) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + tc, err := tests.NewTestAPICluster(ctx, 1) + re.NoError(err) + err = tc.RunInitialServers() + re.NoError(err) + tc.WaitLeader() + leaderServer := tc.GetServer(tc.GetLeader()) + re.NoError(leaderServer.BootstrapCluster()) + pdAddr := tc.GetConfig().GetClientURL() + cmd := pdctlCmd.GetRootCmd() + + // Show keyspace group information. + defaultKeyspaceGroupID := fmt.Sprintf("%d", utils.DefaultKeyspaceGroupID) + args := []string{"-u", pdAddr, "keyspace-group"} + output, err := pdctl.ExecuteCommand(cmd, append(args, defaultKeyspaceGroupID)...) + re.NoError(err) + var keyspaceGroup endpoint.KeyspaceGroup + err = json.Unmarshal(output, &keyspaceGroup) + re.NoError(err) + re.Equal(utils.DefaultKeyspaceGroupID, keyspaceGroup.ID) + re.Contains(keyspaceGroup.Keyspaces, utils.DefaultKeyspaceID) + // Split keyspace group. + handlersutil.MustCreateKeyspaceGroup(re, leaderServer, &handlers.CreateKeyspaceGroupParams{ + KeyspaceGroups: []*endpoint.KeyspaceGroup{ + { + ID: 1, + UserKind: endpoint.Standard.String(), + Members: make([]endpoint.KeyspaceGroupMember, utils.DefaultKeyspaceGroupReplicaCount), + Keyspaces: []uint32{111, 222, 333}, + }, + }, + }) + _, err = pdctl.ExecuteCommand(cmd, append(args, "split", "1", "2", "222", "333")...) + re.NoError(err) + output, err = pdctl.ExecuteCommand(cmd, append(args, "1")...) + re.NoError(err) + keyspaceGroup = endpoint.KeyspaceGroup{} + err = json.Unmarshal(output, &keyspaceGroup) + re.NoError(err) + re.Equal(uint32(1), keyspaceGroup.ID) + re.Equal(keyspaceGroup.Keyspaces, []uint32{111}) + output, err = pdctl.ExecuteCommand(cmd, append(args, "2")...) + re.NoError(err) + keyspaceGroup = endpoint.KeyspaceGroup{} + err = json.Unmarshal(output, &keyspaceGroup) + re.NoError(err) + re.Equal(uint32(2), keyspaceGroup.ID) + re.Equal(keyspaceGroup.Keyspaces, []uint32{222, 333}) +} + +func TestSplitKeyspaceGroup(t *testing.T) { + re := require.New(t) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/keyspace/acceleratedAllocNodes", `return(true)`)) + re.NoError(failpoint.Enable("github.com/tikv/pd/server/delayStartServerLoop", `return(true)`)) + keyspaces := make([]string, 0) + // we test the case which exceed the default max txn ops limit in etcd, which is 128. + for i := 0; i < 129; i++ { + keyspaces = append(keyspaces, fmt.Sprintf("keyspace_%d", i)) + } + tc, err := tests.NewTestAPICluster(ctx, 3, func(conf *config.Config, serverName string) { + conf.Keyspace.PreAlloc = keyspaces + }) + re.NoError(err) + err = tc.RunInitialServers() + re.NoError(err) + pdAddr := tc.GetConfig().GetClientURL() + + ttc, err := tests.NewTestTSOCluster(ctx, 2, pdAddr) + re.NoError(err) + defer ttc.Destroy() + cmd := pdctlCmd.GetRootCmd() + + tc.WaitLeader() + leaderServer := tc.GetServer(tc.GetLeader()) + re.NoError(leaderServer.BootstrapCluster()) + + // split keyspace group. + testutil.Eventually(re, func() bool { + args := []string{"-u", pdAddr, "keyspace-group", "split", "0", "1", "2"} + output, err := pdctl.ExecuteCommand(cmd, args...) + re.NoError(err) + return strings.Contains(string(output), "Success") + }) + + // get all keyspaces + args := []string{"-u", pdAddr, "keyspace-group"} + output, err := pdctl.ExecuteCommand(cmd, args...) + re.NoError(err) + strings.Contains(string(output), "Success") + var keyspaceGroups []*endpoint.KeyspaceGroup + err = json.Unmarshal(output, &keyspaceGroups) + re.NoError(err) + re.Len(keyspaceGroups, 2) + re.Equal(keyspaceGroups[0].ID, uint32(0)) + re.Equal(keyspaceGroups[1].ID, uint32(1)) + + re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/keyspace/acceleratedAllocNodes")) + re.NoError(failpoint.Disable("github.com/tikv/pd/server/delayStartServerLoop")) +} + +func TestExternalAllocNodeWhenStart(t *testing.T) { + re := require.New(t) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + // external alloc node for keyspace group, when keyspace manager update keyspace info to keyspace group + // we hope the keyspace group can be updated correctly. + re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/keyspace/externalAllocNode", `return("127.0.0.1:2379,127.0.0.1:2380")`)) + re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/keyspace/acceleratedAllocNodes", `return(true)`)) + re.NoError(failpoint.Enable("github.com/tikv/pd/server/delayStartServerLoop", `return(true)`)) + keyspaces := make([]string, 0) + for i := 0; i < 10; i++ { + keyspaces = append(keyspaces, fmt.Sprintf("keyspace_%d", i)) + } + tc, err := tests.NewTestAPICluster(ctx, 1, func(conf *config.Config, serverName string) { + conf.Keyspace.PreAlloc = keyspaces + }) + re.NoError(err) + err = tc.RunInitialServers() + re.NoError(err) + pdAddr := tc.GetConfig().GetClientURL() + + cmd := pdctlCmd.GetRootCmd() + + tc.WaitLeader() + leaderServer := tc.GetServer(tc.GetLeader()) + re.NoError(leaderServer.BootstrapCluster()) + + // check keyspace group information. + defaultKeyspaceGroupID := fmt.Sprintf("%d", utils.DefaultKeyspaceGroupID) + args := []string{"-u", pdAddr, "keyspace-group"} + testutil.Eventually(re, func() bool { + output, err := pdctl.ExecuteCommand(cmd, append(args, defaultKeyspaceGroupID)...) + re.NoError(err) + var keyspaceGroup endpoint.KeyspaceGroup + err = json.Unmarshal(output, &keyspaceGroup) + re.NoError(err) + return len(keyspaceGroup.Keyspaces) == len(keyspaces)+1 && len(keyspaceGroup.Members) == 2 + }) + + re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/keyspace/externalAllocNode")) + re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/keyspace/acceleratedAllocNodes")) + re.NoError(failpoint.Disable("github.com/tikv/pd/server/delayStartServerLoop")) +} + +func TestSetNodeAndPriorityKeyspaceGroup(t *testing.T) { + re := require.New(t) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + keyspaces := make([]string, 0) + for i := 0; i < 10; i++ { + keyspaces = append(keyspaces, fmt.Sprintf("keyspace_%d", i)) + } + tc, err := tests.NewTestAPICluster(ctx, 3, func(conf *config.Config, serverName string) { + conf.Keyspace.PreAlloc = keyspaces + }) + re.NoError(err) + err = tc.RunInitialServers() + re.NoError(err) + pdAddr := tc.GetConfig().GetClientURL() + + ttc, err := tests.NewTestTSOCluster(ctx, 2, pdAddr) + re.NoError(err) + defer ttc.Destroy() + tsoAddrs := ttc.GetAddrs() + cmd := pdctlCmd.GetRootCmd() + + tc.WaitLeader() + leaderServer := tc.GetServer(tc.GetLeader()) + re.NoError(leaderServer.BootstrapCluster()) + + // set-node keyspace group. + defaultKeyspaceGroupID := fmt.Sprintf("%d", utils.DefaultKeyspaceGroupID) + testutil.Eventually(re, func() bool { + args := []string{"-u", pdAddr, "keyspace-group", "set-node", defaultKeyspaceGroupID, tsoAddrs[0], tsoAddrs[1]} + output, err := pdctl.ExecuteCommand(cmd, args...) + re.NoError(err) + return strings.Contains(string(output), "Success") + }) + + // set-priority keyspace group. + checkPriority := func(p int) { + testutil.Eventually(re, func() bool { + args := []string{"-u", pdAddr, "keyspace-group", "set-priority", defaultKeyspaceGroupID, tsoAddrs[0]} + if p >= 0 { + args = append(args, strconv.Itoa(p)) + } else { + args = append(args, "--", strconv.Itoa(p)) + } + output, err := pdctl.ExecuteCommand(cmd, args...) + re.NoError(err) + return strings.Contains(string(output), "Success") + }) + + // check keyspace group information. + args := []string{"-u", pdAddr, "keyspace-group"} + output, err := pdctl.ExecuteCommand(cmd, append(args, defaultKeyspaceGroupID)...) + re.NoError(err) + var keyspaceGroup endpoint.KeyspaceGroup + err = json.Unmarshal(output, &keyspaceGroup) + re.NoError(err) + re.Equal(utils.DefaultKeyspaceGroupID, keyspaceGroup.ID) + re.Len(keyspaceGroup.Members, 2) + for _, member := range keyspaceGroup.Members { + re.Contains(tsoAddrs, member.Address) + if member.Address == tsoAddrs[0] { + re.Equal(p, member.Priority) + } else { + re.Equal(0, member.Priority) + } + } + } + + checkPriority(200) + checkPriority(-200) + + // params error for set-node. + args := []string{"-u", pdAddr, "keyspace-group", "set-node", defaultKeyspaceGroupID, tsoAddrs[0]} + output, err := pdctl.ExecuteCommand(cmd, args...) + re.NoError(err) + re.Contains(string(output), "invalid num of nodes") + args = []string{"-u", pdAddr, "keyspace-group", "set-node", defaultKeyspaceGroupID, "", ""} + output, err = pdctl.ExecuteCommand(cmd, args...) + re.NoError(err) + re.Contains(string(output), "Failed to parse the tso node address") + args = []string{"-u", pdAddr, "keyspace-group", "set-node", defaultKeyspaceGroupID, tsoAddrs[0], "http://pingcap.com"} + output, err = pdctl.ExecuteCommand(cmd, args...) + re.NoError(err) + re.Contains(string(output), "node does not exist") + + // params error for set-priority. + args = []string{"-u", pdAddr, "keyspace-group", "set-priority", defaultKeyspaceGroupID, "", "200"} + output, err = pdctl.ExecuteCommand(cmd, args...) + re.NoError(err) + re.Contains(string(output), "Failed to parse the tso node address") + args = []string{"-u", pdAddr, "keyspace-group", "set-priority", defaultKeyspaceGroupID, "http://pingcap.com", "200"} + output, err = pdctl.ExecuteCommand(cmd, args...) + re.NoError(err) + re.Contains(string(output), "node does not exist") + args = []string{"-u", pdAddr, "keyspace-group", "set-priority", defaultKeyspaceGroupID, tsoAddrs[0], "xxx"} + output, err = pdctl.ExecuteCommand(cmd, args...) + re.NoError(err) + re.Contains(string(output), "Failed to parse the priority") +} + +func TestMergeKeyspaceGroup(t *testing.T) { + re := require.New(t) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/keyspace/acceleratedAllocNodes", `return(true)`)) + re.NoError(failpoint.Enable("github.com/tikv/pd/server/delayStartServerLoop", `return(true)`)) + keyspaces := make([]string, 0) + // we test the case which exceed the default max txn ops limit in etcd, which is 128. + for i := 0; i < 129; i++ { + keyspaces = append(keyspaces, fmt.Sprintf("keyspace_%d", i)) + } + tc, err := tests.NewTestAPICluster(ctx, 1, func(conf *config.Config, serverName string) { + conf.Keyspace.PreAlloc = keyspaces + }) + re.NoError(err) + err = tc.RunInitialServers() + re.NoError(err) + pdAddr := tc.GetConfig().GetClientURL() + + ttc, err := tests.NewTestTSOCluster(ctx, 2, pdAddr) + re.NoError(err) + defer ttc.Destroy() + cmd := pdctlCmd.GetRootCmd() + + tc.WaitLeader() + leaderServer := tc.GetServer(tc.GetLeader()) + re.NoError(leaderServer.BootstrapCluster()) + + // split keyspace group. + testutil.Eventually(re, func() bool { + args := []string{"-u", pdAddr, "keyspace-group", "split", "0", "1", "2"} + output, err := pdctl.ExecuteCommand(cmd, args...) + re.NoError(err) + return strings.Contains(string(output), "Success") + }) + + args := []string{"-u", pdAddr, "keyspace-group", "finish-split", "1"} + output, err := pdctl.ExecuteCommand(cmd, args...) + re.NoError(err) + strings.Contains(string(output), "Success") + + // merge keyspace group. + testutil.Eventually(re, func() bool { + args := []string{"-u", pdAddr, "keyspace-group", "merge", "0", "1"} + output, err := pdctl.ExecuteCommand(cmd, args...) + re.NoError(err) + return strings.Contains(string(output), "Success") + }) + + args = []string{"-u", pdAddr, "keyspace-group", "finish-merge", "0"} + output, err = pdctl.ExecuteCommand(cmd, args...) + re.NoError(err) + strings.Contains(string(output), "Success") + args = []string{"-u", pdAddr, "keyspace-group", "0"} + output, err = pdctl.ExecuteCommand(cmd, args...) + re.NoError(err) + var keyspaceGroup endpoint.KeyspaceGroup + err = json.Unmarshal(output, &keyspaceGroup) + re.NoError(err) + re.Len(keyspaceGroup.Keyspaces, 130) + re.Nil(keyspaceGroup.MergeState) + + // split keyspace group multiple times. + for i := 1; i <= 10; i++ { + splitTargetID := fmt.Sprintf("%d", i) + testutil.Eventually(re, func() bool { + args := []string{"-u", pdAddr, "keyspace-group", "split", "0", splitTargetID, splitTargetID} + output, err := pdctl.ExecuteCommand(cmd, args...) + re.NoError(err) + return strings.Contains(string(output), "Success") + }) + args := []string{"-u", pdAddr, "keyspace-group", "finish-split", splitTargetID} + output, err := pdctl.ExecuteCommand(cmd, args...) + re.NoError(err) + strings.Contains(string(output), "Success") + } + + // merge keyspace group with `all` flag. + testutil.Eventually(re, func() bool { + args := []string{"-u", pdAddr, "keyspace-group", "merge", "0", "--all"} + output, err := pdctl.ExecuteCommand(cmd, args...) + re.NoError(err) + return strings.Contains(string(output), "Success") + }) + + args = []string{"-u", pdAddr, "keyspace-group", "finish-merge", "0"} + output, err = pdctl.ExecuteCommand(cmd, args...) + re.NoError(err) + strings.Contains(string(output), "Success") + args = []string{"-u", pdAddr, "keyspace-group", "0"} + output, err = pdctl.ExecuteCommand(cmd, args...) + re.NoError(err) + err = json.Unmarshal(output, &keyspaceGroup) + re.NoError(err) + re.Len(keyspaceGroup.Keyspaces, 130) + re.Nil(keyspaceGroup.MergeState) + + // merge keyspace group with wrong args. + args = []string{"-u", pdAddr, "keyspace-group", "merge"} + output, err = pdctl.ExecuteCommand(cmd, args...) + re.NoError(err) + strings.Contains(string(output), "Must specify the source keyspace group ID(s) or the merge all flag") + args = []string{"-u", pdAddr, "keyspace-group", "merge", "0"} + output, err = pdctl.ExecuteCommand(cmd, args...) + re.NoError(err) + strings.Contains(string(output), "Must specify the source keyspace group ID(s) or the merge all flag") + args = []string{"-u", pdAddr, "keyspace-group", "merge", "0", "1", "--all"} + output, err = pdctl.ExecuteCommand(cmd, args...) + re.NoError(err) + strings.Contains(string(output), "Must specify the source keyspace group ID(s) or the merge all flag") + args = []string{"-u", pdAddr, "keyspace-group", "merge", "1", "--all"} + output, err = pdctl.ExecuteCommand(cmd, args...) + re.NoError(err) + strings.Contains(string(output), "Unable to merge all keyspace groups into a non-default keyspace group") + + re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/keyspace/acceleratedAllocNodes")) + re.NoError(failpoint.Disable("github.com/tikv/pd/server/delayStartServerLoop")) +} + +func TestKeyspaceGroupState(t *testing.T) { + re := require.New(t) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/keyspace/acceleratedAllocNodes", `return(true)`)) + re.NoError(failpoint.Enable("github.com/tikv/pd/server/delayStartServerLoop", `return(true)`)) + keyspaces := make([]string, 0) + for i := 0; i < 10; i++ { + keyspaces = append(keyspaces, fmt.Sprintf("keyspace_%d", i)) + } + tc, err := tests.NewTestAPICluster(ctx, 1, func(conf *config.Config, serverName string) { + conf.Keyspace.PreAlloc = keyspaces + }) + re.NoError(err) + err = tc.RunInitialServers() + re.NoError(err) + pdAddr := tc.GetConfig().GetClientURL() + + ttc, err := tests.NewTestTSOCluster(ctx, 2, pdAddr) + re.NoError(err) + defer ttc.Destroy() + cmd := pdctlCmd.GetRootCmd() + + tc.WaitLeader() + leaderServer := tc.GetServer(tc.GetLeader()) + re.NoError(leaderServer.BootstrapCluster()) + + // split keyspace group. + testutil.Eventually(re, func() bool { + args := []string{"-u", pdAddr, "keyspace-group", "split", "0", "1", "2"} + output, err := pdctl.ExecuteCommand(cmd, args...) + re.NoError(err) + return strings.Contains(string(output), "Success") + }) + args := []string{"-u", pdAddr, "keyspace-group", "finish-split", "1"} + output, err := pdctl.ExecuteCommand(cmd, args...) + re.NoError(err) + strings.Contains(string(output), "Success") + args = []string{"-u", pdAddr, "keyspace-group", "--state", "split"} + output, err = pdctl.ExecuteCommand(cmd, args...) + re.NoError(err) + strings.Contains(string(output), "Success") + var keyspaceGroups []*endpoint.KeyspaceGroup + err = json.Unmarshal(output, &keyspaceGroups) + re.NoError(err) + re.Len(keyspaceGroups, 0) + testutil.Eventually(re, func() bool { + args := []string{"-u", pdAddr, "keyspace-group", "split", "0", "2", "3"} + output, err := pdctl.ExecuteCommand(cmd, args...) + re.NoError(err) + return strings.Contains(string(output), "Success") + }) + args = []string{"-u", pdAddr, "keyspace-group", "--state", "split"} + output, err = pdctl.ExecuteCommand(cmd, args...) + re.NoError(err) + strings.Contains(string(output), "Success") + err = json.Unmarshal(output, &keyspaceGroups) + re.NoError(err) + re.Len(keyspaceGroups, 2) + re.Equal(keyspaceGroups[0].ID, uint32(0)) + re.Equal(keyspaceGroups[1].ID, uint32(2)) + + args = []string{"-u", pdAddr, "keyspace-group", "finish-split", "2"} + output, err = pdctl.ExecuteCommand(cmd, args...) + re.NoError(err) + strings.Contains(string(output), "Success") + // merge keyspace group. + testutil.Eventually(re, func() bool { + args := []string{"-u", pdAddr, "keyspace-group", "merge", "0", "1"} + output, err := pdctl.ExecuteCommand(cmd, args...) + re.NoError(err) + return strings.Contains(string(output), "Success") + }) + + args = []string{"-u", pdAddr, "keyspace-group", "--state", "merge"} + output, err = pdctl.ExecuteCommand(cmd, args...) + re.NoError(err) + strings.Contains(string(output), "Success") + err = json.Unmarshal(output, &keyspaceGroups) + re.NoError(err) + err = json.Unmarshal(output, &keyspaceGroups) + re.NoError(err) + re.Len(keyspaceGroups, 1) + re.Equal(keyspaceGroups[0].ID, uint32(0)) + + re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/keyspace/acceleratedAllocNodes")) + re.NoError(failpoint.Disable("github.com/tikv/pd/server/delayStartServerLoop")) +} + +func TestShowKeyspaceGroupPrimary(t *testing.T) { + re := require.New(t) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/keyspace/acceleratedAllocNodes", `return(true)`)) + re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/tso/fastGroupSplitPatroller", `return(true)`)) + re.NoError(failpoint.Enable("github.com/tikv/pd/server/delayStartServerLoop", `return(true)`)) + keyspaces := make([]string, 0) + for i := 0; i < 10; i++ { + keyspaces = append(keyspaces, fmt.Sprintf("keyspace_%d", i)) + } + tc, err := tests.NewTestAPICluster(ctx, 3, func(conf *config.Config, serverName string) { + conf.Keyspace.PreAlloc = keyspaces + }) + re.NoError(err) + err = tc.RunInitialServers() + re.NoError(err) + pdAddr := tc.GetConfig().GetClientURL() + + ttc, err := tests.NewTestTSOCluster(ctx, 2, pdAddr) + re.NoError(err) + defer ttc.Destroy() + tsoAddrs := ttc.GetAddrs() + cmd := pdctlCmd.GetRootCmd() + + tc.WaitLeader() + leaderServer := tc.GetServer(tc.GetLeader()) + re.NoError(leaderServer.BootstrapCluster()) + defaultKeyspaceGroupID := fmt.Sprintf("%d", utils.DefaultKeyspaceGroupID) + + // check keyspace group 0 information. + var keyspaceGroup endpoint.KeyspaceGroup + testutil.Eventually(re, func() bool { + args := []string{"-u", pdAddr, "keyspace-group"} + output, err := pdctl.ExecuteCommand(cmd, append(args, defaultKeyspaceGroupID)...) + re.NoError(err) + + err = json.Unmarshal(output, &keyspaceGroup) + re.NoError(err) + re.Equal(utils.DefaultKeyspaceGroupID, keyspaceGroup.ID) + return len(keyspaceGroup.Members) == 2 + }) + for _, member := range keyspaceGroup.Members { + re.Contains(tsoAddrs, member.Address) + } + + // get primary for keyspace group 0. + testutil.Eventually(re, func() bool { + args := []string{"-u", pdAddr, "keyspace-group", "primary", defaultKeyspaceGroupID} + output, err := pdctl.ExecuteCommand(cmd, args...) + re.NoError(err) + var resp handlers.GetKeyspaceGroupPrimaryResponse + json.Unmarshal(output, &resp) + return tsoAddrs[0] == resp.Primary || tsoAddrs[1] == resp.Primary + }) + + // split keyspace group. + testutil.Eventually(re, func() bool { + args := []string{"-u", pdAddr, "keyspace-group", "split", "0", "1", "2"} + output, err := pdctl.ExecuteCommand(cmd, args...) + re.NoError(err) + return strings.Contains(string(output), "Success") + }) + + // check keyspace group 1 information. + testutil.Eventually(re, func() bool { + args := []string{"-u", pdAddr, "keyspace-group"} + output, err := pdctl.ExecuteCommand(cmd, append(args, "1")...) + re.NoError(err) + err = json.Unmarshal(output, &keyspaceGroup) + re.NoErrorf(err, "output: %s", string(output)) + return len(keyspaceGroup.Members) == 2 + }) + for _, member := range keyspaceGroup.Members { + re.Contains(tsoAddrs, member.Address) + } + + // get primary for keyspace group 1. + testutil.Eventually(re, func() bool { + args := []string{"-u", pdAddr, "keyspace-group", "primary", "1"} + output, err := pdctl.ExecuteCommand(cmd, args...) + re.NoError(err) + var resp handlers.GetKeyspaceGroupPrimaryResponse + json.Unmarshal(output, &resp) + return tsoAddrs[0] == resp.Primary || tsoAddrs[1] == resp.Primary + }) + + re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/keyspace/acceleratedAllocNodes")) + re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/tso/fastGroupSplitPatroller")) + re.NoError(failpoint.Disable("github.com/tikv/pd/server/delayStartServerLoop")) +} diff --git a/tests/pdctl/keyspace/keyspace_test.go b/tests/pdctl/keyspace/keyspace_test.go new file mode 100644 index 00000000000..a0bab4114df --- /dev/null +++ b/tests/pdctl/keyspace/keyspace_test.go @@ -0,0 +1,103 @@ +// Copyright 2023 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package keyspace_test + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "testing" + + "github.com/pingcap/failpoint" + "github.com/stretchr/testify/require" + "github.com/tikv/pd/pkg/keyspace" + "github.com/tikv/pd/pkg/mcs/utils" + "github.com/tikv/pd/pkg/utils/testutil" + api "github.com/tikv/pd/server/apiv2/handlers" + "github.com/tikv/pd/server/config" + "github.com/tikv/pd/tests" + "github.com/tikv/pd/tests/pdctl" + pdctlCmd "github.com/tikv/pd/tools/pd-ctl/pdctl" +) + +func TestKeyspace(t *testing.T) { + re := require.New(t) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/keyspace/acceleratedAllocNodes", `return(true)`)) + re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/tso/fastGroupSplitPatroller", `return(true)`)) + re.NoError(failpoint.Enable("github.com/tikv/pd/server/delayStartServerLoop", `return(true)`)) + keyspaces := make([]string, 0) + for i := 1; i < 10; i++ { + keyspaces = append(keyspaces, fmt.Sprintf("keyspace_%d", i)) + } + tc, err := tests.NewTestAPICluster(ctx, 3, func(conf *config.Config, serverName string) { + conf.Keyspace.PreAlloc = keyspaces + }) + re.NoError(err) + err = tc.RunInitialServers() + re.NoError(err) + pdAddr := tc.GetConfig().GetClientURL() + + ttc, err := tests.NewTestTSOCluster(ctx, 2, pdAddr) + re.NoError(err) + defer ttc.Destroy() + cmd := pdctlCmd.GetRootCmd() + + tc.WaitLeader() + leaderServer := tc.GetServer(tc.GetLeader()) + re.NoError(leaderServer.BootstrapCluster()) + defaultKeyspaceGroupID := fmt.Sprintf("%d", utils.DefaultKeyspaceGroupID) + + var k api.KeyspaceMeta + keyspaceName := "keyspace_1" + testutil.Eventually(re, func() bool { + args := []string{"-u", pdAddr, "keyspace", keyspaceName} + output, err := pdctl.ExecuteCommand(cmd, args...) + re.NoError(err) + re.NoError(json.Unmarshal(output, &k)) + return k.GetName() == keyspaceName + }) + re.Equal(uint32(1), k.GetId()) + re.Equal(defaultKeyspaceGroupID, k.Config[keyspace.TSOKeyspaceGroupIDKey]) + + // split keyspace group. + newGroupID := "2" + testutil.Eventually(re, func() bool { + args := []string{"-u", pdAddr, "keyspace-group", "split", "0", newGroupID, "1"} + output, err := pdctl.ExecuteCommand(cmd, args...) + re.NoError(err) + return strings.Contains(string(output), "Success") + }) + + // check keyspace group in keyspace whether changed. + testutil.Eventually(re, func() bool { + args := []string{"-u", pdAddr, "keyspace", keyspaceName} + output, err := pdctl.ExecuteCommand(cmd, args...) + re.NoError(err) + re.NoError(json.Unmarshal(output, &k)) + return newGroupID == k.Config[keyspace.TSOKeyspaceGroupIDKey] + }) + + // test error name + args := []string{"-u", pdAddr, "keyspace", "error_name"} + output, err := pdctl.ExecuteCommand(cmd, args...) + re.NoError(err) + re.Contains(string(output), "Fail") + re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/keyspace/acceleratedAllocNodes")) + re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/tso/fastGroupSplitPatroller")) + re.NoError(failpoint.Disable("github.com/tikv/pd/server/delayStartServerLoop")) +} diff --git a/tests/pdctl/operator/operator_test.go b/tests/pdctl/operator/operator_test.go index c53c5a42a0f..ab5687cdc04 100644 --- a/tests/pdctl/operator/operator_test.go +++ b/tests/pdctl/operator/operator_test.go @@ -16,6 +16,7 @@ package operator_test import ( "context" + "encoding/hex" "strconv" "strings" "testing" @@ -87,7 +88,7 @@ func TestOperator(t *testing.T) { {Id: 1, StoreId: 1}, {Id: 2, StoreId: 2}, })) - pdctl.MustPutRegion(re, cluster, 3, 2, []byte("b"), []byte("c"), core.SetPeers([]*metapb.Peer{ + pdctl.MustPutRegion(re, cluster, 3, 2, []byte("b"), []byte("d"), core.SetPeers([]*metapb.Peer{ {Id: 3, StoreId: 1}, {Id: 4, StoreId: 2}, })) @@ -135,33 +136,41 @@ func TestOperator(t *testing.T) { reset: []string{"-u", pdAddr, "operator", "remove", "1"}, }, { - // operator add split-region [--policy=scan|approximate] + // operator add split-region [--policy=scan|approximate|usekey] [--keys=xxx(xxx is hex encoded string)] cmd: []string{"-u", pdAddr, "operator", "add", "split-region", "3", "--policy=scan"}, show: []string{"-u", pdAddr, "operator", "show"}, expect: "split region with policy SCAN", reset: []string{"-u", pdAddr, "operator", "remove", "3"}, }, { - // operator add split-region [--policy=scan|approximate] + // operator add split-region [--policy=scan|approximate|usekey] [--keys=xxx(xxx is hex encoded string)] cmd: []string{"-u", pdAddr, "operator", "add", "split-region", "3", "--policy=approximate"}, show: []string{"-u", pdAddr, "operator", "show"}, expect: "split region with policy APPROXIMATE", reset: []string{"-u", pdAddr, "operator", "remove", "3"}, }, { - // operator add split-region [--policy=scan|approximate] + // operator add split-region [--policy=scan|approximate|usekey] [--keys=xxx(xxx is hex encoded string)] cmd: []string{"-u", pdAddr, "operator", "add", "split-region", "3", "--policy=scan"}, show: []string{"-u", pdAddr, "operator", "check", "3"}, expect: "split region with policy SCAN", reset: []string{"-u", pdAddr, "operator", "remove", "3"}, }, { - // operator add split-region [--policy=scan|approximate] + // operator add split-region [--policy=scan|approximate|usekey] [--keys=xxx(xxx is hex encoded string)] cmd: []string{"-u", pdAddr, "operator", "add", "split-region", "3", "--policy=approximate"}, show: []string{"-u", pdAddr, "operator", "check", "3"}, expect: "status: RUNNING", reset: []string{"-u", pdAddr, "operator", "remove", "3"}, }, + { + // operator add split-region [--policy=scan|approximate|usekey] [--keys=xxx(xxx is hex encoded string)] + cmd: []string{"-u", pdAddr, "operator", "add", "split-region", "3", "--policy=usekey", + "--keys=" + hex.EncodeToString([]byte("c"))}, + show: []string{"-u", pdAddr, "operator", "show"}, + expect: "split: region 3 use policy USEKEY and keys [" + hex.EncodeToString([]byte("c")) + "]", + reset: []string{"-u", pdAddr, "operator", "remove", "3"}, + }, } for _, testCase := range testCases { diff --git a/tests/pdctl/scheduler/scheduler_test.go b/tests/pdctl/scheduler/scheduler_test.go index 12813c8b097..e1c0a210c01 100644 --- a/tests/pdctl/scheduler/scheduler_test.go +++ b/tests/pdctl/scheduler/scheduler_test.go @@ -22,9 +22,9 @@ import ( "github.com/pingcap/kvproto/pkg/metapb" "github.com/stretchr/testify/require" + sc "github.com/tikv/pd/pkg/schedule/config" "github.com/tikv/pd/pkg/utils/testutil" "github.com/tikv/pd/pkg/versioninfo" - "github.com/tikv/pd/server/config" "github.com/tikv/pd/tests" "github.com/tikv/pd/tests/pdctl" pdctlCmd "github.com/tikv/pd/tools/pd-ctl/pdctl" @@ -117,7 +117,6 @@ func TestScheduler(t *testing.T) { mightExec([]string{"-u", pdAddr, "scheduler", "describe", schedulerName}, &result) return len(result) != 0 }, testutil.WithTickInterval(50*time.Millisecond)) - re.Equal(expectedStatus, result["status"]) re.Equal(expectedSummary, result["summary"]) } @@ -340,6 +339,7 @@ func TestScheduler(t *testing.T) { "strict-picking-store": "true", "enable-for-tiflash": "true", "rank-formula-version": "v2", + "split-thresholds": 0.2, } var conf map[string]interface{} mustExec([]string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler", "list"}, &conf) @@ -460,7 +460,11 @@ func TestScheduler(t *testing.T) { checkSchedulerWithStatusCommand(nil, "paused", []string{ "balance-leader-scheduler", }) - checkSchedulerDescribeCommand("balance-leader-scheduler", "paused", "") + result := make(map[string]interface{}) + testutil.Eventually(re, func() bool { + mightExec([]string{"-u", pdAddr, "scheduler", "describe", "balance-leader-scheduler"}, &result) + return len(result) != 0 && result["status"] == "paused" && result["summary"] == "" + }, testutil.WithWaitFor(30*time.Second)) mustUsage([]string{"-u", pdAddr, "scheduler", "resume", "balance-leader-scheduler", "60"}) mustExec([]string{"-u", pdAddr, "scheduler", "resume", "balance-leader-scheduler"}, nil) @@ -472,7 +476,7 @@ func TestScheduler(t *testing.T) { re.Contains(echo, "Success!") cfg := leaderServer.GetServer().GetScheduleConfig() origin := cfg.Schedulers - cfg.Schedulers = config.SchedulerConfigs{{Type: "label", Disable: true}} + cfg.Schedulers = sc.SchedulerConfigs{{Type: "label", Disable: true}} err = leaderServer.GetServer().SetScheduleConfig(*cfg) re.NoError(err) checkSchedulerWithStatusCommand(nil, "disabled", []string{"label-scheduler"}) @@ -481,14 +485,4 @@ func TestScheduler(t *testing.T) { err = leaderServer.GetServer().SetScheduleConfig(*cfg) re.NoError(err) checkSchedulerWithStatusCommand(nil, "disabled", nil) - - // test split bucket scheduler - echo = mustExec([]string{"-u", pdAddr, "scheduler", "config", "split-bucket-scheduler"}, nil) - re.Contains(echo, "\"degree\": 3") - echo = mustExec([]string{"-u", pdAddr, "scheduler", "config", "split-bucket-scheduler", "set", "degree", "10"}, nil) - re.Contains(echo, "Success") - echo = mustExec([]string{"-u", pdAddr, "scheduler", "config", "split-bucket-scheduler"}, nil) - re.Contains(echo, "\"degree\": 10") - echo = mustExec([]string{"-u", pdAddr, "scheduler", "remove", "split-bucket-scheduler"}, nil) - re.Contains(echo, "Success!") } diff --git a/tests/server/api/api_test.go b/tests/server/api/api_test.go index 7fb7e7d1236..080eeb44b1b 100644 --- a/tests/server/api/api_test.go +++ b/tests/server/api/api_test.go @@ -377,6 +377,15 @@ func (suite *middlewareTestSuite) TestRateLimitMiddleware() { } } +func (suite *middlewareTestSuite) TestSwaggerUrl() { + leader := suite.cluster.GetServer(suite.cluster.GetLeader()) + req, _ := http.NewRequest(http.MethodGet, leader.GetAddr()+"/swagger/ui/index", nil) + resp, err := dialClient.Do(req) + suite.NoError(err) + suite.True(resp.StatusCode == http.StatusNotFound) + resp.Body.Close() +} + func (suite *middlewareTestSuite) TestAuditPrometheusBackend() { leader := suite.cluster.GetServer(suite.cluster.GetLeader()) input := map[string]interface{}{ @@ -445,6 +454,7 @@ func (suite *middlewareTestSuite) TestAuditPrometheusBackend() { func (suite *middlewareTestSuite) TestAuditLocalLogBackend() { tempStdoutFile, _ := os.CreateTemp("/tmp", "pd_tests") + defer os.Remove(tempStdoutFile.Name()) cfg := &log.Config{} cfg.File.Filename = tempStdoutFile.Name() cfg.Level = "info" @@ -471,8 +481,6 @@ func (suite *middlewareTestSuite) TestAuditLocalLogBackend() { suite.Contains(string(b), "audit log") suite.NoError(err) suite.Equal(http.StatusOK, resp.StatusCode) - - os.Remove(tempStdoutFile.Name()) } func BenchmarkDoRequestWithLocalLogAudit(b *testing.B) { @@ -656,6 +664,32 @@ func (suite *redirectorTestSuite) TestNotLeader() { suite.NoError(err) } +func (suite *redirectorTestSuite) TestXForwardedFor() { + leader := suite.cluster.GetServer(suite.cluster.GetLeader()) + suite.NoError(leader.BootstrapCluster()) + tempStdoutFile, _ := os.CreateTemp("/tmp", "pd_tests") + defer os.Remove(tempStdoutFile.Name()) + cfg := &log.Config{} + cfg.File.Filename = tempStdoutFile.Name() + cfg.Level = "info" + lg, p, _ := log.InitLogger(cfg) + log.ReplaceGlobals(lg, p) + + follower := suite.cluster.GetServer(suite.cluster.GetFollower()) + addr := follower.GetAddr() + "/pd/api/v1/regions" + request, err := http.NewRequest(http.MethodGet, addr, nil) + suite.NoError(err) + resp, err := dialClient.Do(request) + suite.NoError(err) + defer resp.Body.Close() + suite.Equal(http.StatusOK, resp.StatusCode) + time.Sleep(1 * time.Second) + b, _ := os.ReadFile(tempStdoutFile.Name()) + l := string(b) + suite.Contains(l, "/pd/api/v1/regions") + suite.NotContains(l, suite.cluster.GetConfig().GetClientURLs()) +} + func mustRequestSuccess(re *require.Assertions, s *server.Server) http.Header { resp, err := dialClient.Get(s.GetAddr() + "/pd/api/v1/version") re.NoError(err) diff --git a/tests/server/apiv2/handlers/keyspace_test.go b/tests/server/apiv2/handlers/keyspace_test.go index e13e13737d7..7fd8de013f7 100644 --- a/tests/server/apiv2/handlers/keyspace_test.go +++ b/tests/server/apiv2/handlers/keyspace_test.go @@ -12,36 +12,24 @@ // See the License for the specific language governing permissions and // limitations under the License. -package handlers_test +package handlers import ( - "bytes" "context" - "encoding/json" "fmt" - "io" - "net/http" "testing" + "github.com/pingcap/failpoint" "github.com/pingcap/kvproto/pkg/keyspacepb" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" + "github.com/tikv/pd/pkg/mcs/utils" "github.com/tikv/pd/pkg/utils/testutil" "github.com/tikv/pd/server/apiv2/handlers" - "github.com/tikv/pd/server/keyspace" "github.com/tikv/pd/tests" "go.uber.org/goleak" ) -const keyspacesPrefix = "/pd/api/v2/keyspaces" - -// dialClient used to dial http request. -var dialClient = &http.Client{ - Transport: &http.Transport{ - DisableKeepAlives: true, - }, -} - func TestMain(m *testing.M) { goleak.VerifyTestMain(m, testutil.LeakOptions...) } @@ -67,11 +55,13 @@ func (suite *keyspaceTestSuite) SetupTest() { suite.NotEmpty(cluster.WaitLeader()) suite.server = cluster.GetServer(cluster.GetLeader()) suite.NoError(suite.server.BootstrapCluster()) + suite.NoError(failpoint.Enable("github.com/tikv/pd/pkg/keyspace/skipSplitRegion", "return(true)")) } func (suite *keyspaceTestSuite) TearDownTest() { suite.cleanup() suite.cluster.Destroy() + suite.NoError(failpoint.Disable("github.com/tikv/pd/pkg/keyspace/skipSplitRegion")) } func (suite *keyspaceTestSuite) TestCreateLoadKeyspace() { @@ -81,8 +71,8 @@ func (suite *keyspaceTestSuite) TestCreateLoadKeyspace() { loaded := mustLoadKeyspaces(re, suite.server, created.Name) re.Equal(created, loaded) } - defaultKeyspace := mustLoadKeyspaces(re, suite.server, keyspace.DefaultKeyspaceName) - re.Equal(keyspace.DefaultKeyspaceName, defaultKeyspace.Name) + defaultKeyspace := mustLoadKeyspaces(re, suite.server, utils.DefaultKeyspaceName) + re.Equal(utils.DefaultKeyspaceName, defaultKeyspace.Name) re.Equal(keyspacepb.KeyspaceState_ENABLED, defaultKeyspace.State) } @@ -133,7 +123,7 @@ func (suite *keyspaceTestSuite) TestUpdateKeyspaceState() { re.Equal(keyspacepb.KeyspaceState_TOMBSTONE, tombstone.State) } // Changing default keyspace's state is NOT allowed. - success, _ := sendUpdateStateRequest(re, suite.server, keyspace.DefaultKeyspaceName, &handlers.UpdateStateParam{State: "disabled"}) + success, _ := sendUpdateStateRequest(re, suite.server, utils.DefaultKeyspaceName, &handlers.UpdateStateParam{State: "disabled"}) re.False(success) } @@ -147,48 +137,10 @@ func (suite *keyspaceTestSuite) TestLoadRangeKeyspace() { for i, created := range keyspaces { re.Equal(created, loadResponse.Keyspaces[i+1].KeyspaceMeta) } - re.Equal(keyspace.DefaultKeyspaceName, loadResponse.Keyspaces[0].Name) + re.Equal(utils.DefaultKeyspaceName, loadResponse.Keyspaces[0].Name) re.Equal(keyspacepb.KeyspaceState_ENABLED, loadResponse.Keyspaces[0].State) } -func sendLoadRangeRequest(re *require.Assertions, server *tests.TestServer, token, limit string) *handlers.LoadAllKeyspacesResponse { - // Construct load range request. - httpReq, err := http.NewRequest(http.MethodGet, server.GetAddr()+keyspacesPrefix, nil) - re.NoError(err) - query := httpReq.URL.Query() - query.Add("page_token", token) - query.Add("limit", limit) - httpReq.URL.RawQuery = query.Encode() - // Send request. - httpResp, err := dialClient.Do(httpReq) - re.NoError(err) - defer httpResp.Body.Close() - re.Equal(http.StatusOK, httpResp.StatusCode) - // Receive & decode response. - data, err := io.ReadAll(httpResp.Body) - re.NoError(err) - resp := &handlers.LoadAllKeyspacesResponse{} - re.NoError(json.Unmarshal(data, resp)) - return resp -} - -func sendUpdateStateRequest(re *require.Assertions, server *tests.TestServer, name string, request *handlers.UpdateStateParam) (bool, *keyspacepb.KeyspaceMeta) { - data, err := json.Marshal(request) - re.NoError(err) - httpReq, err := http.NewRequest(http.MethodPut, server.GetAddr()+keyspacesPrefix+"/"+name+"/state", bytes.NewBuffer(data)) - re.NoError(err) - httpResp, err := dialClient.Do(httpReq) - re.NoError(err) - defer httpResp.Body.Close() - if httpResp.StatusCode != http.StatusOK { - return false, nil - } - data, err = io.ReadAll(httpResp.Body) - re.NoError(err) - meta := &handlers.KeyspaceMeta{} - re.NoError(json.Unmarshal(data, meta)) - return true, meta.KeyspaceMeta -} func mustMakeTestKeyspaces(re *require.Assertions, server *tests.TestServer, count int) []*keyspacepb.KeyspaceMeta { testConfig := map[string]string{ "config1": "100", @@ -197,66 +149,14 @@ func mustMakeTestKeyspaces(re *require.Assertions, server *tests.TestServer, cou resultMeta := make([]*keyspacepb.KeyspaceMeta, count) for i := 0; i < count; i++ { createRequest := &handlers.CreateKeyspaceParams{ - Name: fmt.Sprintf("test_keyspace%d", i), + Name: fmt.Sprintf("test_keyspace_%d", i), Config: testConfig, } - resultMeta[i] = mustCreateKeyspace(re, server, createRequest) + resultMeta[i] = MustCreateKeyspace(re, server, createRequest) } return resultMeta } -func mustCreateKeyspace(re *require.Assertions, server *tests.TestServer, request *handlers.CreateKeyspaceParams) *keyspacepb.KeyspaceMeta { - data, err := json.Marshal(request) - re.NoError(err) - httpReq, err := http.NewRequest(http.MethodPost, server.GetAddr()+keyspacesPrefix, bytes.NewBuffer(data)) - re.NoError(err) - resp, err := dialClient.Do(httpReq) - re.NoError(err) - defer resp.Body.Close() - re.Equal(http.StatusOK, resp.StatusCode) - data, err = io.ReadAll(resp.Body) - re.NoError(err) - meta := &handlers.KeyspaceMeta{} - re.NoError(json.Unmarshal(data, meta)) - checkCreateRequest(re, request, meta.KeyspaceMeta) - return meta.KeyspaceMeta -} - -func mustUpdateKeyspaceConfig(re *require.Assertions, server *tests.TestServer, name string, request *handlers.UpdateConfigParams) *keyspacepb.KeyspaceMeta { - data, err := json.Marshal(request) - re.NoError(err) - httpReq, err := http.NewRequest(http.MethodPatch, server.GetAddr()+keyspacesPrefix+"/"+name+"/config", bytes.NewBuffer(data)) - re.NoError(err) - resp, err := dialClient.Do(httpReq) - re.NoError(err) - defer resp.Body.Close() - re.Equal(http.StatusOK, resp.StatusCode) - data, err = io.ReadAll(resp.Body) - re.NoError(err) - meta := &handlers.KeyspaceMeta{} - re.NoError(json.Unmarshal(data, meta)) - return meta.KeyspaceMeta -} - -func mustLoadKeyspaces(re *require.Assertions, server *tests.TestServer, name string) *keyspacepb.KeyspaceMeta { - resp, err := dialClient.Get(server.GetAddr() + keyspacesPrefix + "/" + name) - re.NoError(err) - defer resp.Body.Close() - re.Equal(http.StatusOK, resp.StatusCode) - data, err := io.ReadAll(resp.Body) - re.NoError(err) - meta := &handlers.KeyspaceMeta{} - re.NoError(json.Unmarshal(data, meta)) - return meta.KeyspaceMeta -} - -// checkCreateRequest verifies a keyspace meta matches a create request. -func checkCreateRequest(re *require.Assertions, request *handlers.CreateKeyspaceParams, meta *keyspacepb.KeyspaceMeta) { - re.Equal(request.Name, meta.Name) - re.Equal(keyspacepb.KeyspaceState_ENABLED, meta.State) - re.Equal(request.Config, meta.Config) -} - // checkUpdateRequest verifies a keyspace meta matches a update request. func checkUpdateRequest(re *require.Assertions, request *handlers.UpdateConfigParams, oldConfig, newConfig map[string]string) { expected := map[string]string{} diff --git a/tests/server/apiv2/handlers/testutil.go b/tests/server/apiv2/handlers/testutil.go new file mode 100644 index 00000000000..aca29ebeb52 --- /dev/null +++ b/tests/server/apiv2/handlers/testutil.go @@ -0,0 +1,259 @@ +// Copyright 2023 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package handlers + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "net/http" + + "github.com/pingcap/kvproto/pkg/keyspacepb" + "github.com/stretchr/testify/require" + "github.com/tikv/pd/pkg/storage/endpoint" + "github.com/tikv/pd/server/apiv2/handlers" + "github.com/tikv/pd/tests" +) + +const ( + keyspacesPrefix = "/pd/api/v2/keyspaces" + keyspaceGroupsPrefix = "/pd/api/v2/tso/keyspace-groups" +) + +// dialClient used to dial http request. +var dialClient = &http.Client{ + Transport: &http.Transport{ + DisableKeepAlives: true, + }, +} + +func sendLoadRangeRequest(re *require.Assertions, server *tests.TestServer, token, limit string) *handlers.LoadAllKeyspacesResponse { + // Construct load range request. + httpReq, err := http.NewRequest(http.MethodGet, server.GetAddr()+keyspacesPrefix, nil) + re.NoError(err) + query := httpReq.URL.Query() + query.Add("page_token", token) + query.Add("limit", limit) + httpReq.URL.RawQuery = query.Encode() + // Send request. + httpResp, err := dialClient.Do(httpReq) + re.NoError(err) + defer httpResp.Body.Close() + re.Equal(http.StatusOK, httpResp.StatusCode) + // Receive & decode response. + data, err := io.ReadAll(httpResp.Body) + re.NoError(err) + resp := &handlers.LoadAllKeyspacesResponse{} + re.NoError(json.Unmarshal(data, resp)) + return resp +} + +func sendUpdateStateRequest(re *require.Assertions, server *tests.TestServer, name string, request *handlers.UpdateStateParam) (bool, *keyspacepb.KeyspaceMeta) { + data, err := json.Marshal(request) + re.NoError(err) + httpReq, err := http.NewRequest(http.MethodPut, server.GetAddr()+keyspacesPrefix+"/"+name+"/state", bytes.NewBuffer(data)) + re.NoError(err) + httpResp, err := dialClient.Do(httpReq) + re.NoError(err) + defer httpResp.Body.Close() + if httpResp.StatusCode != http.StatusOK { + return false, nil + } + data, err = io.ReadAll(httpResp.Body) + re.NoError(err) + meta := &handlers.KeyspaceMeta{} + re.NoError(json.Unmarshal(data, meta)) + return true, meta.KeyspaceMeta +} + +// MustCreateKeyspace creates a keyspace with HTTP API. +func MustCreateKeyspace(re *require.Assertions, server *tests.TestServer, request *handlers.CreateKeyspaceParams) *keyspacepb.KeyspaceMeta { + data, err := json.Marshal(request) + re.NoError(err) + httpReq, err := http.NewRequest(http.MethodPost, server.GetAddr()+keyspacesPrefix, bytes.NewBuffer(data)) + re.NoError(err) + resp, err := dialClient.Do(httpReq) + re.NoError(err) + defer resp.Body.Close() + re.Equal(http.StatusOK, resp.StatusCode) + data, err = io.ReadAll(resp.Body) + re.NoError(err) + meta := &handlers.KeyspaceMeta{} + re.NoError(json.Unmarshal(data, meta)) + checkCreateRequest(re, request, meta.KeyspaceMeta) + return meta.KeyspaceMeta +} + +// checkCreateRequest verifies a keyspace meta matches a create request. +func checkCreateRequest(re *require.Assertions, request *handlers.CreateKeyspaceParams, meta *keyspacepb.KeyspaceMeta) { + re.Equal(request.Name, meta.Name) + re.Equal(keyspacepb.KeyspaceState_ENABLED, meta.State) + re.Equal(request.Config, meta.Config) +} + +func mustUpdateKeyspaceConfig(re *require.Assertions, server *tests.TestServer, name string, request *handlers.UpdateConfigParams) *keyspacepb.KeyspaceMeta { + data, err := json.Marshal(request) + re.NoError(err) + httpReq, err := http.NewRequest(http.MethodPatch, server.GetAddr()+keyspacesPrefix+"/"+name+"/config", bytes.NewBuffer(data)) + re.NoError(err) + resp, err := dialClient.Do(httpReq) + re.NoError(err) + defer resp.Body.Close() + re.Equal(http.StatusOK, resp.StatusCode) + data, err = io.ReadAll(resp.Body) + re.NoError(err) + meta := &handlers.KeyspaceMeta{} + re.NoError(json.Unmarshal(data, meta)) + return meta.KeyspaceMeta +} + +func mustLoadKeyspaces(re *require.Assertions, server *tests.TestServer, name string) *keyspacepb.KeyspaceMeta { + resp, err := dialClient.Get(server.GetAddr() + keyspacesPrefix + "/" + name) + re.NoError(err) + defer resp.Body.Close() + re.Equal(http.StatusOK, resp.StatusCode) + data, err := io.ReadAll(resp.Body) + re.NoError(err) + meta := &handlers.KeyspaceMeta{} + re.NoError(json.Unmarshal(data, meta)) + return meta.KeyspaceMeta +} + +// MustLoadKeyspaceGroups loads all keyspace groups from the server. +func MustLoadKeyspaceGroups(re *require.Assertions, server *tests.TestServer, token, limit string) []*endpoint.KeyspaceGroup { + // Construct load range request. + httpReq, err := http.NewRequest(http.MethodGet, server.GetAddr()+keyspaceGroupsPrefix, nil) + re.NoError(err) + query := httpReq.URL.Query() + query.Add("page_token", token) + query.Add("limit", limit) + httpReq.URL.RawQuery = query.Encode() + // Send request. + httpResp, err := dialClient.Do(httpReq) + re.NoError(err) + defer httpResp.Body.Close() + data, err := io.ReadAll(httpResp.Body) + re.NoError(err) + re.Equal(http.StatusOK, httpResp.StatusCode, string(data)) + var resp []*endpoint.KeyspaceGroup + re.NoError(json.Unmarshal(data, &resp)) + return resp +} + +func tryCreateKeyspaceGroup(re *require.Assertions, server *tests.TestServer, request *handlers.CreateKeyspaceGroupParams) (int, string) { + data, err := json.Marshal(request) + re.NoError(err) + httpReq, err := http.NewRequest(http.MethodPost, server.GetAddr()+keyspaceGroupsPrefix, bytes.NewBuffer(data)) + re.NoError(err) + resp, err := dialClient.Do(httpReq) + re.NoError(err) + defer resp.Body.Close() + data, err = io.ReadAll(resp.Body) + re.NoError(err) + return resp.StatusCode, string(data) +} + +// MustLoadKeyspaceGroupByID loads the keyspace group by ID with HTTP API. +func MustLoadKeyspaceGroupByID(re *require.Assertions, server *tests.TestServer, id uint32) *endpoint.KeyspaceGroup { + kg, code := TryLoadKeyspaceGroupByID(re, server, id) + re.Equal(http.StatusOK, code) + return kg +} + +// TryLoadKeyspaceGroupByID loads the keyspace group by ID with HTTP API. +func TryLoadKeyspaceGroupByID(re *require.Assertions, server *tests.TestServer, id uint32) (*endpoint.KeyspaceGroup, int) { + httpReq, err := http.NewRequest(http.MethodGet, server.GetAddr()+keyspaceGroupsPrefix+fmt.Sprintf("/%d", id), nil) + re.NoError(err) + resp, err := dialClient.Do(httpReq) + re.NoError(err) + defer resp.Body.Close() + data, err := io.ReadAll(resp.Body) + re.NoError(err) + if resp.StatusCode != http.StatusOK { + return nil, resp.StatusCode + } + + var kg endpoint.KeyspaceGroup + re.NoError(json.Unmarshal(data, &kg)) + return &kg, resp.StatusCode +} + +// MustCreateKeyspaceGroup creates a keyspace group with HTTP API. +func MustCreateKeyspaceGroup(re *require.Assertions, server *tests.TestServer, request *handlers.CreateKeyspaceGroupParams) { + code, data := tryCreateKeyspaceGroup(re, server, request) + re.Equal(http.StatusOK, code, data) +} + +// FailCreateKeyspaceGroupWithCode fails to create a keyspace group with HTTP API. +func FailCreateKeyspaceGroupWithCode(re *require.Assertions, server *tests.TestServer, request *handlers.CreateKeyspaceGroupParams, expect int) { + code, data := tryCreateKeyspaceGroup(re, server, request) + re.Equal(expect, code, data) +} + +// MustDeleteKeyspaceGroup deletes a keyspace group with HTTP API. +func MustDeleteKeyspaceGroup(re *require.Assertions, server *tests.TestServer, id uint32) { + httpReq, err := http.NewRequest(http.MethodDelete, server.GetAddr()+keyspaceGroupsPrefix+fmt.Sprintf("/%d", id), nil) + re.NoError(err) + resp, err := dialClient.Do(httpReq) + re.NoError(err) + defer resp.Body.Close() + data, err := io.ReadAll(resp.Body) + re.NoError(err) + re.Equal(http.StatusOK, resp.StatusCode, string(data)) +} + +// MustSplitKeyspaceGroup splits a keyspace group with HTTP API. +func MustSplitKeyspaceGroup(re *require.Assertions, server *tests.TestServer, id uint32, request *handlers.SplitKeyspaceGroupByIDParams) { + data, err := json.Marshal(request) + re.NoError(err) + httpReq, err := http.NewRequest(http.MethodPost, server.GetAddr()+keyspaceGroupsPrefix+fmt.Sprintf("/%d/split", id), bytes.NewBuffer(data)) + re.NoError(err) + // Send request. + resp, err := dialClient.Do(httpReq) + re.NoError(err) + defer resp.Body.Close() + data, err = io.ReadAll(resp.Body) + re.NoError(err) + re.Equal(http.StatusOK, resp.StatusCode, string(data)) +} + +// MustFinishSplitKeyspaceGroup finishes a keyspace group split with HTTP API. +func MustFinishSplitKeyspaceGroup(re *require.Assertions, server *tests.TestServer, id uint32) { + httpReq, err := http.NewRequest(http.MethodDelete, server.GetAddr()+keyspaceGroupsPrefix+fmt.Sprintf("/%d/split", id), nil) + re.NoError(err) + // Send request. + resp, err := dialClient.Do(httpReq) + re.NoError(err) + defer resp.Body.Close() + data, err := io.ReadAll(resp.Body) + re.NoError(err) + re.Equal(http.StatusOK, resp.StatusCode, string(data)) +} + +// MustMergeKeyspaceGroup merges keyspace groups with HTTP API. +func MustMergeKeyspaceGroup(re *require.Assertions, server *tests.TestServer, id uint32, request *handlers.MergeKeyspaceGroupsParams) { + data, err := json.Marshal(request) + re.NoError(err) + httpReq, err := http.NewRequest(http.MethodPost, server.GetAddr()+keyspaceGroupsPrefix+fmt.Sprintf("/%d/merge", id), bytes.NewBuffer(data)) + re.NoError(err) + // Send request. + resp, err := dialClient.Do(httpReq) + re.NoError(err) + defer resp.Body.Close() + data, err = io.ReadAll(resp.Body) + re.NoError(err) + re.Equal(http.StatusOK, resp.StatusCode, string(data)) +} diff --git a/tests/server/apiv2/handlers/tso_keyspace_group_test.go b/tests/server/apiv2/handlers/tso_keyspace_group_test.go new file mode 100644 index 00000000000..1f0189c532f --- /dev/null +++ b/tests/server/apiv2/handlers/tso_keyspace_group_test.go @@ -0,0 +1,174 @@ +// Copyright 2023 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package handlers + +import ( + "context" + "net/http" + "testing" + + "github.com/stretchr/testify/suite" + "github.com/tikv/pd/pkg/mcs/utils" + "github.com/tikv/pd/pkg/storage/endpoint" + "github.com/tikv/pd/server/apiv2/handlers" + "github.com/tikv/pd/tests" +) + +type keyspaceGroupTestSuite struct { + suite.Suite + ctx context.Context + cancel context.CancelFunc + cluster *tests.TestCluster + server *tests.TestServer +} + +func TestKeyspaceGroupTestSuite(t *testing.T) { + suite.Run(t, new(keyspaceGroupTestSuite)) +} + +func (suite *keyspaceGroupTestSuite) SetupTest() { + suite.ctx, suite.cancel = context.WithCancel(context.Background()) + cluster, err := tests.NewTestAPICluster(suite.ctx, 1) + suite.cluster = cluster + suite.NoError(err) + suite.NoError(cluster.RunInitialServers()) + suite.NotEmpty(cluster.WaitLeader()) + suite.server = cluster.GetServer(cluster.GetLeader()) + suite.NoError(suite.server.BootstrapCluster()) +} + +func (suite *keyspaceGroupTestSuite) TearDownTest() { + suite.cancel() + suite.cluster.Destroy() +} + +func (suite *keyspaceGroupTestSuite) TestCreateKeyspaceGroups() { + re := suite.Require() + kgs := &handlers.CreateKeyspaceGroupParams{KeyspaceGroups: []*endpoint.KeyspaceGroup{ + { + ID: uint32(1), + UserKind: endpoint.Standard.String(), + }, + { + ID: uint32(2), + UserKind: endpoint.Standard.String(), + }, + }} + MustCreateKeyspaceGroup(re, suite.server, kgs) + + // miss user kind, use default value. + kgs = &handlers.CreateKeyspaceGroupParams{KeyspaceGroups: []*endpoint.KeyspaceGroup{ + { + ID: uint32(3), + }, + }} + MustCreateKeyspaceGroup(re, suite.server, kgs) + + // invalid user kind. + kgs = &handlers.CreateKeyspaceGroupParams{KeyspaceGroups: []*endpoint.KeyspaceGroup{ + { + ID: uint32(4), + UserKind: "invalid", + }, + }} + FailCreateKeyspaceGroupWithCode(re, suite.server, kgs, http.StatusBadRequest) + + // miss ID. + kgs = &handlers.CreateKeyspaceGroupParams{KeyspaceGroups: []*endpoint.KeyspaceGroup{ + { + UserKind: endpoint.Standard.String(), + }, + }} + FailCreateKeyspaceGroupWithCode(re, suite.server, kgs, http.StatusInternalServerError) + + // invalid ID. + kgs = &handlers.CreateKeyspaceGroupParams{KeyspaceGroups: []*endpoint.KeyspaceGroup{ + { + ID: utils.MaxKeyspaceGroupCount + 1, + UserKind: endpoint.Standard.String(), + }, + }} + FailCreateKeyspaceGroupWithCode(re, suite.server, kgs, http.StatusBadRequest) + + // repeated ID. + kgs = &handlers.CreateKeyspaceGroupParams{KeyspaceGroups: []*endpoint.KeyspaceGroup{ + { + ID: uint32(2), + UserKind: endpoint.Standard.String(), + }, + }} + FailCreateKeyspaceGroupWithCode(re, suite.server, kgs, http.StatusInternalServerError) +} + +func (suite *keyspaceGroupTestSuite) TestLoadKeyspaceGroup() { + re := suite.Require() + kgs := &handlers.CreateKeyspaceGroupParams{KeyspaceGroups: []*endpoint.KeyspaceGroup{ + { + ID: uint32(1), + UserKind: endpoint.Standard.String(), + }, + { + ID: uint32(2), + UserKind: endpoint.Standard.String(), + }, + }} + + MustCreateKeyspaceGroup(re, suite.server, kgs) + resp := MustLoadKeyspaceGroups(re, suite.server, "0", "0") + re.Len(resp, 3) +} + +func (suite *keyspaceGroupTestSuite) TestSplitKeyspaceGroup() { + re := suite.Require() + kgs := &handlers.CreateKeyspaceGroupParams{KeyspaceGroups: []*endpoint.KeyspaceGroup{ + { + ID: uint32(1), + UserKind: endpoint.Standard.String(), + Keyspaces: []uint32{111, 222, 333}, + Members: make([]endpoint.KeyspaceGroupMember, utils.DefaultKeyspaceGroupReplicaCount), + }, + }} + + MustCreateKeyspaceGroup(re, suite.server, kgs) + resp := MustLoadKeyspaceGroups(re, suite.server, "0", "0") + re.Len(resp, 2) + MustSplitKeyspaceGroup(re, suite.server, 1, &handlers.SplitKeyspaceGroupByIDParams{ + NewID: uint32(2), + Keyspaces: []uint32{111, 222}, + }) + resp = MustLoadKeyspaceGroups(re, suite.server, "0", "0") + re.Len(resp, 3) + // Check keyspace group 1. + kg1 := MustLoadKeyspaceGroupByID(re, suite.server, 1) + re.Equal(uint32(1), kg1.ID) + re.Equal([]uint32{333}, kg1.Keyspaces) + re.True(kg1.IsSplitSource()) + re.Equal(kg1.ID, kg1.SplitSource()) + // Check keyspace group 2. + kg2 := MustLoadKeyspaceGroupByID(re, suite.server, 2) + re.Equal(uint32(2), kg2.ID) + re.Equal([]uint32{111, 222}, kg2.Keyspaces) + re.True(kg2.IsSplitTarget()) + re.Equal(kg1.ID, kg2.SplitSource()) + // They should have the same user kind and members. + re.Equal(kg1.UserKind, kg2.UserKind) + re.Equal(kg1.Members, kg2.Members) + // Finish the split and check the split state. + MustFinishSplitKeyspaceGroup(re, suite.server, 2) + kg1 = MustLoadKeyspaceGroupByID(re, suite.server, 1) + re.False(kg1.IsSplitting()) + kg2 = MustLoadKeyspaceGroupByID(re, suite.server, 2) + re.False(kg2.IsSplitting()) +} diff --git a/tests/server/cluster/cluster_test.go b/tests/server/cluster/cluster_test.go index 9761ee3415d..c15520aca3c 100644 --- a/tests/server/cluster/cluster_test.go +++ b/tests/server/cluster/cluster_test.go @@ -35,8 +35,10 @@ import ( "github.com/tikv/pd/pkg/dashboard" "github.com/tikv/pd/pkg/id" "github.com/tikv/pd/pkg/mock/mockid" + sc "github.com/tikv/pd/pkg/schedule/config" "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/storage" + "github.com/tikv/pd/pkg/syncer" "github.com/tikv/pd/pkg/tso" "github.com/tikv/pd/pkg/utils/testutil" "github.com/tikv/pd/pkg/utils/tsoutil" @@ -44,7 +46,6 @@ import ( "github.com/tikv/pd/server" "github.com/tikv/pd/server/cluster" "github.com/tikv/pd/server/config" - syncer "github.com/tikv/pd/server/region_syncer" "github.com/tikv/pd/tests" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -294,15 +295,15 @@ func testPutStore(re *require.Assertions, clusterID uint64, rc *cluster.RaftClus re.NoError(err) re.Equal(pdpb.ErrorType_OK, resp.GetHeader().GetError().GetType()) - rc.GetAllocator().Alloc() - id, err := rc.GetAllocator().Alloc() + rc.AllocID() + id, err := rc.AllocID() re.NoError(err) // Put new store with a duplicated address when old store is up will fail. resp, err = putStore(grpcPDClient, clusterID, newMetaStore(id, store.GetAddress(), "2.1.0", metapb.StoreState_Up, getTestDeployPath(id))) re.NoError(err) re.Equal(pdpb.ErrorType_UNKNOWN, resp.GetHeader().GetError().GetType()) - id, err = rc.GetAllocator().Alloc() + id, err = rc.AllocID() re.NoError(err) // Put new store with a duplicated address when old store is offline will fail. resetStoreState(re, rc, store.GetId(), metapb.StoreState_Offline) @@ -310,7 +311,7 @@ func testPutStore(re *require.Assertions, clusterID uint64, rc *cluster.RaftClus re.NoError(err) re.Equal(pdpb.ErrorType_UNKNOWN, resp.GetHeader().GetError().GetType()) - id, err = rc.GetAllocator().Alloc() + id, err = rc.AllocID() re.NoError(err) // Put new store with a duplicated address when old store is tombstone is OK. resetStoreState(re, rc, store.GetId(), metapb.StoreState_Tombstone) @@ -319,7 +320,7 @@ func testPutStore(re *require.Assertions, clusterID uint64, rc *cluster.RaftClus re.NoError(err) re.Equal(pdpb.ErrorType_OK, resp.GetHeader().GetError().GetType()) - id, err = rc.GetAllocator().Alloc() + id, err = rc.AllocID() re.NoError(err) deployPath := getTestDeployPath(id) // Put a new store. @@ -626,7 +627,6 @@ func TestConcurrentHandleRegion(t *testing.T) { storeAddrs := []string{"127.0.1.1:0", "127.0.1.1:1", "127.0.1.1:2"} rc := leaderServer.GetRaftCluster() re.NotNil(rc) - rc.SetStorage(storage.NewStorageWithMemoryBackend()) stores := make([]*metapb.Store, 0, len(storeAddrs)) id := leaderServer.GetAllocator() for _, addr := range storeAddrs { @@ -813,7 +813,7 @@ func TestLoadClusterInfo(t *testing.T) { rc := cluster.NewRaftCluster(ctx, svr.ClusterID(), syncer.NewRegionSyncer(svr), svr.GetClient(), svr.GetHTTPClient()) // Cluster is not bootstrapped. - rc.InitCluster(svr.GetAllocator(), svr.GetPersistOptions(), svr.GetStorage(), svr.GetBasicCluster()) + rc.InitCluster(svr.GetAllocator(), svr.GetPersistOptions(), svr.GetStorage(), svr.GetBasicCluster(), svr.GetKeyspaceGroupManager()) raftCluster, err := rc.LoadClusterInfo() re.NoError(err) re.Nil(raftCluster) @@ -831,7 +831,7 @@ func TestLoadClusterInfo(t *testing.T) { } for _, store := range stores { - re.NoError(testStorage.SaveStore(store)) + re.NoError(testStorage.SaveStoreMeta(store)) } regions := make([]*metapb.Region, 0, n) @@ -851,7 +851,7 @@ func TestLoadClusterInfo(t *testing.T) { re.NoError(testStorage.Flush()) raftCluster = cluster.NewRaftCluster(ctx, svr.ClusterID(), syncer.NewRegionSyncer(svr), svr.GetClient(), svr.GetHTTPClient()) - raftCluster.InitCluster(mockid.NewIDAllocator(), svr.GetPersistOptions(), testStorage, basicCluster) + raftCluster.InitCluster(mockid.NewIDAllocator(), svr.GetPersistOptions(), testStorage, basicCluster, svr.GetKeyspaceGroupManager()) raftCluster, err = raftCluster.LoadClusterInfo() re.NoError(err) re.NotNil(raftCluster) @@ -862,7 +862,7 @@ func TestLoadClusterInfo(t *testing.T) { for _, store := range raftCluster.GetMetaStores() { re.Equal(stores[store.GetId()], store) } - re.Equal(n, raftCluster.GetRegionCount()) + re.Equal(n, raftCluster.GetTotalRegionCount()) for _, region := range raftCluster.GetMetaRegions() { re.Equal(regions[region.GetId()], region) } @@ -883,7 +883,7 @@ func TestLoadClusterInfo(t *testing.T) { re.NoError(testStorage.SaveRegion(region)) } re.NoError(storage.TryLoadRegionsOnce(ctx, testStorage, raftCluster.GetBasicCluster().PutRegion)) - re.Equal(n, raftCluster.GetRegionCount()) + re.Equal(n, raftCluster.GetTotalRegionCount()) } func TestTiFlashWithPlacementRules(t *testing.T) { @@ -922,7 +922,7 @@ func TestTiFlashWithPlacementRules(t *testing.T) { re.NoError(err) re.Equal(pdpb.ErrorType_OK, resp.GetHeader().GetError().GetType()) // test TiFlash store limit - expect := map[uint64]config.StoreLimitConfig{11: {AddPeer: 30, RemovePeer: 30}} + expect := map[uint64]sc.StoreLimitConfig{11: {AddPeer: 30, RemovePeer: 30}} re.Equal(expect, svr.GetScheduleConfig().StoreLimit) // cannot disable placement rules with TiFlash nodes @@ -1056,7 +1056,6 @@ func TestOfflineStoreLimit(t *testing.T) { storeAddrs := []string{"127.0.1.1:0", "127.0.1.1:1"} rc := leaderServer.GetRaftCluster() re.NotNil(rc) - rc.SetStorage(storage.NewStorageWithMemoryBackend()) id := leaderServer.GetAllocator() for _, addr := range storeAddrs { storeID, err := id.Alloc() @@ -1148,7 +1147,6 @@ func TestUpgradeStoreLimit(t *testing.T) { bootstrapCluster(re, clusterID, grpcPDClient) rc := leaderServer.GetRaftCluster() re.NotNil(rc) - rc.SetStorage(storage.NewStorageWithMemoryBackend()) store := newMetaStore(1, "127.0.1.1:0", "4.0.0", metapb.StoreState_Up, "test/store1") resp, err := putStore(grpcPDClient, clusterID, store) re.NoError(err) @@ -1171,7 +1169,7 @@ func TestUpgradeStoreLimit(t *testing.T) { // restart PD // Here we use an empty storelimit to simulate the upgrade progress. scheduleCfg := rc.GetScheduleConfig().Clone() - scheduleCfg.StoreLimit = map[uint64]config.StoreLimitConfig{} + scheduleCfg.StoreLimit = map[uint64]sc.StoreLimitConfig{} re.NoError(leaderServer.GetServer().SetScheduleConfig(*scheduleCfg)) err = leaderServer.Stop() re.NoError(err) @@ -1208,7 +1206,6 @@ func TestStaleTermHeartbeat(t *testing.T) { storeAddrs := []string{"127.0.1.1:0", "127.0.1.1:1", "127.0.1.1:2"} rc := leaderServer.GetRaftCluster() re.NotNil(rc) - rc.SetStorage(storage.NewStorageWithMemoryBackend()) peers := make([]*metapb.Peer, 0, len(storeAddrs)) id := leaderServer.GetAllocator() for _, addr := range storeAddrs { @@ -1438,7 +1435,7 @@ func TestTransferLeaderBack(t *testing.T) { leaderServer := tc.GetServer(tc.GetLeader()) svr := leaderServer.GetServer() rc := cluster.NewRaftCluster(ctx, svr.ClusterID(), syncer.NewRegionSyncer(svr), svr.GetClient(), svr.GetHTTPClient()) - rc.InitCluster(svr.GetAllocator(), svr.GetPersistOptions(), svr.GetStorage(), svr.GetBasicCluster()) + rc.InitCluster(svr.GetAllocator(), svr.GetPersistOptions(), svr.GetStorage(), svr.GetBasicCluster(), svr.GetKeyspaceGroupManager()) storage := rc.GetStorage() meta := &metapb.Cluster{Id: 123} re.NoError(storage.SaveMeta(meta)) @@ -1450,7 +1447,7 @@ func TestTransferLeaderBack(t *testing.T) { } for _, store := range stores { - re.NoError(storage.SaveStore(store)) + re.NoError(storage.SaveStoreMeta(store)) } rc, err = rc.LoadClusterInfo() re.NoError(err) diff --git a/tests/server/keyspace/keyspace_test.go b/tests/server/keyspace/keyspace_test.go index f53cf3dc9fd..a36a7379550 100644 --- a/tests/server/keyspace/keyspace_test.go +++ b/tests/server/keyspace/keyspace_test.go @@ -27,16 +27,16 @@ import ( "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" "github.com/tikv/pd/pkg/codec" + "github.com/tikv/pd/pkg/keyspace" "github.com/tikv/pd/pkg/schedule/labeler" "github.com/tikv/pd/pkg/storage/endpoint" "github.com/tikv/pd/server/config" - "github.com/tikv/pd/server/keyspace" "github.com/tikv/pd/tests" ) type keyspaceTestSuite struct { suite.Suite - cleanup func() + cancel context.CancelFunc cluster *tests.TestCluster server *tests.TestServer manager *keyspace.Manager @@ -51,7 +51,7 @@ func TestKeyspaceTestSuite(t *testing.T) { func (suite *keyspaceTestSuite) SetupTest() { ctx, cancel := context.WithCancel(context.Background()) - suite.cleanup = cancel + suite.cancel = cancel cluster, err := tests.NewTestCluster(ctx, 3, func(conf *config.Config, serverName string) { conf.Keyspace.PreAlloc = preAllocKeyspace }) @@ -65,7 +65,7 @@ func (suite *keyspaceTestSuite) SetupTest() { } func (suite *keyspaceTestSuite) TearDownTest() { - suite.cleanup() + suite.cancel() suite.cluster.Destroy() } @@ -81,8 +81,9 @@ func (suite *keyspaceTestSuite) TestRegionLabeler() { var err error for i := 0; i < count; i++ { keyspaces[i], err = manager.CreateKeyspace(&keyspace.CreateKeyspaceRequest{ - Name: fmt.Sprintf("test_keyspace%d", i), - Now: now, + Name: fmt.Sprintf("test_keyspace_%d", i), + CreateTime: now, + IsPreAlloc: true, // skip wait region split }) re.NoError(err) } diff --git a/tests/server/member/member_test.go b/tests/server/member/member_test.go index 1cdc267a99a..ca89e66a041 100644 --- a/tests/server/member/member_test.go +++ b/tests/server/member/member_test.go @@ -248,6 +248,30 @@ func TestLeaderResignWithBlock(t *testing.T) { re.NoError(failpoint.Disable("github.com/tikv/pd/server/raftclusterIsBusy")) } +func TestPDLeaderLostWhileEtcdLeaderIntact(t *testing.T) { + re := require.New(t) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + cluster, err := tests.NewTestCluster(ctx, 2) + defer cluster.Destroy() + re.NoError(err) + + err = cluster.RunInitialServers() + re.NoError(err) + + leader1 := cluster.WaitLeader() + memberID := cluster.GetServer(leader1).GetLeader().GetMemberId() + + re.NoError(failpoint.Enable("github.com/tikv/pd/server/leaderLoopCheckAgain", fmt.Sprintf("return(\"%d\")", memberID))) + re.NoError(failpoint.Enable("github.com/tikv/pd/server/exitCampaignLeader", fmt.Sprintf("return(\"%d\")", memberID))) + re.NoError(failpoint.Enable("github.com/tikv/pd/server/timeoutWaitPDLeader", `return(true)`)) + leader2 := waitLeaderChange(re, cluster, leader1) + re.NotEqual(leader1, leader2) + re.NoError(failpoint.Disable("github.com/tikv/pd/server/leaderLoopCheckAgain")) + re.NoError(failpoint.Disable("github.com/tikv/pd/server/exitCampaignLeader")) + re.NoError(failpoint.Disable("github.com/tikv/pd/server/timeoutWaitPDLeader")) +} + func waitLeaderChange(re *require.Assertions, cluster *tests.TestCluster, old string) string { var leader string testutil.Eventually(re, func() bool { diff --git a/tests/server/region_syncer/region_syncer_test.go b/tests/server/region_syncer/region_syncer_test.go index aa9f76d157d..afa5c87cdcc 100644 --- a/tests/server/region_syncer/region_syncer_test.go +++ b/tests/server/region_syncer/region_syncer_test.go @@ -49,7 +49,7 @@ func TestRegionSyncer(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/storage/regionStorageFastFlush", `return(true)`)) - re.NoError(failpoint.Enable("github.com/tikv/pd/server/syncer/noFastExitSync", `return(true)`)) + re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/syncer/noFastExitSync", `return(true)`)) cluster, err := tests.NewTestCluster(ctx, 3, func(conf *config.Config, serverName string) { conf.PDServerCfg.UseRegionStorage = true }) defer cluster.Destroy() @@ -151,7 +151,7 @@ func TestRegionSyncer(t *testing.T) { re.Equal(region.GetLeader(), r.GetLeader()) re.Equal(region.GetBuckets(), r.GetBuckets()) } - re.NoError(failpoint.Disable("github.com/tikv/pd/server/syncer/noFastExitSync")) + re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/syncer/noFastExitSync")) re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/storage/regionStorageFastFlush")) } @@ -202,7 +202,7 @@ func TestPrepareChecker(t *testing.T) { re := require.New(t) ctx, cancel := context.WithCancel(context.Background()) defer cancel() - re.NoError(failpoint.Enable("github.com/tikv/pd/server/cluster/changeCoordinatorTicker", `return(true)`)) + re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/schedule/changeCoordinatorTicker", `return(true)`)) cluster, err := tests.NewTestCluster(ctx, 1, func(conf *config.Config, serverName string) { conf.PDServerCfg.UseRegionStorage = true }) defer cluster.Destroy() re.NoError(err) @@ -243,7 +243,7 @@ func TestPrepareChecker(t *testing.T) { } time.Sleep(time.Second) re.True(rc.IsPrepared()) - re.NoError(failpoint.Disable("github.com/tikv/pd/server/cluster/changeCoordinatorTicker")) + re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/schedule/changeCoordinatorTicker")) } func initRegions(regionLen int) []*core.RegionInfo { diff --git a/tests/server/storage/hot_region_storage_test.go b/tests/server/storage/hot_region_storage_test.go index 44b1cfd274a..fee4944826c 100644 --- a/tests/server/storage/hot_region_storage_test.go +++ b/tests/server/storage/hot_region_storage_test.go @@ -192,7 +192,7 @@ func TestHotRegionStorageReservedDayConfigChange(t *testing.T) { re.NoError(err) re.Nil(next) schedule := leaderServer.GetConfig().Schedule - // set reserved day to zero,close hot region storage + // set reserved day to zero, close hot region storage schedule.HotRegionsReservedDays = 0 leaderServer.GetServer().SetScheduleConfig(schedule) time.Sleep(3 * interval) @@ -210,7 +210,7 @@ func TestHotRegionStorageReservedDayConfigChange(t *testing.T) { next, err = iter.Next() re.NoError(err) re.Nil(next) - // set reserved day to one,open hot region storage + // set reserved day to one, open hot region storage schedule.HotRegionsReservedDays = 1 leaderServer.GetServer().SetScheduleConfig(schedule) time.Sleep(3 * interval) diff --git a/tests/server/tso/allocator_test.go b/tests/server/tso/allocator_test.go index b326645164a..8b1ad3dc7d1 100644 --- a/tests/server/tso/allocator_test.go +++ b/tests/server/tso/allocator_test.go @@ -85,7 +85,7 @@ func TestAllocatorLeader(t *testing.T) { allocatorLeaderMemberIDs := make([]uint64, 0, dcLocationNum) for _, allocator := range allAllocatorLeaders { allocatorLeader, _ := allocator.(*tso.LocalTSOAllocator) - allocatorLeaderMemberIDs = append(allocatorLeaderMemberIDs, allocatorLeader.GetMember().GetMemberId()) + allocatorLeaderMemberIDs = append(allocatorLeaderMemberIDs, allocatorLeader.GetMember().ID()) } for _, server := range cluster.GetServers() { // Filter out Global TSO Allocator diff --git a/tests/server/tso/common_test.go b/tests/server/tso/common_test.go index f528103db84..877fcb10982 100644 --- a/tests/server/tso/common_test.go +++ b/tests/server/tso/common_test.go @@ -28,7 +28,7 @@ import ( ) const ( - tsoRequestConcurrencyNumber = 5 + tsoRequestConcurrencyNumber = 3 tsoRequestRound = 30 tsoCount = 10 ) diff --git a/tests/server/tso/consistency_test.go b/tests/server/tso/consistency_test.go index 27fd68fbbef..db6e2135d2b 100644 --- a/tests/server/tso/consistency_test.go +++ b/tests/server/tso/consistency_test.go @@ -25,7 +25,6 @@ import ( "github.com/pingcap/failpoint" "github.com/pingcap/kvproto/pkg/pdpb" - "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" "github.com/tikv/pd/pkg/tso" "github.com/tikv/pd/pkg/utils/grpcutil" @@ -61,67 +60,6 @@ func (suite *tsoConsistencyTestSuite) TearDownSuite() { suite.cancel() } -// TestNormalGlobalTSO is used to test the normal way of global TSO generation. -func (suite *tsoConsistencyTestSuite) TestNormalGlobalTSO() { - cluster, err := tests.NewTestCluster(suite.ctx, 1) - defer cluster.Destroy() - suite.NoError(err) - suite.NoError(cluster.RunInitialServers()) - cluster.WaitLeader() - - leaderServer := cluster.GetServer(cluster.GetLeader()) - grpcPDClient := testutil.MustNewGrpcClient(suite.Require(), leaderServer.GetAddr()) - clusterID := leaderServer.GetClusterID() - req := &pdpb.TsoRequest{ - Header: testutil.NewRequestHeader(clusterID), - Count: uint32(tsoCount), - DcLocation: tso.GlobalDCLocation, - } - suite.requestGlobalTSOConcurrently(grpcPDClient, req) - // Test Global TSO after the leader change - leaderServer.GetServer().GetMember().ResetLeader() - cluster.WaitLeader() - suite.requestGlobalTSOConcurrently(grpcPDClient, req) -} - -func (suite *tsoConsistencyTestSuite) requestGlobalTSOConcurrently(grpcPDClient pdpb.PDClient, req *pdpb.TsoRequest) { - var wg sync.WaitGroup - wg.Add(tsoRequestConcurrencyNumber) - for i := 0; i < tsoRequestConcurrencyNumber; i++ { - go func() { - defer wg.Done() - last := &pdpb.Timestamp{ - Physical: 0, - Logical: 0, - } - for j := 0; j < tsoRequestRound; j++ { - ts := suite.testGetNormalGlobalTimestamp(grpcPDClient, req) - // Check whether the TSO fallbacks - suite.Equal(1, tsoutil.CompareTimestamp(ts, last)) - last = ts - time.Sleep(10 * time.Millisecond) - } - }() - } - wg.Wait() -} - -func (suite *tsoConsistencyTestSuite) testGetNormalGlobalTimestamp(pdCli pdpb.PDClient, req *pdpb.TsoRequest) *pdpb.Timestamp { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - tsoClient, err := pdCli.Tso(ctx) - suite.NoError(err) - defer tsoClient.CloseSend() - suite.NoError(tsoClient.Send(req)) - resp, err := tsoClient.Recv() - suite.NoError(err) - suite.Equal(req.GetCount(), resp.GetCount()) - res := resp.GetTimestamp() - suite.Greater(res.GetPhysical(), int64(0)) - suite.GreaterOrEqual(uint32(res.GetLogical())>>res.GetSuffixBits(), req.GetCount()) - return res -} - // TestSynchronizedGlobalTSO is used to test the synchronized way of global TSO generation. func (suite *tsoConsistencyTestSuite) TestSynchronizedGlobalTSO() { dcLocationConfig := map[string]string{ @@ -396,60 +334,3 @@ func (suite *tsoConsistencyTestSuite) testTSO(cluster *tests.TestCluster, dcLoca } wg.Wait() } - -func TestFallbackTSOConsistency(t *testing.T) { - re := require.New(t) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/tso/fallBackSync", `return(true)`)) - re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/tso/fallBackUpdate", `return(true)`)) - var err error - cluster, err := tests.NewTestCluster(ctx, 1) - re.NoError(err) - defer cluster.Destroy() - - re.NoError(cluster.RunInitialServers()) - cluster.WaitLeader() - - server := cluster.GetServer(cluster.GetLeader()) - grpcPDClient := testutil.MustNewGrpcClient(re, server.GetAddr()) - svr := server.GetServer() - svr.Close() - re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/tso/fallBackSync")) - re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/tso/fallBackUpdate")) - re.NoError(svr.Run()) - cluster.WaitLeader() - var wg sync.WaitGroup - wg.Add(tsoRequestConcurrencyNumber) - for i := 0; i < tsoRequestConcurrencyNumber; i++ { - go func() { - defer wg.Done() - last := &pdpb.Timestamp{ - Physical: 0, - Logical: 0, - } - for j := 0; j < tsoRequestRound; j++ { - clusterID := server.GetClusterID() - req := &pdpb.TsoRequest{ - Header: testutil.NewRequestHeader(clusterID), - Count: 10, - DcLocation: tso.GlobalDCLocation, - } - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - tsoClient, err := grpcPDClient.Tso(ctx) - re.NoError(err) - defer tsoClient.CloseSend() - re.NoError(tsoClient.Send(req)) - resp, err := tsoClient.Recv() - re.NoError(err) - ts := checkAndReturnTimestampResponse(re, req, resp) - re.Equal(1, tsoutil.CompareTimestamp(ts, last)) - last = ts - time.Sleep(10 * time.Millisecond) - } - }() - } - wg.Wait() -} diff --git a/tests/server/tso/global_tso_test.go b/tests/server/tso/global_tso_test.go index 66f90f9e760..a6340e2671c 100644 --- a/tests/server/tso/global_tso_test.go +++ b/tests/server/tso/global_tso_test.go @@ -19,7 +19,6 @@ package tso_test import ( "context" - "sync" "testing" "time" @@ -43,62 +42,6 @@ import ( // which will coordinate and synchronize a TSO with other Local TSO Allocator // leaders. -func TestConcurrentlyReset(t *testing.T) { - re := require.New(t) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - cluster, err := tests.NewTestCluster(ctx, 1) - defer cluster.Destroy() - re.NoError(err) - - re.NoError(cluster.RunInitialServers()) - - cluster.WaitLeader() - leader := cluster.GetServer(cluster.GetLeader()) - re.NotNil(leader) - - var wg sync.WaitGroup - wg.Add(2) - now := time.Now() - for i := 0; i < 2; i++ { - go func() { - defer wg.Done() - for i := 0; i <= 100; i++ { - physical := now.Add(time.Duration(2*i)*time.Minute).UnixNano() / int64(time.Millisecond) - ts := uint64(physical << 18) - leader.GetServer().GetHandler().ResetTS(ts, false, false) - } - }() - } - wg.Wait() -} - -func TestZeroTSOCount(t *testing.T) { - re := require.New(t) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - cluster, err := tests.NewTestCluster(ctx, 1) - defer cluster.Destroy() - re.NoError(err) - re.NoError(cluster.RunInitialServers()) - cluster.WaitLeader() - - leaderServer := cluster.GetServer(cluster.GetLeader()) - grpcPDClient := testutil.MustNewGrpcClient(re, leaderServer.GetAddr()) - clusterID := leaderServer.GetClusterID() - - req := &pdpb.TsoRequest{ - Header: testutil.NewRequestHeader(clusterID), - DcLocation: tso.GlobalDCLocation, - } - tsoClient, err := grpcPDClient.Tso(ctx) - re.NoError(err) - defer tsoClient.CloseSend() - re.NoError(tsoClient.Send(req)) - _, err = tsoClient.Recv() - re.Error(err) -} - func TestRequestFollower(t *testing.T) { re := require.New(t) ctx, cancel := context.WithCancel(context.Background()) @@ -211,7 +154,7 @@ func TestLogicalOverflow(t *testing.T) { defer tsoClient.CloseSend() begin := time.Now() - for i := 0; i < 2; i += 1 { // the 2nd request may (but not must) overflow, as max logical interval is 262144 + for i := 0; i < 3; i++ { req := &pdpb.TsoRequest{ Header: testutil.NewRequestHeader(clusterID), Count: 150000, @@ -220,12 +163,13 @@ func TestLogicalOverflow(t *testing.T) { re.NoError(tsoClient.Send(req)) _, err = tsoClient.Recv() re.NoError(err) + if i == 1 { + // the 2nd request may (but not must) overflow, as max logical interval is 262144 + re.Less(time.Since(begin), updateInterval+20*time.Millisecond) // additional 20ms for gRPC latency + } } - elapse := time.Since(begin) - if updateInterval >= 20*time.Millisecond { // on small interval, the physical may update before overflow - re.GreaterOrEqual(elapse, updateInterval) - } - re.Less(elapse, updateInterval+20*time.Millisecond) // additional 20ms for gRPC latency + // the 3rd request must overflow + re.GreaterOrEqual(time.Since(begin), updateInterval) } for _, updateInterval := range []int{1, 5, 30, 50} { diff --git a/tests/server/tso/tso_test.go b/tests/server/tso/tso_test.go index 7b318ad7d36..48df02a6c27 100644 --- a/tests/server/tso/tso_test.go +++ b/tests/server/tso/tso_test.go @@ -70,7 +70,7 @@ func TestLoadTimestamp(t *testing.T) { re.Greater(newTS.GetPhysical()-lastTS.GetPhysical(), int64(0)) } - failpoint.Disable("github.com/tikv/pd/pkg/tso/systemTimeSlow") + re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/tso/systemTimeSlow")) } func requestLocalTSOs(re *require.Assertions, cluster *tests.TestCluster, dcLocationConfig map[string]string) map[string]*pdpb.Timestamp { diff --git a/tests/testutil.go b/tests/testutil.go new file mode 100644 index 00000000000..25ff86c274f --- /dev/null +++ b/tests/testutil.go @@ -0,0 +1,117 @@ +// Copyright 2023 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tests + +import ( + "context" + "os" + "sync" + "time" + + "github.com/pingcap/log" + "github.com/stretchr/testify/require" + bs "github.com/tikv/pd/pkg/basicserver" + rm "github.com/tikv/pd/pkg/mcs/resourcemanager/server" + tso "github.com/tikv/pd/pkg/mcs/tso/server" + "github.com/tikv/pd/pkg/utils/logutil" + "github.com/tikv/pd/pkg/utils/testutil" +) + +var once sync.Once + +// InitLogger initializes the logger for test. +func InitLogger(cfg *tso.Config) (err error) { + once.Do(func() { + // Setup the logger. + err = logutil.SetupLogger(cfg.Log, &cfg.Logger, &cfg.LogProps, cfg.Security.RedactInfoLog) + if err != nil { + return + } + log.ReplaceGlobals(cfg.Logger, cfg.LogProps) + // Flushing any buffered log entries. + log.Sync() + }) + return err +} + +// StartSingleResourceManagerTestServer creates and starts a resource manager server with default config for testing. +func StartSingleResourceManagerTestServer(ctx context.Context, re *require.Assertions, backendEndpoints, listenAddrs string) (*rm.Server, func()) { + cfg := rm.NewConfig() + cfg.BackendEndpoints = backendEndpoints + cfg.ListenAddr = listenAddrs + cfg, err := rm.GenerateConfig(cfg) + re.NoError(err) + + s, cleanup, err := rm.NewTestServer(ctx, re, cfg) + re.NoError(err) + testutil.Eventually(re, func() bool { + return !s.IsClosed() + }, testutil.WithWaitFor(5*time.Second), testutil.WithTickInterval(50*time.Millisecond)) + + return s, cleanup +} + +// StartSingleTSOTestServerWithoutCheck creates and starts a tso server with default config for testing. +func StartSingleTSOTestServerWithoutCheck(ctx context.Context, re *require.Assertions, backendEndpoints, listenAddrs string) (*tso.Server, func(), error) { + cfg := tso.NewConfig() + cfg.BackendEndpoints = backendEndpoints + cfg.ListenAddr = listenAddrs + cfg, err := tso.GenerateConfig(cfg) + re.NoError(err) + // Setup the logger. + err = InitLogger(cfg) + re.NoError(err) + return NewTSOTestServer(ctx, cfg) +} + +// StartSingleTSOTestServer creates and starts a tso server with default config for testing. +func StartSingleTSOTestServer(ctx context.Context, re *require.Assertions, backendEndpoints, listenAddrs string) (*tso.Server, func()) { + s, cleanup, err := StartSingleTSOTestServerWithoutCheck(ctx, re, backendEndpoints, listenAddrs) + re.NoError(err) + testutil.Eventually(re, func() bool { + return !s.IsClosed() + }, testutil.WithWaitFor(5*time.Second), testutil.WithTickInterval(50*time.Millisecond)) + + return s, cleanup +} + +// NewTSOTestServer creates a tso server with given config for testing. +func NewTSOTestServer(ctx context.Context, cfg *tso.Config) (*tso.Server, testutil.CleanupFunc, error) { + s := tso.CreateServer(ctx, cfg) + if err := s.Run(); err != nil { + return nil, nil, err + } + cleanup := func() { + s.Close() + os.RemoveAll(cfg.DataDir) + } + return s, cleanup, nil +} + +// WaitForPrimaryServing waits for one of servers being elected to be the primary/leader +func WaitForPrimaryServing(re *require.Assertions, serverMap map[string]bs.Server) string { + var primary string + testutil.Eventually(re, func() bool { + for name, s := range serverMap { + if s.IsServing() { + primary = name + return true + } + } + return false + }, testutil.WithWaitFor(5*time.Second), testutil.WithTickInterval(50*time.Millisecond)) + + return primary +} diff --git a/tests/tso_cluster.go b/tests/tso_cluster.go new file mode 100644 index 00000000000..dc23c24d779 --- /dev/null +++ b/tests/tso_cluster.go @@ -0,0 +1,219 @@ +// Copyright 2023 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tests + +import ( + "context" + "fmt" + "sync" + "time" + + "github.com/stretchr/testify/require" + tso "github.com/tikv/pd/pkg/mcs/tso/server" + mcsutils "github.com/tikv/pd/pkg/mcs/utils" + "github.com/tikv/pd/pkg/storage/endpoint" + "github.com/tikv/pd/pkg/utils/tempurl" + "github.com/tikv/pd/pkg/utils/testutil" +) + +// TestTSOCluster is a test cluster for TSO. +type TestTSOCluster struct { + ctx context.Context + + backendEndpoints string + servers map[string]*tso.Server + cleanupFuncs map[string]testutil.CleanupFunc +} + +// NewTestTSOCluster creates a new TSO test cluster. +func NewTestTSOCluster(ctx context.Context, initialServerCount int, backendEndpoints string) (tc *TestTSOCluster, err error) { + tc = &TestTSOCluster{ + ctx: ctx, + backendEndpoints: backendEndpoints, + servers: make(map[string]*tso.Server, initialServerCount), + cleanupFuncs: make(map[string]testutil.CleanupFunc, initialServerCount), + } + for i := 0; i < initialServerCount; i++ { + err = tc.AddServer(tempurl.Alloc()) + if err != nil { + return nil, err + } + } + return tc, nil +} + +// RestartTestTSOCluster restarts the TSO test cluster. +func RestartTestTSOCluster( + ctx context.Context, cluster *TestTSOCluster, +) (newCluster *TestTSOCluster, err error) { + newCluster = &TestTSOCluster{ + ctx: ctx, + backendEndpoints: cluster.backendEndpoints, + servers: make(map[string]*tso.Server, len(cluster.servers)), + cleanupFuncs: make(map[string]testutil.CleanupFunc, len(cluster.servers)), + } + var ( + serverMap sync.Map + cleanupMap sync.Map + errorMap sync.Map + ) + wg := sync.WaitGroup{} + for addr, cleanup := range cluster.cleanupFuncs { + wg.Add(1) + go func(addr string, clean testutil.CleanupFunc) { + defer wg.Done() + clean() + serverCfg := cluster.servers[addr].GetConfig() + newServer, newCleanup, err := NewTSOTestServer(newCluster.ctx, serverCfg) + serverMap.Store(addr, newServer) + cleanupMap.Store(addr, newCleanup) + errorMap.Store(addr, err) + }(addr, cleanup) + } + wg.Wait() + + errorMap.Range(func(key, value interface{}) bool { + if value != nil { + err = value.(error) + return false + } + addr := key.(string) + newServer, _ := serverMap.Load(addr) + newCleanup, _ := cleanupMap.Load(addr) + newCluster.servers[addr] = newServer.(*tso.Server) + newCluster.cleanupFuncs[addr] = newCleanup.(testutil.CleanupFunc) + return true + }) + + if err != nil { + return nil, fmt.Errorf("failed to restart the cluster." + err.Error()) + } + + return newCluster, nil +} + +// AddServer adds a new TSO server to the test cluster. +func (tc *TestTSOCluster) AddServer(addr string) error { + cfg := tso.NewConfig() + cfg.BackendEndpoints = tc.backendEndpoints + cfg.ListenAddr = addr + cfg.Name = cfg.ListenAddr + generatedCfg, err := tso.GenerateConfig(cfg) + if err != nil { + return err + } + err = InitLogger(generatedCfg) + if err != nil { + return err + } + server, cleanup, err := NewTSOTestServer(tc.ctx, generatedCfg) + if err != nil { + return err + } + tc.servers[generatedCfg.GetListenAddr()] = server + tc.cleanupFuncs[generatedCfg.GetListenAddr()] = cleanup + return nil +} + +// Destroy stops and destroy the test cluster. +func (tc *TestTSOCluster) Destroy() { + for _, cleanup := range tc.cleanupFuncs { + cleanup() + } + tc.cleanupFuncs = nil + tc.servers = nil +} + +// DestroyServer stops and destroy the test server by the given address. +func (tc *TestTSOCluster) DestroyServer(addr string) { + tc.cleanupFuncs[addr]() + delete(tc.cleanupFuncs, addr) + delete(tc.servers, addr) +} + +// ResignPrimary resigns the primary TSO server. +func (tc *TestTSOCluster) ResignPrimary(keyspaceID, keyspaceGroupID uint32) error { + primaryServer := tc.GetPrimaryServer(keyspaceID, keyspaceGroupID) + if primaryServer == nil { + return fmt.Errorf("no tso server serves this keyspace %d", keyspaceID) + } + return primaryServer.ResignPrimary(keyspaceID, keyspaceGroupID) +} + +// GetPrimaryServer returns the primary TSO server of the given keyspace +func (tc *TestTSOCluster) GetPrimaryServer(keyspaceID, keyspaceGroupID uint32) *tso.Server { + for _, server := range tc.servers { + if server.IsKeyspaceServing(keyspaceID, keyspaceGroupID) { + return server + } + } + return nil +} + +// WaitForPrimaryServing waits for one of servers being elected to be the primary/leader of the given keyspace. +func (tc *TestTSOCluster) WaitForPrimaryServing(re *require.Assertions, keyspaceID, keyspaceGroupID uint32) *tso.Server { + var primary *tso.Server + testutil.Eventually(re, func() bool { + for _, server := range tc.servers { + if server.IsKeyspaceServing(keyspaceID, keyspaceGroupID) { + primary = server + return true + } + } + return false + }, testutil.WithWaitFor(5*time.Second), testutil.WithTickInterval(50*time.Millisecond)) + + return primary +} + +// WaitForDefaultPrimaryServing waits for one of servers being elected to be the primary/leader of the default keyspace. +func (tc *TestTSOCluster) WaitForDefaultPrimaryServing(re *require.Assertions) *tso.Server { + return tc.WaitForPrimaryServing(re, mcsutils.DefaultKeyspaceID, mcsutils.DefaultKeyspaceGroupID) +} + +// GetServer returns the TSO server by the given address. +func (tc *TestTSOCluster) GetServer(addr string) *tso.Server { + for srvAddr, server := range tc.servers { + if srvAddr == addr { + return server + } + } + return nil +} + +// GetServers returns all TSO servers. +func (tc *TestTSOCluster) GetServers() map[string]*tso.Server { + return tc.servers +} + +// GetKeyspaceGroupMember converts the TSO servers to KeyspaceGroupMember and returns. +func (tc *TestTSOCluster) GetKeyspaceGroupMember() (members []endpoint.KeyspaceGroupMember) { + for _, server := range tc.servers { + members = append(members, endpoint.KeyspaceGroupMember{ + Address: server.GetAddr(), + Priority: mcsutils.DefaultKeyspaceGroupReplicaPriority, + }) + } + return +} + +// GetAddrs returns all TSO server addresses. +func (tc *TestTSOCluster) GetAddrs() []string { + addrs := make([]string, 0, len(tc.servers)) + for _, server := range tc.servers { + addrs = append(addrs, server.GetAddr()) + } + return addrs +} diff --git a/tools/pd-backup/pdbackup/backup.go b/tools/pd-backup/pdbackup/backup.go index a645138b68b..f0b17e48267 100644 --- a/tools/pd-backup/pdbackup/backup.go +++ b/tools/pd-backup/pdbackup/backup.go @@ -25,6 +25,7 @@ import ( "path" "strconv" + "github.com/tikv/pd/pkg/storage/endpoint" "github.com/tikv/pd/pkg/utils/etcdutil" "github.com/tikv/pd/pkg/utils/typeutil" "github.com/tikv/pd/server/config" @@ -74,8 +75,7 @@ func GetBackupInfo(client *clientv3.Client, pdAddr string) (*BackupInfo, error) backInfo.AllocIDMax = allocIDMax - timestampPath := path.Join(rootPath, "timestamp") - resp, err = etcdutil.EtcdKVGet(client, timestampPath) + resp, err = etcdutil.EtcdKVGet(client, endpoint.TimestampPath(rootPath)) if err != nil { return nil, err } diff --git a/tools/pd-backup/pdbackup/backup_test.go b/tools/pd-backup/pdbackup/backup_test.go index c747ace21de..d93fd77a336 100644 --- a/tools/pd-backup/pdbackup/backup_test.go +++ b/tools/pd-backup/pdbackup/backup_test.go @@ -16,6 +16,8 @@ import ( "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" + sc "github.com/tikv/pd/pkg/schedule/config" + "github.com/tikv/pd/pkg/storage/endpoint" "github.com/tikv/pd/pkg/utils/etcdutil" "github.com/tikv/pd/pkg/utils/testutil" "github.com/tikv/pd/pkg/utils/typeutil" @@ -94,7 +96,7 @@ func setupServer() (*httptest.Server, *config.Config) { InitialClusterState: "new", InitialClusterToken: "test-token", LeaderLease: int64(1), - Replication: config.ReplicationConfig{ + Replication: sc.ReplicationConfig{ LocationLabels: typeutil.StringSlice{}, }, PDServerCfg: config.PDServerConfig{ @@ -133,10 +135,9 @@ func (s *backupTestSuite) BeforeTest(suiteName, testName string) { var ( rootPath = path.Join(pdRootPath, strconv.FormatUint(clusterID, 10)) - timestampPath = path.Join(rootPath, "timestamp") allocTimestampMaxBytes = typeutil.Uint64ToBytes(allocTimestampMax) ) - _, err = s.etcdClient.Put(ctx, timestampPath, string(allocTimestampMaxBytes)) + _, err = s.etcdClient.Put(ctx, endpoint.TimestampPath(rootPath), string(allocTimestampMaxBytes)) s.NoError(err) var ( diff --git a/tools/pd-ctl/pdctl/command/cluster_command.go b/tools/pd-ctl/pdctl/command/cluster_command.go index 60d2b786e3f..43722d4e58d 100644 --- a/tools/pd-ctl/pdctl/command/cluster_command.go +++ b/tools/pd-ctl/pdctl/command/cluster_command.go @@ -20,8 +20,10 @@ import ( "github.com/spf13/cobra" ) -const clusterPrefix = "pd/api/v1/cluster" -const clusterStatusPrefix = "pd/api/v1/cluster/status" +const ( + clusterPrefix = "pd/api/v1/cluster" + clusterStatusPrefix = "pd/api/v1/cluster/status" +) // NewClusterCommand return a cluster subcommand of rootCmd func NewClusterCommand() *cobra.Command { diff --git a/tools/pd-ctl/pdctl/command/config_command.go b/tools/pd-ctl/pdctl/command/config_command.go index fdfa95f0e68..2d0c3844995 100644 --- a/tools/pd-ctl/pdctl/command/config_command.go +++ b/tools/pd-ctl/pdctl/command/config_command.go @@ -468,9 +468,10 @@ func NewPlacementRulesCommand() *cobra.Command { } ruleGroupDelete := &cobra.Command{ Use: "delete ", - Short: "delete rule group configuration", - Run: deleteRuleGroupFunc, + Short: "delete rule group configuration. Note: this command will be deprecated soon, use instead", + Run: delRuleBundle, } + ruleGroupDelete.Flags().Bool("regexp", false, "match group id by regular expression") ruleGroup.AddCommand(ruleGroupShow, ruleGroupSet, ruleGroupDelete) ruleBundle := &cobra.Command{ Use: "rule-bundle", @@ -663,19 +664,6 @@ func updateRuleGroupFunc(cmd *cobra.Command, args []string) { }) } -func deleteRuleGroupFunc(cmd *cobra.Command, args []string) { - if len(args) != 1 { - cmd.Println(cmd.UsageString()) - return - } - _, err := doRequest(cmd, path.Join(ruleGroupPrefix, args[0]), http.MethodDelete, http.Header{}) - if err != nil { - cmd.Printf("Failed to remove rule group config: %s \n", err) - return - } - cmd.Println("Success!") -} - func getRuleBundle(cmd *cobra.Command, args []string) { if len(args) != 1 { cmd.Println(cmd.UsageString()) diff --git a/tools/pd-ctl/pdctl/command/global.go b/tools/pd-ctl/pdctl/command/global.go index 623ab3edfba..85fe63ac8be 100644 --- a/tools/pd-ctl/pdctl/command/global.go +++ b/tools/pd-ctl/pdctl/command/global.go @@ -165,7 +165,7 @@ func getEndpoints(cmd *cobra.Command) []string { return strings.Split(addrs, ",") } -func postJSON(cmd *cobra.Command, prefix string, input map[string]interface{}) { +func requestJSON(cmd *cobra.Command, method, prefix string, input map[string]interface{}) { data, err := json.Marshal(input) if err != nil { cmd.Println(err) @@ -175,29 +175,49 @@ func postJSON(cmd *cobra.Command, prefix string, input map[string]interface{}) { endpoints := getEndpoints(cmd) err = tryURLs(cmd, endpoints, func(endpoint string) error { var msg []byte - var r *http.Response + var req *http.Request + var resp *http.Response url := endpoint + "/" + prefix - r, err = dialClient.Post(url, "application/json", bytes.NewBuffer(data)) + switch method { + case http.MethodPost, http.MethodPut, http.MethodPatch, http.MethodDelete, http.MethodGet: + req, err = http.NewRequest(method, url, bytes.NewBuffer(data)) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/json") + resp, err = dialClient.Do(req) + default: + err := errors.Errorf("method %s not supported", method) + return err + } if err != nil { return err } - defer r.Body.Close() - if r.StatusCode != http.StatusOK { - msg, err = io.ReadAll(r.Body) + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + msg, err = io.ReadAll(resp.Body) if err != nil { return err } - return errors.Errorf("[%d] %s", r.StatusCode, msg) + return errors.Errorf("[%d] %s", resp.StatusCode, msg) } return nil }) if err != nil { - cmd.Printf("Failed! %s", err) + cmd.Printf("Failed! %s\n", err) return } cmd.Println("Success!") } +func postJSON(cmd *cobra.Command, prefix string, input map[string]interface{}) { + requestJSON(cmd, http.MethodPost, prefix, input) +} + +func patchJSON(cmd *cobra.Command, prefix string, input map[string]interface{}) { + requestJSON(cmd, http.MethodPatch, prefix, input) +} + // do send a request to server. Default is Get. func do(endpoint, prefix, method string, resp *string, customHeader http.Header, b *bodyOption) error { var err error diff --git a/tools/pd-ctl/pdctl/command/hot_command.go b/tools/pd-ctl/pdctl/command/hot_command.go index c78b4a38d8a..09160d8f2b9 100644 --- a/tools/pd-ctl/pdctl/command/hot_command.go +++ b/tools/pd-ctl/pdctl/command/hot_command.go @@ -32,6 +32,7 @@ const ( hotWriteRegionsPrefix = "pd/api/v1/hotspot/regions/write" hotStoresPrefix = "pd/api/v1/hotspot/stores" hotRegionsHistoryPrefix = "pd/api/v1/hotspot/regions/history" + hotBucketsPrefix = "pd/api/v1/hotspot/buckets" ) // NewHotSpotCommand return a hot subcommand of rootCmd @@ -44,6 +45,7 @@ func NewHotSpotCommand() *cobra.Command { cmd.AddCommand(NewHotReadRegionCommand()) cmd.AddCommand(NewHotStoreCommand()) cmd.AddCommand(NewHotRegionsHistoryCommand()) + cmd.AddCommand(NewHotBucketsCommand()) return cmd } @@ -58,7 +60,7 @@ func NewHotWriteRegionCommand() *cobra.Command { } func showHotWriteRegionsCommandFunc(cmd *cobra.Command, args []string) { - prefix, err := parseOptionalArgs(hotWriteRegionsPrefix, args) + prefix, err := parseOptionalArgs(hotWriteRegionsPrefix, "store_id", args) if err != nil { cmd.Println(err) return @@ -82,7 +84,7 @@ func NewHotReadRegionCommand() *cobra.Command { } func showHotReadRegionsCommandFunc(cmd *cobra.Command, args []string) { - prefix, err := parseOptionalArgs(hotReadRegionsPrefix, args) + prefix, err := parseOptionalArgs(hotReadRegionsPrefix, "store_id", args) if err != nil { cmd.Println(err) return @@ -126,6 +128,31 @@ func NewHotRegionsHistoryCommand() *cobra.Command { return cmd } +// NewHotBucketsCommand return a hot buckets subcommand of hotSpotCmd +func NewHotBucketsCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "buckets [region_id]", + Short: "show the hot buckets", + Run: showHotBucketsCommandFunc, + } + return cmd +} + +func showHotBucketsCommandFunc(cmd *cobra.Command, args []string) { + prefix, err := parseOptionalArgs(hotBucketsPrefix, "region_id", args) + if err != nil { + cmd.Printf("Failed to get hotspot buckets: %s\n", err) + return + } + + r, err := doRequest(cmd, prefix, http.MethodGet, http.Header{}) + if err != nil { + cmd.Printf("Failed to get hotspot buckets: %s\n", err) + return + } + cmd.Println(r) +} + func showHotRegionsHistoryCommandFunc(cmd *cobra.Command, args []string) { if len(args) < 2 || len(args)%2 != 0 { cmd.Println(cmd.UsageString()) @@ -171,19 +198,19 @@ func showHotRegionsHistoryCommandFunc(cmd *cobra.Command, args []string) { cmd.Println(string(resp)) } -func parseOptionalArgs(prefix string, args []string) (string, error) { +func parseOptionalArgs(prefix string, param string, args []string) (string, error) { argsLen := len(args) if argsLen > 0 { prefix += "?" } for i, arg := range args { if _, err := strconv.Atoi(arg); err != nil { - return "", errors.Errorf("store id should be a number, but got %s", arg) + return "", errors.Errorf("args should be a number, but got %s", arg) } if i != argsLen { - prefix = prefix + "store_id=" + arg + "&" + prefix = prefix + param + "=" + arg + "&" } else { - prefix = prefix + "store_id=" + arg + prefix = prefix + param + "=" + arg } } return prefix, nil diff --git a/tools/pd-ctl/pdctl/command/keyspace_command.go b/tools/pd-ctl/pdctl/command/keyspace_command.go new file mode 100644 index 00000000000..a68e2f05a80 --- /dev/null +++ b/tools/pd-ctl/pdctl/command/keyspace_command.go @@ -0,0 +1,48 @@ +// Copyright 2023 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package command + +import ( + "fmt" + "net/http" + + "github.com/spf13/cobra" +) + +const keyspacePrefix = "pd/api/v2/keyspaces" + +// NewKeyspaceCommand returns a keyspace subcommand of rootCmd. +func NewKeyspaceCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "keyspace [command] [flags]", + Short: "show keyspace information", + Run: showKeyspaceCommandFunc, + } + return cmd +} + +func showKeyspaceCommandFunc(cmd *cobra.Command, args []string) { + if len(args) != 1 { + cmd.Usage() + return + } + + resp, err := doRequest(cmd, fmt.Sprintf("%s/%s?force_refresh_group_id=true", keyspacePrefix, args[0]), http.MethodGet, http.Header{}) + if err != nil { + cmd.Printf("Failed to get the keyspace information: %s\n", err) + return + } + cmd.Println(resp) +} diff --git a/tools/pd-ctl/pdctl/command/keyspace_group_command.go b/tools/pd-ctl/pdctl/command/keyspace_group_command.go new file mode 100644 index 00000000000..6dea6d78dd6 --- /dev/null +++ b/tools/pd-ctl/pdctl/command/keyspace_group_command.go @@ -0,0 +1,419 @@ +// Copyright 2023 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package command + +import ( + "encoding/json" + "fmt" + "net/http" + "net/url" + "strconv" + "strings" + + "github.com/spf13/cobra" + mcsutils "github.com/tikv/pd/pkg/mcs/utils" + "github.com/tikv/pd/pkg/storage/endpoint" +) + +const keyspaceGroupsPrefix = "pd/api/v2/tso/keyspace-groups" + +// NewKeyspaceGroupCommand return a keyspace group subcommand of rootCmd +func NewKeyspaceGroupCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "keyspace-group [command] [flags]", + Short: "show keyspace group information", + Run: showKeyspaceGroupsCommandFunc, + } + cmd.AddCommand(newSplitKeyspaceGroupCommand()) + cmd.AddCommand(newSplitRangeKeyspaceGroupCommand()) + cmd.AddCommand(newFinishSplitKeyspaceGroupCommand()) + cmd.AddCommand(newMergeKeyspaceGroupCommand()) + cmd.AddCommand(newFinishMergeKeyspaceGroupCommand()) + cmd.AddCommand(newSetNodesKeyspaceGroupCommand()) + cmd.AddCommand(newSetPriorityKeyspaceGroupCommand()) + cmd.AddCommand(newShowKeyspaceGroupPrimaryCommand()) + cmd.Flags().String("state", "", "state filter") + return cmd +} + +func newSplitKeyspaceGroupCommand() *cobra.Command { + r := &cobra.Command{ + Use: "split []", + Short: "split the keyspace group with the given ID and transfer the keyspaces into the newly split one", + Run: splitKeyspaceGroupCommandFunc, + } + return r +} + +func newSplitRangeKeyspaceGroupCommand() *cobra.Command { + r := &cobra.Command{ + Use: "split-range ", + Short: "split the keyspace group with the given ID and transfer the keyspaces in the given range (both ends inclusive) into the newly split one", + Run: splitRangeKeyspaceGroupCommandFunc, + } + return r +} + +func newFinishSplitKeyspaceGroupCommand() *cobra.Command { + r := &cobra.Command{ + Use: "finish-split ", + Short: "finish split the keyspace group with the given ID", + Run: finishSplitKeyspaceGroupCommandFunc, + Hidden: true, + } + return r +} + +func newMergeKeyspaceGroupCommand() *cobra.Command { + r := &cobra.Command{ + Use: "merge []", + Short: "merge the keyspace group with the given IDs into the target one", + Run: mergeKeyspaceGroupCommandFunc, + } + r.Flags().Bool("all", false, "merge all keyspace groups into the default one") + return r +} + +func newFinishMergeKeyspaceGroupCommand() *cobra.Command { + r := &cobra.Command{ + Use: "finish-merge ", + Short: "finish merge the keyspace group with the given ID", + Run: finishMergeKeyspaceGroupCommandFunc, + Hidden: true, + } + return r +} + +func newSetNodesKeyspaceGroupCommand() *cobra.Command { + r := &cobra.Command{ + Use: "set-node [...]", + Short: "set the address of tso nodes for keyspace group with the given ID", + Run: setNodesKeyspaceGroupCommandFunc, + } + return r +} + +func newSetPriorityKeyspaceGroupCommand() *cobra.Command { + r := &cobra.Command{ + Use: "set-priority ", + Short: "set the priority of tso nodes for keyspace group with the given ID. If the priority is negative, it need to add a prefix with -- to avoid identified as flag.", + Run: setPriorityKeyspaceGroupCommandFunc, + } + return r +} + +func newShowKeyspaceGroupPrimaryCommand() *cobra.Command { + r := &cobra.Command{ + Use: "primary ", + Short: "show th primary of tso nodes for keyspace group with the given ID.", + Run: showKeyspaceGroupPrimaryCommandFunc, + } + return r +} + +func showKeyspaceGroupsCommandFunc(cmd *cobra.Command, args []string) { + prefix := keyspaceGroupsPrefix + if len(args) > 1 { + cmd.Usage() + return + } + cFunc := convertToKeyspaceGroups + if len(args) == 1 { + if _, err := strconv.Atoi(args[0]); err != nil { + cmd.Println("keyspace_group_id should be a number") + return + } + prefix = fmt.Sprintf("%s/%s", keyspaceGroupsPrefix, args[0]) + cFunc = convertToKeyspaceGroup + } else { + flags := cmd.Flags() + state, err := flags.GetString("state") + if err != nil { + cmd.Printf("Failed to get state: %s\n", err) + } + stateValue := "" + if len(state) != 0 { + state = strings.ToLower(state) + switch state { + case "merge", "split": + stateValue = fmt.Sprintf("state=%v", state) + default: + cmd.Println("Unknown state: " + state) + return + } + } + + if len(stateValue) != 0 { + prefix = fmt.Sprintf("%v?%v", keyspaceGroupsPrefix, stateValue) + } + } + r, err := doRequest(cmd, prefix, http.MethodGet, http.Header{}) + if err != nil { + cmd.Printf("Failed to get the keyspace groups information: %s\n", err) + return + } + r = cFunc(r) + cmd.Println(r) +} + +func splitKeyspaceGroupCommandFunc(cmd *cobra.Command, args []string) { + if len(args) < 3 { + cmd.Usage() + return + } + _, err := strconv.ParseUint(args[0], 10, 32) + if err != nil { + cmd.Printf("Failed to parse the old keyspace group ID: %s\n", err) + return + } + newID, err := strconv.ParseUint(args[1], 10, 32) + if err != nil { + cmd.Printf("Failed to parse the new keyspace group ID: %s\n", err) + return + } + keyspaces := make([]uint32, 0, len(args)-2) + for _, arg := range args[2:] { + id, err := strconv.ParseUint(arg, 10, 32) + if err != nil { + cmd.Printf("Failed to parse the keyspace ID: %s\n", err) + return + } + keyspaces = append(keyspaces, uint32(id)) + } + postJSON(cmd, fmt.Sprintf("%s/%s/split", keyspaceGroupsPrefix, args[0]), map[string]interface{}{ + "new-id": uint32(newID), + "keyspaces": keyspaces, + }) +} + +func splitRangeKeyspaceGroupCommandFunc(cmd *cobra.Command, args []string) { + if len(args) < 4 { + cmd.Usage() + return + } + _, err := strconv.ParseUint(args[0], 10, 32) + if err != nil { + cmd.Printf("Failed to parse the old keyspace group ID: %s\n", err) + return + } + newID, err := strconv.ParseUint(args[1], 10, 32) + if err != nil { + cmd.Printf("Failed to parse the new keyspace group ID: %s\n", err) + return + } + startKeyspaceID, err := strconv.ParseUint(args[2], 10, 32) + if err != nil { + cmd.Printf("Failed to parse the start keyspace ID: %s\n", err) + return + } + endKeyspaceID, err := strconv.ParseUint(args[3], 10, 32) + if err != nil { + cmd.Printf("Failed to parse the end keyspace ID: %s\n", err) + return + } + postJSON(cmd, fmt.Sprintf("%s/%s/split", keyspaceGroupsPrefix, args[0]), map[string]interface{}{ + "new-id": uint32(newID), + "start-keyspace-id": uint32(startKeyspaceID), + "end-keyspace-id": uint32(endKeyspaceID), + }) +} + +func finishSplitKeyspaceGroupCommandFunc(cmd *cobra.Command, args []string) { + if len(args) < 1 { + cmd.Usage() + return + } + _, err := strconv.ParseUint(args[0], 10, 32) + if err != nil { + cmd.Printf("Failed to parse the keyspace group ID: %s\n", err) + return + } + _, err = doRequest(cmd, fmt.Sprintf("%s/%s/split", keyspaceGroupsPrefix, args[0]), http.MethodDelete, http.Header{}) + if err != nil { + cmd.Println(err) + return + } + cmd.Println("Success!") +} + +func mergeKeyspaceGroupCommandFunc(cmd *cobra.Command, args []string) { + var ( + targetGroupID uint32 + params = map[string]interface{}{} + argNum = len(args) + ) + mergeAll, err := cmd.Flags().GetBool("all") + if err != nil { + cmd.Printf("Failed to get the merge all flag: %s\n", err) + return + } + if argNum == 1 && mergeAll { + target, err := strconv.ParseUint(args[0], 10, 32) + if err != nil { + cmd.Printf("Failed to parse the target keyspace group ID: %s\n", err) + return + } + targetGroupID = uint32(target) + if targetGroupID != mcsutils.DefaultKeyspaceGroupID { + cmd.Println("Unable to merge all keyspace groups into a non-default keyspace group") + return + } + params["merge-all-into-default"] = true + } else if argNum >= 2 && !mergeAll { + target, err := strconv.ParseUint(args[0], 10, 32) + if err != nil { + cmd.Printf("Failed to parse the target keyspace group ID: %s\n", err) + return + } + targetGroupID = uint32(target) + groups := make([]uint32, 0, len(args)-1) + for _, arg := range args[1:] { + id, err := strconv.ParseUint(arg, 10, 32) + if err != nil { + cmd.Printf("Failed to parse the keyspace ID: %s\n", err) + return + } + groups = append(groups, uint32(id)) + } + params["merge-list"] = groups + } else { + cmd.Println("Must specify the source keyspace group ID(s) or the merge all flag") + cmd.Usage() + return + } + // TODO: implement the retry mechanism under merge all flag. + postJSON(cmd, fmt.Sprintf("%s/%d/merge", keyspaceGroupsPrefix, targetGroupID), params) +} + +func finishMergeKeyspaceGroupCommandFunc(cmd *cobra.Command, args []string) { + if len(args) < 1 { + cmd.Usage() + return + } + _, err := strconv.ParseUint(args[0], 10, 32) + if err != nil { + cmd.Printf("Failed to parse the keyspace group ID: %s\n", err) + return + } + _, err = doRequest(cmd, fmt.Sprintf("%s/%s/merge", keyspaceGroupsPrefix, args[0]), http.MethodDelete, http.Header{}) + if err != nil { + cmd.Println(err) + return + } + cmd.Println("Success!") +} + +func setNodesKeyspaceGroupCommandFunc(cmd *cobra.Command, args []string) { + if len(args) < 2 { + cmd.Usage() + return + } + _, err := strconv.ParseUint(args[0], 10, 32) + if err != nil { + cmd.Printf("Failed to parse the keyspace group ID: %s\n", err) + return + } + nodes := make([]string, 0, len(args)-1) + for _, arg := range args[1:] { + u, err := url.ParseRequestURI(arg) + if u == nil || err != nil { + cmd.Printf("Failed to parse the tso node address: %s\n", err) + return + } + nodes = append(nodes, arg) + } + patchJSON(cmd, fmt.Sprintf("%s/%s", keyspaceGroupsPrefix, args[0]), map[string]interface{}{ + "Nodes": nodes, + }) +} + +func setPriorityKeyspaceGroupCommandFunc(cmd *cobra.Command, args []string) { + if len(args) < 3 { + cmd.Usage() + return + } + _, err := strconv.ParseUint(args[0], 10, 32) + if err != nil { + cmd.Printf("Failed to parse the keyspace group ID: %s\n", err) + return + } + + node := args[1] + u, err := url.ParseRequestURI(node) + if u == nil || err != nil { + cmd.Printf("Failed to parse the tso node address: %s\n", err) + return + } + + // Escape the node address to avoid the error of parsing the url + // But the url.PathEscape will escape the '/' to '%2F', which % will cause the error of parsing the url + // So we need to replace the % to \% + node = url.PathEscape(node) + node = strings.ReplaceAll(node, "%", "\\%") + + priority, err := strconv.ParseInt(args[2], 10, 32) + if err != nil { + cmd.Printf("Failed to parse the priority: %s\n", err) + return + } + + patchJSON(cmd, fmt.Sprintf("%s/%s/%s", keyspaceGroupsPrefix, args[0], node), map[string]interface{}{ + "Priority": priority, + }) +} + +func showKeyspaceGroupPrimaryCommandFunc(cmd *cobra.Command, args []string) { + if len(args) < 1 { + cmd.Usage() + return + } + _, err := strconv.ParseUint(args[0], 10, 32) + if err != nil { + cmd.Printf("Failed to parse the keyspace group ID: %s\n", err) + return + } + r, err := doRequest(cmd, fmt.Sprintf("%s/%s?fields=primary", keyspaceGroupsPrefix, args[0]), http.MethodGet, http.Header{}) + if err != nil { + cmd.Printf("Failed to get the keyspace group primary information: %s\n", err) + return + } + cmd.Println(r) +} + +func convertToKeyspaceGroup(content string) string { + kg := endpoint.KeyspaceGroup{} + err := json.Unmarshal([]byte(content), &kg) + if err != nil { + return content + } + byteArr, err := json.MarshalIndent(kg, "", " ") + if err != nil { + return content + } + return string(byteArr) +} + +func convertToKeyspaceGroups(content string) string { + kgs := []*endpoint.KeyspaceGroup{} + err := json.Unmarshal([]byte(content), &kgs) + if err != nil { + return content + } + byteArr, err := json.MarshalIndent(kgs, "", " ") + if err != nil { + return content + } + return string(byteArr) +} diff --git a/tools/pd-ctl/pdctl/command/label_command.go b/tools/pd-ctl/pdctl/command/label_command.go index e67c4358e35..60762383241 100644 --- a/tools/pd-ctl/pdctl/command/label_command.go +++ b/tools/pd-ctl/pdctl/command/label_command.go @@ -21,9 +21,9 @@ import ( "github.com/spf13/cobra" "github.com/tikv/pd/pkg/core" + sc "github.com/tikv/pd/pkg/schedule/config" "github.com/tikv/pd/pkg/statistics" "github.com/tikv/pd/server/api" - "github.com/tikv/pd/server/config" ) var ( @@ -95,13 +95,13 @@ func NewCheckLabels() *cobra.Command { } } -func getReplicationConfig(cmd *cobra.Command, _ []string) (*config.ReplicationConfig, error) { +func getReplicationConfig(cmd *cobra.Command, _ []string) (*sc.ReplicationConfig, error) { prefix := configPrefix + "/replicate" body, err := doRequest(cmd, prefix, http.MethodGet, http.Header{}) if err != nil { return nil, err } - var config config.ReplicationConfig + var config sc.ReplicationConfig if err := json.Unmarshal([]byte(body), &config); err != nil { return nil, err } diff --git a/tools/pd-ctl/pdctl/command/operator.go b/tools/pd-ctl/pdctl/command/operator.go index f4652fc78f8..1db237b55d0 100644 --- a/tools/pd-ctl/pdctl/command/operator.go +++ b/tools/pd-ctl/pdctl/command/operator.go @@ -46,7 +46,7 @@ const ( ` ) -// NewOperatorCommand returns a operator command. +// NewOperatorCommand returns an operator command. func NewOperatorCommand() *cobra.Command { c := &cobra.Command{ Use: "operator", @@ -351,16 +351,17 @@ func removePeerCommandFunc(cmd *cobra.Command, args []string) { // NewSplitRegionCommand returns a command to split a region. func NewSplitRegionCommand() *cobra.Command { c := &cobra.Command{ - Use: "split-region [--policy=scan|approximate]", + Use: "split-region [--policy=scan|approximate|usekey] [--keys]", Short: "split a region", Run: splitRegionCommandFunc, } c.Flags().String("policy", "scan", "the policy to get region split key") + c.Flags().String("keys", "", "the split key, hex encoded") return c } func splitRegionCommandFunc(cmd *cobra.Command, args []string) { - if len(args) != 1 { + if len(args) < 1 { cmd.Println(cmd.UsageString()) return } @@ -373,7 +374,7 @@ func splitRegionCommandFunc(cmd *cobra.Command, args []string) { policy := cmd.Flags().Lookup("policy").Value.String() switch policy { - case "scan", "approximate": + case "scan", "approximate", "usekey": break default: cmd.Println("Error: unknown policy") @@ -384,6 +385,10 @@ func splitRegionCommandFunc(cmd *cobra.Command, args []string) { input["name"] = cmd.Name() input["region_id"] = ids[0] input["policy"] = policy + keys := cmd.Flags().Lookup("keys").Value.String() + if len(keys) > 0 { + input["keys"] = []string{keys} + } postJSON(cmd, operatorsPrefix, input) } diff --git a/tools/pd-ctl/pdctl/command/plugin_command.go b/tools/pd-ctl/pdctl/command/plugin_command.go index f6ae7a97159..a713c8fd063 100644 --- a/tools/pd-ctl/pdctl/command/plugin_command.go +++ b/tools/pd-ctl/pdctl/command/plugin_command.go @@ -20,7 +20,7 @@ import ( "net/http" "github.com/spf13/cobra" - "github.com/tikv/pd/server/cluster" + "github.com/tikv/pd/pkg/schedule" ) var ( @@ -59,11 +59,11 @@ func NewUnloadPluginCommand() *cobra.Command { } func loadPluginCommandFunc(cmd *cobra.Command, args []string) { - sendPluginCommand(cmd, cluster.PluginLoad, args) + sendPluginCommand(cmd, schedule.PluginLoad, args) } func unloadPluginCommandFunc(cmd *cobra.Command, args []string) { - sendPluginCommand(cmd, cluster.PluginUnload, args) + sendPluginCommand(cmd, schedule.PluginUnload, args) } func sendPluginCommand(cmd *cobra.Command, action string, args []string) { @@ -80,9 +80,9 @@ func sendPluginCommand(cmd *cobra.Command, action string, args []string) { return } switch action { - case cluster.PluginLoad: + case schedule.PluginLoad: _, err = doRequest(cmd, pluginPrefix, http.MethodPost, http.Header{"Content-Type": {"application/json"}}, WithBody(bytes.NewBuffer(reqData))) - case cluster.PluginUnload: + case schedule.PluginUnload: _, err = doRequest(cmd, pluginPrefix, http.MethodDelete, http.Header{"Content-Type": {"application/json"}}, WithBody(bytes.NewBuffer(reqData))) default: cmd.Printf("Unknown action %s\n", action) diff --git a/tools/pd-ctl/pdctl/command/region_command.go b/tools/pd-ctl/pdctl/command/region_command.go index fcebb30e6d8..33191bbe12b 100644 --- a/tools/pd-ctl/pdctl/command/region_command.go +++ b/tools/pd-ctl/pdctl/command/region_command.go @@ -45,6 +45,7 @@ var ( regionsKeyPrefix = "pd/api/v1/regions/key" regionsSiblingPrefix = "pd/api/v1/regions/sibling" regionsRangeHolesPrefix = "pd/api/v1/regions/range-holes" + regionsKeyspacePrefix = "pd/api/v1/regions/keyspace" regionIDPrefix = "pd/api/v1/region/id" regionKeyPrefix = "pd/api/v1/region/key" ) @@ -60,6 +61,7 @@ func NewRegionCommand() *cobra.Command { r.AddCommand(NewRegionWithCheckCommand()) r.AddCommand(NewRegionWithSiblingCommand()) r.AddCommand(NewRegionWithStoreCommand()) + r.AddCommand(NewRegionWithKeyspaceCommand()) r.AddCommand(NewRegionsByKeysCommand()) r.AddCommand(NewRangesWithRangeHolesCommand()) @@ -463,6 +465,43 @@ func showRegionWithStoreCommandFunc(cmd *cobra.Command, args []string) { cmd.Println(r) } +// NewRegionWithKeyspaceCommand returns regions with keyspace subcommand of regionCmd +func NewRegionWithKeyspaceCommand() *cobra.Command { + r := &cobra.Command{ + Use: "keyspace ", + Short: "show region information of the given keyspace", + } + r.AddCommand(&cobra.Command{ + Use: "id ", + Short: "show region information for the given keyspace id", + Run: showRegionWithKeyspaceCommandFunc, + }) + return r +} + +func showRegionWithKeyspaceCommandFunc(cmd *cobra.Command, args []string) { + if len(args) < 1 || len(args) > 2 { + cmd.Println(cmd.UsageString()) + return + } + + keyspaceID := args[0] + prefix := regionsKeyspacePrefix + "/id/" + keyspaceID + if len(args) == 2 { + if _, err := strconv.Atoi(args[1]); err != nil { + cmd.Println("limit should be a number") + return + } + prefix += "?limit=" + args[1] + } + r, err := doRequest(cmd, prefix, http.MethodGet, http.Header{}) + if err != nil { + cmd.Printf("Failed to get regions with the given keyspace: %s\n", err) + return + } + cmd.Println(r) +} + const ( rangeHolesLongDesc = `There are some cases that the region range is not continuous, for example, the region doesn't send the heartbeat to PD after a splitting. This command will output all empty ranges without any region info.` diff --git a/tools/pd-ctl/pdctl/command/store_command.go b/tools/pd-ctl/pdctl/command/store_command.go index 79ed9673e83..1dee1c13a72 100644 --- a/tools/pd-ctl/pdctl/command/store_command.go +++ b/tools/pd-ctl/pdctl/command/store_command.go @@ -565,7 +565,6 @@ func labelStoreCommandFunc(cmd *cobra.Command, args []string) { } else if rewrite, _ := cmd.Flags().GetBool("rewrite"); rewrite { prefix += "?force=true" } - cmd.Println(prefix) postJSON(cmd, prefix, labels) } diff --git a/tools/pd-ctl/pdctl/command/tso_command.go b/tools/pd-ctl/pdctl/command/tso_command.go index 2eedf9a2a3e..689420854ee 100644 --- a/tools/pd-ctl/pdctl/command/tso_command.go +++ b/tools/pd-ctl/pdctl/command/tso_command.go @@ -21,7 +21,7 @@ import ( "github.com/tikv/pd/pkg/utils/tsoutil" ) -// NewTSOCommand return a ping subcommand of rootCmd +// NewTSOCommand return a TSO subcommand of rootCmd func NewTSOCommand() *cobra.Command { cmd := &cobra.Command{ Use: "tso ", diff --git a/tools/pd-ctl/pdctl/ctl.go b/tools/pd-ctl/pdctl/ctl.go index 252c7471eaa..86494c046eb 100644 --- a/tools/pd-ctl/pdctl/ctl.go +++ b/tools/pd-ctl/pdctl/ctl.go @@ -65,6 +65,8 @@ func GetRootCmd() *cobra.Command { command.NewMinResolvedTSCommand(), command.NewCompletionCommand(), command.NewUnsafeCommand(), + command.NewKeyspaceGroupCommand(), + command.NewKeyspaceCommand(), ) rootCmd.Flags().ParseErrorsWhitelist.UnknownFlags = true diff --git a/tools/pd-simulator/main.go b/tools/pd-simulator/main.go index 1b40ceabae7..60d8874d083 100644 --- a/tools/pd-simulator/main.go +++ b/tools/pd-simulator/main.go @@ -31,6 +31,7 @@ import ( "github.com/tikv/pd/pkg/schedule/schedulers" "github.com/tikv/pd/pkg/statistics" "github.com/tikv/pd/pkg/utils/logutil" + "github.com/tikv/pd/pkg/utils/testutil" "github.com/tikv/pd/server" "github.com/tikv/pd/server/api" "github.com/tikv/pd/server/config" @@ -42,7 +43,7 @@ import ( ) var ( - pdAddr = flag.String("pd", "", "pd address") + pdAddr = flag.String("pd-endpoints", "", "pd address") configFile = flag.String("config", "conf/simconfig.toml", "config file") caseName = flag.String("case", "", "case name") serverLogLevel = flag.String("serverLog", "info", "pd server log level") @@ -132,7 +133,7 @@ func runHTTPServer() { } // NewSingleServer creates a pd server for simulator. -func NewSingleServer(ctx context.Context, simConfig *simulator.SimConfig) (*server.Server, server.CleanupFunc) { +func NewSingleServer(ctx context.Context, simConfig *simulator.SimConfig) (*server.Server, testutil.CleanupFunc) { err := logutil.SetupLogger(simConfig.ServerConfig.Log, &simConfig.ServerConfig.Logger, &simConfig.ServerConfig.LogProps) if err == nil { log.ReplaceGlobals(simConfig.ServerConfig.Logger, simConfig.ServerConfig.LogProps) @@ -157,7 +158,7 @@ func cleanServer(cfg *config.Config) { os.RemoveAll(cfg.DataDir) } -func simStart(pdAddr string, simCase string, simConfig *simulator.SimConfig, clean ...server.CleanupFunc) { +func simStart(pdAddr string, simCase string, simConfig *simulator.SimConfig, clean ...testutil.CleanupFunc) { start := time.Now() driver, err := simulator.NewDriver(pdAddr, simCase, simConfig) if err != nil { @@ -196,7 +197,7 @@ EXIT: } driver.Stop() - if len(clean) != 0 { + if len(clean) != 0 && clean[0] != nil { clean[0]() } diff --git a/tools/pd-simulator/simulator/cases/balance_region.go b/tools/pd-simulator/simulator/cases/balance_region.go index 39f3ef29379..0a013cf3876 100644 --- a/tools/pd-simulator/simulator/cases/balance_region.go +++ b/tools/pd-simulator/simulator/cases/balance_region.go @@ -17,7 +17,6 @@ package cases import ( "time" - "github.com/docker/go-units" "github.com/pingcap/kvproto/pkg/metapb" "github.com/tikv/pd/pkg/core" "github.com/tikv/pd/tools/pd-simulator/simulator/info" @@ -55,8 +54,6 @@ func newRedundantBalanceRegion() *Case { ID: IDAllocator.nextID(), Peers: peers, Leader: peers[0], - Size: 96 * units.MiB, - Keys: 960000, }) } diff --git a/tools/pd-simulator/simulator/cases/import_data.go b/tools/pd-simulator/simulator/cases/import_data.go index 546a175792b..0e7f7770a48 100644 --- a/tools/pd-simulator/simulator/cases/import_data.go +++ b/tools/pd-simulator/simulator/cases/import_data.go @@ -84,7 +84,7 @@ func newImportData() *Case { leaderTotal := 0 peerTotal := 0 res := make([]*core.RegionInfo, 0, 100) - regions.ScanRangeWithIterator([]byte(table12), func(region *core.RegionInfo) bool { + regions.ScanRegionWithIterator([]byte(table12), func(region *core.RegionInfo) bool { if bytes.Compare(region.GetEndKey(), []byte(table13)) < 0 { res = append(res, regions.GetRegion(region.GetID())) return true @@ -116,7 +116,7 @@ func newImportData() *Case { tablePeerLog = fmt.Sprintf("%s [store %d]:%.2f%%", tablePeerLog, storeID, float64(peerCount)/float64(peerTotal)*100) } } - regionTotal := regions.GetRegionCount() + regionTotal := regions.GetTotalRegionCount() totalLeaderLog := fmt.Sprintf("%d leader:", regionTotal) totalPeerLog := fmt.Sprintf("%d peer:", regionTotal*3) isEnd := false diff --git a/tools/pd-simulator/simulator/client.go b/tools/pd-simulator/simulator/client.go index 36e0d27869b..8dd1ee1646e 100644 --- a/tools/pd-simulator/simulator/client.go +++ b/tools/pd-simulator/simulator/client.go @@ -149,21 +149,22 @@ func (c *client) createHeartbeatStream() (pdpb.PD_RegionHeartbeatClient, context cancel context.CancelFunc ctx context.Context ) + ticker := time.NewTicker(time.Second) + defer ticker.Stop() for { ctx, cancel = context.WithCancel(c.ctx) stream, err = c.pdClient().RegionHeartbeat(ctx) - if err != nil { - simutil.Logger.Error("create region heartbeat stream error", zap.String("tag", c.tag), zap.Error(err)) - cancel() - select { - case <-time.After(time.Second): - continue - case <-c.ctx.Done(): - simutil.Logger.Info("cancel create stream loop") - return nil, ctx, cancel - } + if err == nil { + break + } + simutil.Logger.Error("create region heartbeat stream error", zap.String("tag", c.tag), zap.Error(err)) + cancel() + select { + case <-c.ctx.Done(): + simutil.Logger.Info("cancel create stream loop") + return nil, ctx, cancel + case <-ticker.C: } - break } return stream, ctx, cancel } diff --git a/tools/pd-simulator/simulator/config.go b/tools/pd-simulator/simulator/config.go index 461241cc52f..4f197fb83c2 100644 --- a/tools/pd-simulator/simulator/config.go +++ b/tools/pd-simulator/simulator/config.go @@ -21,6 +21,7 @@ import ( "github.com/BurntSushi/toml" "github.com/docker/go-units" + sc "github.com/tikv/pd/pkg/schedule/config" "github.com/tikv/pd/pkg/schedule/placement" "github.com/tikv/pd/pkg/utils/configutil" "github.com/tikv/pd/pkg/utils/tempurl" @@ -40,6 +41,8 @@ const ( defaultRegionSplitSize = 96 * units.MiB defaultCapacity = 1 * units.TiB defaultExtraUsedSpace = 0 + // TSO Proxy related + defaultMaxConcurrentTSOProxyStreamings = 5000 // server defaultLeaderLease = 3 defaultTSOSaveInterval = 200 * time.Millisecond @@ -78,7 +81,7 @@ type Coprocessor struct { // NewSimConfig create a new configuration of the simulator. func NewSimConfig(serverLogLevel string) *SimConfig { - config.DefaultStoreLimit = config.StoreLimit{AddPeer: 2000, RemovePeer: 2000} + sc.DefaultStoreLimit = sc.StoreLimit{AddPeer: 2000, RemovePeer: 2000} cfg := &config.Config{ Name: "pd", ClientUrls: tempurl.Alloc(), @@ -105,6 +108,8 @@ func (sc *SimConfig) Adjust(meta *toml.MetaData) error { configutil.AdjustUint64(&sc.Coprocessor.RegionSplitKey, defaultRegionSplitKeys) configutil.AdjustByteSize(&sc.Coprocessor.RegionSplitSize, defaultRegionSplitSize) + configutil.AdjustInt(&sc.ServerConfig.MaxConcurrentTSOProxyStreamings, defaultMaxConcurrentTSOProxyStreamings) + configutil.AdjustInt64(&sc.ServerConfig.LeaderLease, defaultLeaderLease) configutil.AdjustDuration(&sc.ServerConfig.TSOSaveInterval, defaultTSOSaveInterval) configutil.AdjustDuration(&sc.ServerConfig.TickInterval, defaultTickInterval) @@ -113,6 +118,9 @@ func (sc *SimConfig) Adjust(meta *toml.MetaData) error { return sc.ServerConfig.Adjust(meta, false) } +func (sc *SimConfig) speed() uint64 { + return uint64(time.Second / sc.SimTickInterval.Duration) +} // PDConfig saves some config which may be changed in PD. type PDConfig struct { diff --git a/tools/pd-simulator/simulator/node.go b/tools/pd-simulator/simulator/node.go index cd76d80b3c4..b8fb422d6dd 100644 --- a/tools/pd-simulator/simulator/node.go +++ b/tools/pd-simulator/simulator/node.go @@ -52,6 +52,7 @@ type Node struct { limiter *ratelimit.RateLimiter sizeMutex sync.Mutex hasExtraUsedSpace bool + snapStats []*pdpb.SnapshotStat } // NewNode returns a Node. @@ -91,8 +92,8 @@ func NewNode(s *cases.Store, pdAddr string, config *SimConfig) (*Node, error) { cancel() return nil, err } - ratio := int64(time.Second) / config.SimTickInterval.Milliseconds() - speed := config.StoreIOMBPerSecond * units.MiB * ratio + ratio := config.speed() + speed := config.StoreIOMBPerSecond * units.MiB * int64(ratio) return &Node{ Store: store, stats: stats, @@ -104,6 +105,7 @@ func NewNode(s *cases.Store, pdAddr string, config *SimConfig) (*Node, error) { limiter: ratelimit.NewRateLimiter(float64(speed), int(speed)), tick: uint64(rand.Intn(storeHeartBeatPeriod)), hasExtraUsedSpace: s.HasExtraUsedSpace, + snapStats: make([]*pdpb.SnapshotStat, 0), }, nil } @@ -191,6 +193,10 @@ func (n *Node) storeHeartBeat() { return } ctx, cancel := context.WithTimeout(n.ctx, pdTimeout) + stats := make([]*pdpb.SnapshotStat, len(n.snapStats)) + copy(stats, n.snapStats) + n.snapStats = n.snapStats[:0] + n.stats.SnapshotStats = stats err := n.client.StoreHeartbeat(ctx, &n.stats.StoreStats) if err != nil { simutil.Logger.Info("report heartbeat error", @@ -279,3 +285,12 @@ func (n *Node) decUsedSize(size uint64) { defer n.sizeMutex.Unlock() n.stats.ToCompactionSize += size } + +func (n *Node) registerSnapStats(generate, send, total uint64) { + stat := pdpb.SnapshotStat{ + GenerateDurationSec: generate, + SendDurationSec: send, + TotalDurationSec: total, + } + n.snapStats = append(n.snapStats, &stat) +} diff --git a/tools/pd-simulator/simulator/raft.go b/tools/pd-simulator/simulator/raft.go index 617edd7f84c..fccf75781d3 100644 --- a/tools/pd-simulator/simulator/raft.go +++ b/tools/pd-simulator/simulator/raft.go @@ -291,7 +291,7 @@ func (r *RaftEngine) GetRegionByKey(regionKey []byte) *core.RegionInfo { func (r *RaftEngine) BootstrapRegion() *core.RegionInfo { r.RLock() defer r.RUnlock() - regions := r.regionsInfo.ScanRange(nil, nil, 1) + regions := r.regionsInfo.ScanRegions(nil, nil, 1) if len(regions) > 0 { return regions[0] } diff --git a/tools/pd-simulator/simulator/task.go b/tools/pd-simulator/simulator/task.go index 8e55902615f..b1c609b503d 100644 --- a/tools/pd-simulator/simulator/task.go +++ b/tools/pd-simulator/simulator/task.go @@ -415,13 +415,14 @@ func (a *addPeer) tick(engine *RaftEngine, region *core.RegionInfo) (newRegion * pendingPeers := append(region.GetPendingPeers(), a.peer) return region.Clone(core.WithAddPeer(a.peer), core.WithIncConfVer(), core.WithPendingPeers(pendingPeers)), false } + speed := engine.storeConfig.speed() // Step 2: Process Snapshot - if !processSnapshot(sendNode, a.sendingStat) { + if !processSnapshot(sendNode, a.sendingStat, speed) { return nil, false } sendStoreID := fmt.Sprintf("store-%d", sendNode.Id) snapshotCounter.WithLabelValues(sendStoreID, "send").Inc() - if !processSnapshot(recvNode, a.receivingStat) { + if !processSnapshot(recvNode, a.receivingStat, speed) { return nil, false } recvStoreID := fmt.Sprintf("store-%d", recvNode.Id) @@ -492,10 +493,11 @@ func removeDownPeers(region *core.RegionInfo, removePeer *metapb.Peer) core.Regi } type snapshotStat struct { - action snapAction - remainSize int64 - status snapStatus - start time.Time + action snapAction + remainSize int64 + status snapStatus + start time.Time + generateStart time.Time } func newSnapshotState(size int64, action snapAction) *snapshotStat { @@ -510,7 +512,7 @@ func newSnapshotState(size int64, action snapAction) *snapshotStat { } } -func processSnapshot(n *Node, stat *snapshotStat) bool { +func processSnapshot(n *Node, stat *snapshotStat, speed uint64) bool { if stat.status == finished { return true } @@ -522,6 +524,7 @@ func processSnapshot(n *Node, stat *snapshotStat) bool { return false } stat.status = running + stat.generateStart = time.Now() // If the statement is true, it will start to send or Receive the snapshot. if stat.action == generate { n.stats.SendingSnapCount++ @@ -542,6 +545,9 @@ func processSnapshot(n *Node, stat *snapshotStat) bool { } if stat.status == running { stat.status = finished + totalSec := uint64(time.Since(stat.start).Seconds()) * speed + generateSec := uint64(time.Since(stat.generateStart).Seconds()) * speed + n.registerSnapStats(generateSec, 0, totalSec) if stat.action == generate { n.stats.SendingSnapCount-- } else { diff --git a/tools/pd-tso-bench/go.mod b/tools/pd-tso-bench/go.mod index 906a0e6ab1d..6c10dbb5fce 100644 --- a/tools/pd-tso-bench/go.mod +++ b/tools/pd-tso-bench/go.mod @@ -8,7 +8,8 @@ require ( github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 github.com/prometheus/client_golang v1.11.1 github.com/tikv/pd/client v0.0.0-00010101000000-000000000000 - go.uber.org/zap v1.20.0 + go.uber.org/zap v1.24.0 + google.golang.org/grpc v1.54.0 ) replace github.com/tikv/pd/client => ../../client diff --git a/tools/pd-tso-bench/go.sum b/tools/pd-tso-bench/go.sum index 25344882ec7..1fa45d063ff 100644 --- a/tools/pd-tso-bench/go.sum +++ b/tools/pd-tso-bench/go.sum @@ -3,6 +3,7 @@ cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.44.3/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= @@ -15,6 +16,7 @@ cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOY cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= +cloud.google.com/go v0.75.0/go.mod h1:VGuuCn7PG0dwsd5XPVm2Mm3wlh3EL55/79EKB6hlPTY= cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= @@ -32,47 +34,84 @@ cloud.google.com/go v0.102.0/go.mod h1:oWcCzKlqJ5zgHQt9YsaeTY9KzIvjyy0ArmiBUgpQ+ cloud.google.com/go v0.102.1/go.mod h1:XZ77E9qnTEnrgEOvr4xzfdX5TRo7fB4T2F4O6+34hIU= cloud.google.com/go v0.104.0/go.mod h1:OO6xxXdJyvuJPcEPBLN9BJPD+jep5G1+2U5B5gkRYtA= cloud.google.com/go v0.105.0/go.mod h1:PrLgOJNe5nfE9UMxKxgXj4mD3voiP+YQ6gdt6KMFOKM= +cloud.google.com/go v0.107.0/go.mod h1:wpc2eNrD7hXUTy8EKS10jkxpZBjASrORK7goS+3YX2I= +cloud.google.com/go v0.110.0/go.mod h1:SJnCLqQ0FCFGSZMUNUf84MV3Aia54kn7pi8st7tMzaY= cloud.google.com/go/accessapproval v1.4.0/go.mod h1:zybIuC3KpDOvotz59lFe5qxRZx6C75OtwbisN56xYB4= cloud.google.com/go/accessapproval v1.5.0/go.mod h1:HFy3tuiGvMdcd/u+Cu5b9NkO1pEICJ46IR82PoUdplw= +cloud.google.com/go/accessapproval v1.6.0/go.mod h1:R0EiYnwV5fsRFiKZkPHr6mwyk2wxUJ30nL4j2pcFY2E= cloud.google.com/go/accesscontextmanager v1.3.0/go.mod h1:TgCBehyr5gNMz7ZaH9xubp+CE8dkrszb4oK9CWyvD4o= cloud.google.com/go/accesscontextmanager v1.4.0/go.mod h1:/Kjh7BBu/Gh83sv+K60vN9QE5NJcd80sU33vIe2IFPE= +cloud.google.com/go/accesscontextmanager v1.6.0/go.mod h1:8XCvZWfYw3K/ji0iVnp+6pu7huxoQTLmxAbVjbloTtM= +cloud.google.com/go/accesscontextmanager v1.7.0/go.mod h1:CEGLewx8dwa33aDAZQujl7Dx+uYhS0eay198wB/VumQ= cloud.google.com/go/aiplatform v1.22.0/go.mod h1:ig5Nct50bZlzV6NvKaTwmplLLddFx0YReh9WfTO5jKw= cloud.google.com/go/aiplatform v1.24.0/go.mod h1:67UUvRBKG6GTayHKV8DBv2RtR1t93YRu5B1P3x99mYY= cloud.google.com/go/aiplatform v1.27.0/go.mod h1:Bvxqtl40l0WImSb04d0hXFU7gDOiq9jQmorivIiWcKg= +cloud.google.com/go/aiplatform v1.35.0/go.mod h1:7MFT/vCaOyZT/4IIFfxH4ErVg/4ku6lKv3w0+tFTgXQ= +cloud.google.com/go/aiplatform v1.36.1/go.mod h1:WTm12vJRPARNvJ+v6P52RDHCNe4AhvjcIZ/9/RRHy/k= +cloud.google.com/go/aiplatform v1.37.0/go.mod h1:IU2Cv29Lv9oCn/9LkFiiuKfwrRTq+QQMbW+hPCxJGZw= cloud.google.com/go/analytics v0.11.0/go.mod h1:DjEWCu41bVbYcKyvlws9Er60YE4a//bK6mnhWvQeFNI= cloud.google.com/go/analytics v0.12.0/go.mod h1:gkfj9h6XRf9+TS4bmuhPEShsh3hH8PAZzm/41OOhQd4= +cloud.google.com/go/analytics v0.17.0/go.mod h1:WXFa3WSym4IZ+JiKmavYdJwGG/CvpqiqczmL59bTD9M= +cloud.google.com/go/analytics v0.18.0/go.mod h1:ZkeHGQlcIPkw0R/GW+boWHhCOR43xz9RN/jn7WcqfIE= +cloud.google.com/go/analytics v0.19.0/go.mod h1:k8liqf5/HCnOUkbawNtrWWc+UAzyDlW89doe8TtoDsE= cloud.google.com/go/apigateway v1.3.0/go.mod h1:89Z8Bhpmxu6AmUxuVRg/ECRGReEdiP3vQtk4Z1J9rJk= cloud.google.com/go/apigateway v1.4.0/go.mod h1:pHVY9MKGaH9PQ3pJ4YLzoj6U5FUDeDFBllIz7WmzJoc= +cloud.google.com/go/apigateway v1.5.0/go.mod h1:GpnZR3Q4rR7LVu5951qfXPJCHquZt02jf7xQx7kpqN8= cloud.google.com/go/apigeeconnect v1.3.0/go.mod h1:G/AwXFAKo0gIXkPTVfZDd2qA1TxBXJ3MgMRBQkIi9jc= cloud.google.com/go/apigeeconnect v1.4.0/go.mod h1:kV4NwOKqjvt2JYR0AoIWo2QGfoRtn/pkS3QlHp0Ni04= +cloud.google.com/go/apigeeconnect v1.5.0/go.mod h1:KFaCqvBRU6idyhSNyn3vlHXc8VMDJdRmwDF6JyFRqZ8= +cloud.google.com/go/apigeeregistry v0.4.0/go.mod h1:EUG4PGcsZvxOXAdyEghIdXwAEi/4MEaoqLMLDMIwKXY= +cloud.google.com/go/apigeeregistry v0.5.0/go.mod h1:YR5+s0BVNZfVOUkMa5pAR2xGd0A473vA5M7j247o1wM= +cloud.google.com/go/apigeeregistry v0.6.0/go.mod h1:BFNzW7yQVLZ3yj0TKcwzb8n25CFBri51GVGOEUcgQsc= +cloud.google.com/go/apikeys v0.4.0/go.mod h1:XATS/yqZbaBK0HOssf+ALHp8jAlNHUgyfprvNcBIszU= +cloud.google.com/go/apikeys v0.5.0/go.mod h1:5aQfwY4D+ewMMWScd3hm2en3hCj+BROlyrt3ytS7KLI= +cloud.google.com/go/apikeys v0.6.0/go.mod h1:kbpXu5upyiAlGkKrJgQl8A0rKNNJ7dQ377pdroRSSi8= cloud.google.com/go/appengine v1.4.0/go.mod h1:CS2NhuBuDXM9f+qscZ6V86m1MIIqPj3WC/UoEuR1Sno= cloud.google.com/go/appengine v1.5.0/go.mod h1:TfasSozdkFI0zeoxW3PTBLiNqRmzraodCWatWI9Dmak= +cloud.google.com/go/appengine v1.6.0/go.mod h1:hg6i0J/BD2cKmDJbaFSYHFyZkgBEfQrDg/X0V5fJn84= +cloud.google.com/go/appengine v1.7.0/go.mod h1:eZqpbHFCqRGa2aCdope7eC0SWLV1j0neb/QnMJVWx6A= +cloud.google.com/go/appengine v1.7.1/go.mod h1:IHLToyb/3fKutRysUlFO0BPt5j7RiQ45nrzEJmKTo6E= cloud.google.com/go/area120 v0.5.0/go.mod h1:DE/n4mp+iqVyvxHN41Vf1CR602GiHQjFPusMFW6bGR4= cloud.google.com/go/area120 v0.6.0/go.mod h1:39yFJqWVgm0UZqWTOdqkLhjoC7uFfgXRC8g/ZegeAh0= +cloud.google.com/go/area120 v0.7.0/go.mod h1:a3+8EUD1SX5RUcCs3MY5YasiO1z6yLiNLRiFrykbynY= +cloud.google.com/go/area120 v0.7.1/go.mod h1:j84i4E1RboTWjKtZVWXPqvK5VHQFJRF2c1Nm69pWm9k= cloud.google.com/go/artifactregistry v1.6.0/go.mod h1:IYt0oBPSAGYj/kprzsBjZ/4LnG/zOcHyFHjWPCi6SAQ= cloud.google.com/go/artifactregistry v1.7.0/go.mod h1:mqTOFOnGZx8EtSqK/ZWcsm/4U8B77rbcLP6ruDU2Ixk= cloud.google.com/go/artifactregistry v1.8.0/go.mod h1:w3GQXkJX8hiKN0v+at4b0qotwijQbYUqF2GWkZzAhC0= cloud.google.com/go/artifactregistry v1.9.0/go.mod h1:2K2RqvA2CYvAeARHRkLDhMDJ3OXy26h3XW+3/Jh2uYc= +cloud.google.com/go/artifactregistry v1.11.1/go.mod h1:lLYghw+Itq9SONbCa1YWBoWs1nOucMH0pwXN1rOBZFI= +cloud.google.com/go/artifactregistry v1.11.2/go.mod h1:nLZns771ZGAwVLzTX/7Al6R9ehma4WUEhZGWV6CeQNQ= +cloud.google.com/go/artifactregistry v1.12.0/go.mod h1:o6P3MIvtzTOnmvGagO9v/rOjjA0HmhJ+/6KAXrmYDCI= +cloud.google.com/go/artifactregistry v1.13.0/go.mod h1:uy/LNfoOIivepGhooAUpL1i30Hgee3Cu0l4VTWHUC08= cloud.google.com/go/asset v1.5.0/go.mod h1:5mfs8UvcM5wHhqtSv8J1CtxxaQq3AdBxxQi2jGW/K4o= cloud.google.com/go/asset v1.7.0/go.mod h1:YbENsRK4+xTiL+Ofoj5Ckf+O17kJtgp3Y3nn4uzZz5s= cloud.google.com/go/asset v1.8.0/go.mod h1:mUNGKhiqIdbr8X7KNayoYvyc4HbbFO9URsjbytpUaW0= cloud.google.com/go/asset v1.9.0/go.mod h1:83MOE6jEJBMqFKadM9NLRcs80Gdw76qGuHn8m3h8oHQ= cloud.google.com/go/asset v1.10.0/go.mod h1:pLz7uokL80qKhzKr4xXGvBQXnzHn5evJAEAtZiIb0wY= +cloud.google.com/go/asset v1.11.1/go.mod h1:fSwLhbRvC9p9CXQHJ3BgFeQNM4c9x10lqlrdEUYXlJo= +cloud.google.com/go/asset v1.12.0/go.mod h1:h9/sFOa4eDIyKmH6QMpm4eUK3pDojWnUhTgJlk762Hg= +cloud.google.com/go/asset v1.13.0/go.mod h1:WQAMyYek/b7NBpYq/K4KJWcRqzoalEsxz/t/dTk4THw= cloud.google.com/go/assuredworkloads v1.5.0/go.mod h1:n8HOZ6pff6re5KYfBXcFvSViQjDwxFkAkmUFffJRbbY= cloud.google.com/go/assuredworkloads v1.6.0/go.mod h1:yo2YOk37Yc89Rsd5QMVECvjaMKymF9OP+QXWlKXUkXw= cloud.google.com/go/assuredworkloads v1.7.0/go.mod h1:z/736/oNmtGAyU47reJgGN+KVoYoxeLBoj4XkKYscNI= cloud.google.com/go/assuredworkloads v1.8.0/go.mod h1:AsX2cqyNCOvEQC8RMPnoc0yEarXQk6WEKkxYfL6kGIo= cloud.google.com/go/assuredworkloads v1.9.0/go.mod h1:kFuI1P78bplYtT77Tb1hi0FMxM0vVpRC7VVoJC3ZoT0= +cloud.google.com/go/assuredworkloads v1.10.0/go.mod h1:kwdUQuXcedVdsIaKgKTp9t0UJkE5+PAVNhdQm4ZVq2E= cloud.google.com/go/automl v1.5.0/go.mod h1:34EjfoFGMZ5sgJ9EoLsRtdPSNZLcfflJR39VbVNS2M0= cloud.google.com/go/automl v1.6.0/go.mod h1:ugf8a6Fx+zP0D59WLhqgTDsQI9w07o64uf/Is3Nh5p8= cloud.google.com/go/automl v1.7.0/go.mod h1:RL9MYCCsJEOmt0Wf3z9uzG0a7adTT1fe+aObgSpkCt8= cloud.google.com/go/automl v1.8.0/go.mod h1:xWx7G/aPEe/NP+qzYXktoBSDfjO+vnKMGgsApGJJquM= +cloud.google.com/go/automl v1.12.0/go.mod h1:tWDcHDp86aMIuHmyvjuKeeHEGq76lD7ZqfGLN6B0NuU= cloud.google.com/go/baremetalsolution v0.3.0/go.mod h1:XOrocE+pvK1xFfleEnShBlNAXf+j5blPPxrhjKgnIFc= cloud.google.com/go/baremetalsolution v0.4.0/go.mod h1:BymplhAadOO/eBa7KewQ0Ppg4A4Wplbn+PsFKRLo0uI= +cloud.google.com/go/baremetalsolution v0.5.0/go.mod h1:dXGxEkmR9BMwxhzBhV0AioD0ULBmuLZI8CdwalUxuss= cloud.google.com/go/batch v0.3.0/go.mod h1:TR18ZoAekj1GuirsUsR1ZTKN3FC/4UDnScjT8NXImFE= cloud.google.com/go/batch v0.4.0/go.mod h1:WZkHnP43R/QCGQsZ+0JyG4i79ranE2u8xvjq/9+STPE= +cloud.google.com/go/batch v0.7.0/go.mod h1:vLZN95s6teRUqRQ4s3RLDsH8PvboqBK+rn1oevL159g= cloud.google.com/go/beyondcorp v0.2.0/go.mod h1:TB7Bd+EEtcw9PCPQhCJtJGjk/7TC6ckmnSFS+xwTfm4= cloud.google.com/go/beyondcorp v0.3.0/go.mod h1:E5U5lcrcXMsCuoDNyGrpyTm/hn7ne941Jz2vmksAxW8= +cloud.google.com/go/beyondcorp v0.4.0/go.mod h1:3ApA0mbhHx6YImmuubf5pyW8srKnCEPON32/5hj+RmM= +cloud.google.com/go/beyondcorp v0.5.0/go.mod h1:uFqj9X+dSfrheVp7ssLTaRHd2EHqSL4QZmH4e8WXGGU= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= @@ -82,26 +121,42 @@ cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM7 cloud.google.com/go/bigquery v1.42.0/go.mod h1:8dRTJxhtG+vwBKzE5OseQn/hiydoQN3EedCaOdYmxRA= cloud.google.com/go/bigquery v1.43.0/go.mod h1:ZMQcXHsl+xmU1z36G2jNGZmKp9zNY5BUua5wDgmNCfw= cloud.google.com/go/bigquery v1.44.0/go.mod h1:0Y33VqXTEsbamHJvJHdFmtqHvMIY28aK1+dFsvaChGc= +cloud.google.com/go/bigquery v1.47.0/go.mod h1:sA9XOgy0A8vQK9+MWhEQTY6Tix87M/ZurWFIxmF9I/E= +cloud.google.com/go/bigquery v1.48.0/go.mod h1:QAwSz+ipNgfL5jxiaK7weyOhzdoAy1zFm0Nf1fysJac= +cloud.google.com/go/bigquery v1.49.0/go.mod h1:Sv8hMmTFFYBlt/ftw2uN6dFdQPzBlREY9yBh7Oy7/4Q= +cloud.google.com/go/bigquery v1.50.0/go.mod h1:YrleYEh2pSEbgTBZYMJ5SuSr0ML3ypjRB1zgf7pvQLU= cloud.google.com/go/billing v1.4.0/go.mod h1:g9IdKBEFlItS8bTtlrZdVLWSSdSyFUZKXNS02zKMOZY= cloud.google.com/go/billing v1.5.0/go.mod h1:mztb1tBc3QekhjSgmpf/CV4LzWXLzCArwpLmP2Gm88s= cloud.google.com/go/billing v1.6.0/go.mod h1:WoXzguj+BeHXPbKfNWkqVtDdzORazmCjraY+vrxcyvI= cloud.google.com/go/billing v1.7.0/go.mod h1:q457N3Hbj9lYwwRbnlD7vUpyjq6u5U1RAOArInEiD5Y= +cloud.google.com/go/billing v1.12.0/go.mod h1:yKrZio/eu+okO/2McZEbch17O5CB5NpZhhXG6Z766ss= +cloud.google.com/go/billing v1.13.0/go.mod h1:7kB2W9Xf98hP9Sr12KfECgfGclsH3CQR0R08tnRlRbc= cloud.google.com/go/binaryauthorization v1.1.0/go.mod h1:xwnoWu3Y84jbuHa0zd526MJYmtnVXn0syOjaJgy4+dM= cloud.google.com/go/binaryauthorization v1.2.0/go.mod h1:86WKkJHtRcv5ViNABtYMhhNWRrD1Vpi//uKEy7aYEfI= cloud.google.com/go/binaryauthorization v1.3.0/go.mod h1:lRZbKgjDIIQvzYQS1p99A7/U1JqvqeZg0wiI5tp6tg0= cloud.google.com/go/binaryauthorization v1.4.0/go.mod h1:tsSPQrBd77VLplV70GUhBf/Zm3FsKmgSqgm4UmiDItk= +cloud.google.com/go/binaryauthorization v1.5.0/go.mod h1:OSe4OU1nN/VswXKRBmciKpo9LulY41gch5c68htf3/Q= cloud.google.com/go/certificatemanager v1.3.0/go.mod h1:n6twGDvcUBFu9uBgt4eYvvf3sQ6My8jADcOVwHmzadg= cloud.google.com/go/certificatemanager v1.4.0/go.mod h1:vowpercVFyqs8ABSmrdV+GiFf2H/ch3KyudYQEMM590= +cloud.google.com/go/certificatemanager v1.6.0/go.mod h1:3Hh64rCKjRAX8dXgRAyOcY5vQ/fE1sh8o+Mdd6KPgY8= cloud.google.com/go/channel v1.8.0/go.mod h1:W5SwCXDJsq/rg3tn3oG0LOxpAo6IMxNa09ngphpSlnk= cloud.google.com/go/channel v1.9.0/go.mod h1:jcu05W0my9Vx4mt3/rEHpfxc9eKi9XwsdDL8yBMbKUk= +cloud.google.com/go/channel v1.11.0/go.mod h1:IdtI0uWGqhEeatSB62VOoJ8FSUhJ9/+iGkJVqp74CGE= +cloud.google.com/go/channel v1.12.0/go.mod h1:VkxCGKASi4Cq7TbXxlaBezonAYpp1GCnKMY6tnMQnLU= cloud.google.com/go/cloudbuild v1.3.0/go.mod h1:WequR4ULxlqvMsjDEEEFnOG5ZSRSgWOywXYDb1vPE6U= cloud.google.com/go/cloudbuild v1.4.0/go.mod h1:5Qwa40LHiOXmz3386FrjrYM93rM/hdRr7b53sySrTqA= +cloud.google.com/go/cloudbuild v1.6.0/go.mod h1:UIbc/w9QCbH12xX+ezUsgblrWv+Cv4Tw83GiSMHOn9M= +cloud.google.com/go/cloudbuild v1.7.0/go.mod h1:zb5tWh2XI6lR9zQmsm1VRA+7OCuve5d8S+zJUul8KTg= +cloud.google.com/go/cloudbuild v1.9.0/go.mod h1:qK1d7s4QlO0VwfYn5YuClDGg2hfmLZEb4wQGAbIgL1s= cloud.google.com/go/clouddms v1.3.0/go.mod h1:oK6XsCDdW4Ib3jCCBugx+gVjevp2TMXFtgxvPSee3OM= cloud.google.com/go/clouddms v1.4.0/go.mod h1:Eh7sUGCC+aKry14O1NRljhjyrr0NFC0G2cjwX0cByRk= +cloud.google.com/go/clouddms v1.5.0/go.mod h1:QSxQnhikCLUw13iAbffF2CZxAER3xDGNHjsTAkQJcQA= cloud.google.com/go/cloudtasks v1.5.0/go.mod h1:fD92REy1x5woxkKEkLdvavGnPJGEn8Uic9nWuLzqCpY= cloud.google.com/go/cloudtasks v1.6.0/go.mod h1:C6Io+sxuke9/KNRkbQpihnW93SWDU3uXt92nu85HkYI= cloud.google.com/go/cloudtasks v1.7.0/go.mod h1:ImsfdYWwlWNJbdgPIIGJWC+gemEGTBK/SunNQQNCAb4= cloud.google.com/go/cloudtasks v1.8.0/go.mod h1:gQXUIwCSOI4yPVK7DgTVFiiP0ZW/eQkydWzwVMdHxrI= +cloud.google.com/go/cloudtasks v1.9.0/go.mod h1:w+EyLsVkLWHcOaqNEyvcKAsWp9p29dL6uL9Nst1cI7Y= +cloud.google.com/go/cloudtasks v1.10.0/go.mod h1:NDSoTLkZ3+vExFEWu2UJV1arUyzVDAiZtdWcsUyNwBs= cloud.google.com/go/compute v0.1.0/go.mod h1:GAesmwr110a34z04OlxYkATPBEfVhkymfTBXtfbBFow= cloud.google.com/go/compute v1.3.0/go.mod h1:cCZiE1NHEtai4wiufUhW8I8S1JKkAnhnQJWM7YD99wM= cloud.google.com/go/compute v1.5.0/go.mod h1:9SMHyhJlzhlkJqrPAc839t2BZFTSk6Jdj6mkzQJeu0M= @@ -112,307 +167,491 @@ cloud.google.com/go/compute v1.10.0/go.mod h1:ER5CLbMxl90o2jtNbGSbtfOpQKR0t15FOt cloud.google.com/go/compute v1.12.0/go.mod h1:e8yNOBcBONZU1vJKCvCoDw/4JQsA0dpM4x/6PIIOocU= cloud.google.com/go/compute v1.12.1/go.mod h1:e8yNOBcBONZU1vJKCvCoDw/4JQsA0dpM4x/6PIIOocU= cloud.google.com/go/compute v1.13.0/go.mod h1:5aPTS0cUNMIc1CE546K+Th6weJUNQErARyZtRXDJ8GE= +cloud.google.com/go/compute v1.14.0/go.mod h1:YfLtxrj9sU4Yxv+sXzZkyPjEyPBZfXHUvjxega5vAdo= +cloud.google.com/go/compute v1.15.1/go.mod h1:bjjoF/NtFUrkD/urWfdHaKuOPDR5nWIs63rR+SXhcpA= +cloud.google.com/go/compute v1.18.0/go.mod h1:1X7yHxec2Ga+Ss6jPyjxRxpu2uu7PLgsOVXvgU0yacs= +cloud.google.com/go/compute v1.19.0/go.mod h1:rikpw2y+UMidAe9tISo04EHNOIf42RLYF/q8Bs93scU= cloud.google.com/go/compute/metadata v0.1.0/go.mod h1:Z1VN+bulIf6bt4P/C37K4DyZYZEXYonfTBHHFPO/4UU= +cloud.google.com/go/compute/metadata v0.2.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= cloud.google.com/go/compute/metadata v0.2.1/go.mod h1:jgHgmJd2RKBGzXqF5LR2EZMGxBkeanZ9wwa75XHJgOM= +cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= cloud.google.com/go/contactcenterinsights v1.3.0/go.mod h1:Eu2oemoePuEFc/xKFPjbTuPSj0fYJcPls9TFlPNnHHY= cloud.google.com/go/contactcenterinsights v1.4.0/go.mod h1:L2YzkGbPsv+vMQMCADxJoT9YiTTnSEd6fEvCeHTYVck= +cloud.google.com/go/contactcenterinsights v1.6.0/go.mod h1:IIDlT6CLcDoyv79kDv8iWxMSTZhLxSCofVV5W6YFM/w= cloud.google.com/go/container v1.6.0/go.mod h1:Xazp7GjJSeUYo688S+6J5V+n/t+G5sKBTFkKNudGRxg= cloud.google.com/go/container v1.7.0/go.mod h1:Dp5AHtmothHGX3DwwIHPgq45Y8KmNsgN3amoYfxVkLo= +cloud.google.com/go/container v1.13.1/go.mod h1:6wgbMPeQRw9rSnKBCAJXnds3Pzj03C4JHamr8asWKy4= +cloud.google.com/go/container v1.14.0/go.mod h1:3AoJMPhHfLDxLvrlVWaK57IXzaPnLaZq63WX59aQBfM= +cloud.google.com/go/container v1.15.0/go.mod h1:ft+9S0WGjAyjDggg5S06DXj+fHJICWg8L7isCQe9pQA= cloud.google.com/go/containeranalysis v0.5.1/go.mod h1:1D92jd8gRR/c0fGMlymRgxWD3Qw9C1ff6/T7mLgVL8I= cloud.google.com/go/containeranalysis v0.6.0/go.mod h1:HEJoiEIu+lEXM+k7+qLCci0h33lX3ZqoYFdmPcoO7s4= +cloud.google.com/go/containeranalysis v0.7.0/go.mod h1:9aUL+/vZ55P2CXfuZjS4UjQ9AgXoSw8Ts6lemfmxBxI= +cloud.google.com/go/containeranalysis v0.9.0/go.mod h1:orbOANbwk5Ejoom+s+DUCTTJ7IBdBQJDcSylAx/on9s= cloud.google.com/go/datacatalog v1.3.0/go.mod h1:g9svFY6tuR+j+hrTw3J2dNcmI0dzmSiyOzm8kpLq0a0= cloud.google.com/go/datacatalog v1.5.0/go.mod h1:M7GPLNQeLfWqeIm3iuiruhPzkt65+Bx8dAKvScX8jvs= cloud.google.com/go/datacatalog v1.6.0/go.mod h1:+aEyF8JKg+uXcIdAmmaMUmZ3q1b/lKLtXCmXdnc0lbc= cloud.google.com/go/datacatalog v1.7.0/go.mod h1:9mEl4AuDYWw81UGc41HonIHH7/sn52H0/tc8f8ZbZIE= cloud.google.com/go/datacatalog v1.8.0/go.mod h1:KYuoVOv9BM8EYz/4eMFxrr4DUKhGIOXxZoKYF5wdISM= +cloud.google.com/go/datacatalog v1.8.1/go.mod h1:RJ58z4rMp3gvETA465Vg+ag8BGgBdnRPEMMSTr5Uv+M= +cloud.google.com/go/datacatalog v1.12.0/go.mod h1:CWae8rFkfp6LzLumKOnmVh4+Zle4A3NXLzVJ1d1mRm0= +cloud.google.com/go/datacatalog v1.13.0/go.mod h1:E4Rj9a5ZtAxcQJlEBTLgMTphfP11/lNaAshpoBgemX8= cloud.google.com/go/dataflow v0.6.0/go.mod h1:9QwV89cGoxjjSR9/r7eFDqqjtvbKxAK2BaYU6PVk9UM= cloud.google.com/go/dataflow v0.7.0/go.mod h1:PX526vb4ijFMesO1o202EaUmouZKBpjHsTlCtB4parQ= +cloud.google.com/go/dataflow v0.8.0/go.mod h1:Rcf5YgTKPtQyYz8bLYhFoIV/vP39eL7fWNcSOyFfLJE= cloud.google.com/go/dataform v0.3.0/go.mod h1:cj8uNliRlHpa6L3yVhDOBrUXH+BPAO1+KFMQQNSThKo= cloud.google.com/go/dataform v0.4.0/go.mod h1:fwV6Y4Ty2yIFL89huYlEkwUPtS7YZinZbzzj5S9FzCE= cloud.google.com/go/dataform v0.5.0/go.mod h1:GFUYRe8IBa2hcomWplodVmUx/iTL0FrsauObOM3Ipr0= +cloud.google.com/go/dataform v0.6.0/go.mod h1:QPflImQy33e29VuapFdf19oPbE4aYTJxr31OAPV+ulA= +cloud.google.com/go/dataform v0.7.0/go.mod h1:7NulqnVozfHvWUBpMDfKMUESr+85aJsC/2O0o3jWPDE= cloud.google.com/go/datafusion v1.4.0/go.mod h1:1Zb6VN+W6ALo85cXnM1IKiPw+yQMKMhB9TsTSRDo/38= cloud.google.com/go/datafusion v1.5.0/go.mod h1:Kz+l1FGHB0J+4XF2fud96WMmRiq/wj8N9u007vyXZ2w= +cloud.google.com/go/datafusion v1.6.0/go.mod h1:WBsMF8F1RhSXvVM8rCV3AeyWVxcC2xY6vith3iw3S+8= cloud.google.com/go/datalabeling v0.5.0/go.mod h1:TGcJ0G2NzcsXSE/97yWjIZO0bXj0KbVlINXMG9ud42I= cloud.google.com/go/datalabeling v0.6.0/go.mod h1:WqdISuk/+WIGeMkpw/1q7bK/tFEZxsrFJOJdY2bXvTQ= +cloud.google.com/go/datalabeling v0.7.0/go.mod h1:WPQb1y08RJbmpM3ww0CSUAGweL0SxByuW2E+FU+wXcM= cloud.google.com/go/dataplex v1.3.0/go.mod h1:hQuRtDg+fCiFgC8j0zV222HvzFQdRd+SVX8gdmFcZzA= cloud.google.com/go/dataplex v1.4.0/go.mod h1:X51GfLXEMVJ6UN47ESVqvlsRplbLhcsAt0kZCCKsU0A= +cloud.google.com/go/dataplex v1.5.2/go.mod h1:cVMgQHsmfRoI5KFYq4JtIBEUbYwc3c7tXmIDhRmNNVQ= +cloud.google.com/go/dataplex v1.6.0/go.mod h1:bMsomC/aEJOSpHXdFKFGQ1b0TDPIeL28nJObeO1ppRs= cloud.google.com/go/dataproc v1.7.0/go.mod h1:CKAlMjII9H90RXaMpSxQ8EU6dQx6iAYNPcYPOkSbi8s= cloud.google.com/go/dataproc v1.8.0/go.mod h1:5OW+zNAH0pMpw14JVrPONsxMQYMBqJuzORhIBfBn9uI= +cloud.google.com/go/dataproc v1.12.0/go.mod h1:zrF3aX0uV3ikkMz6z4uBbIKyhRITnxvr4i3IjKsKrw4= cloud.google.com/go/dataqna v0.5.0/go.mod h1:90Hyk596ft3zUQ8NkFfvICSIfHFh1Bc7C4cK3vbhkeo= cloud.google.com/go/dataqna v0.6.0/go.mod h1:1lqNpM7rqNLVgWBJyk5NF6Uen2PHym0jtVJonplVsDA= +cloud.google.com/go/dataqna v0.7.0/go.mod h1:Lx9OcIIeqCrw1a6KdO3/5KMP1wAmTc0slZWwP12Qq3c= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/datastore v1.10.0/go.mod h1:PC5UzAmDEkAmkfaknstTYbNpgE49HAgW2J1gcgUfmdM= +cloud.google.com/go/datastore v1.11.0/go.mod h1:TvGxBIHCS50u8jzG+AW/ppf87v1of8nwzFNgEZU1D3c= cloud.google.com/go/datastream v1.2.0/go.mod h1:i/uTP8/fZwgATHS/XFu0TcNUhuA0twZxxQ3EyCUQMwo= cloud.google.com/go/datastream v1.3.0/go.mod h1:cqlOX8xlyYF/uxhiKn6Hbv6WjwPPuI9W2M9SAXwaLLQ= cloud.google.com/go/datastream v1.4.0/go.mod h1:h9dpzScPhDTs5noEMQVWP8Wx8AFBRyS0s8KWPx/9r0g= cloud.google.com/go/datastream v1.5.0/go.mod h1:6TZMMNPwjUqZHBKPQ1wwXpb0d5VDVPl2/XoS5yi88q4= +cloud.google.com/go/datastream v1.6.0/go.mod h1:6LQSuswqLa7S4rPAOZFVjHIG3wJIjZcZrw8JDEDJuIs= +cloud.google.com/go/datastream v1.7.0/go.mod h1:uxVRMm2elUSPuh65IbZpzJNMbuzkcvu5CjMqVIUHrww= cloud.google.com/go/deploy v1.4.0/go.mod h1:5Xghikd4VrmMLNaF6FiRFDlHb59VM59YoDQnOUdsH/c= cloud.google.com/go/deploy v1.5.0/go.mod h1:ffgdD0B89tToyW/U/D2eL0jN2+IEV/3EMuXHA0l4r+s= +cloud.google.com/go/deploy v1.6.0/go.mod h1:f9PTHehG/DjCom3QH0cntOVRm93uGBDt2vKzAPwpXQI= +cloud.google.com/go/deploy v1.8.0/go.mod h1:z3myEJnA/2wnB4sgjqdMfgxCA0EqC3RBTNcVPs93mtQ= cloud.google.com/go/dialogflow v1.15.0/go.mod h1:HbHDWs33WOGJgn6rfzBW1Kv807BE3O1+xGbn59zZWI4= cloud.google.com/go/dialogflow v1.16.1/go.mod h1:po6LlzGfK+smoSmTBnbkIZY2w8ffjz/RcGSS+sh1el0= cloud.google.com/go/dialogflow v1.17.0/go.mod h1:YNP09C/kXA1aZdBgC/VtXX74G/TKn7XVCcVumTflA+8= cloud.google.com/go/dialogflow v1.18.0/go.mod h1:trO7Zu5YdyEuR+BhSNOqJezyFQ3aUzz0njv7sMx/iek= cloud.google.com/go/dialogflow v1.19.0/go.mod h1:JVmlG1TwykZDtxtTXujec4tQ+D8SBFMoosgy+6Gn0s0= +cloud.google.com/go/dialogflow v1.29.0/go.mod h1:b+2bzMe+k1s9V+F2jbJwpHPzrnIyHihAdRFMtn2WXuM= +cloud.google.com/go/dialogflow v1.31.0/go.mod h1:cuoUccuL1Z+HADhyIA7dci3N5zUssgpBJmCzI6fNRB4= +cloud.google.com/go/dialogflow v1.32.0/go.mod h1:jG9TRJl8CKrDhMEcvfcfFkkpp8ZhgPz3sBGmAUYJ2qE= cloud.google.com/go/dlp v1.6.0/go.mod h1:9eyB2xIhpU0sVwUixfBubDoRwP+GjeUoxxeueZmqvmM= cloud.google.com/go/dlp v1.7.0/go.mod h1:68ak9vCiMBjbasxeVD17hVPxDEck+ExiHavX8kiHG+Q= +cloud.google.com/go/dlp v1.9.0/go.mod h1:qdgmqgTyReTz5/YNSSuueR8pl7hO0o9bQ39ZhtgkWp4= cloud.google.com/go/documentai v1.7.0/go.mod h1:lJvftZB5NRiFSX4moiye1SMxHx0Bc3x1+p9e/RfXYiU= cloud.google.com/go/documentai v1.8.0/go.mod h1:xGHNEB7CtsnySCNrCFdCyyMz44RhFEEX2Q7UD0c5IhU= cloud.google.com/go/documentai v1.9.0/go.mod h1:FS5485S8R00U10GhgBC0aNGrJxBP8ZVpEeJ7PQDZd6k= cloud.google.com/go/documentai v1.10.0/go.mod h1:vod47hKQIPeCfN2QS/jULIvQTugbmdc0ZvxxfQY1bg4= +cloud.google.com/go/documentai v1.16.0/go.mod h1:o0o0DLTEZ+YnJZ+J4wNfTxmDVyrkzFvttBXXtYRMHkM= +cloud.google.com/go/documentai v1.18.0/go.mod h1:F6CK6iUH8J81FehpskRmhLq/3VlwQvb7TvwOceQ2tbs= cloud.google.com/go/domains v0.6.0/go.mod h1:T9Rz3GasrpYk6mEGHh4rymIhjlnIuB4ofT1wTxDeT4Y= cloud.google.com/go/domains v0.7.0/go.mod h1:PtZeqS1xjnXuRPKE/88Iru/LdfoRyEHYA9nFQf4UKpg= +cloud.google.com/go/domains v0.8.0/go.mod h1:M9i3MMDzGFXsydri9/vW+EWz9sWb4I6WyHqdlAk0idE= cloud.google.com/go/edgecontainer v0.1.0/go.mod h1:WgkZ9tp10bFxqO8BLPqv2LlfmQF1X8lZqwW4r1BTajk= cloud.google.com/go/edgecontainer v0.2.0/go.mod h1:RTmLijy+lGpQ7BXuTDa4C4ssxyXT34NIuHIgKuP4s5w= +cloud.google.com/go/edgecontainer v0.3.0/go.mod h1:FLDpP4nykgwwIfcLt6zInhprzw0lEi2P1fjO6Ie0qbc= +cloud.google.com/go/edgecontainer v1.0.0/go.mod h1:cttArqZpBB2q58W/upSG++ooo6EsblxDIolxa3jSjbY= cloud.google.com/go/errorreporting v0.3.0/go.mod h1:xsP2yaAp+OAW4OIm60An2bbLpqIhKXdWR/tawvl7QzU= cloud.google.com/go/essentialcontacts v1.3.0/go.mod h1:r+OnHa5jfj90qIfZDO/VztSFqbQan7HV75p8sA+mdGI= cloud.google.com/go/essentialcontacts v1.4.0/go.mod h1:8tRldvHYsmnBCHdFpvU+GL75oWiBKl80BiqlFh9tp+8= +cloud.google.com/go/essentialcontacts v1.5.0/go.mod h1:ay29Z4zODTuwliK7SnX8E86aUF2CTzdNtvv42niCX0M= cloud.google.com/go/eventarc v1.7.0/go.mod h1:6ctpF3zTnaQCxUjHUdcfgcA1A2T309+omHZth7gDfmc= cloud.google.com/go/eventarc v1.8.0/go.mod h1:imbzxkyAU4ubfsaKYdQg04WS1NvncblHEup4kvF+4gw= +cloud.google.com/go/eventarc v1.10.0/go.mod h1:u3R35tmZ9HvswGRBnF48IlYgYeBcPUCjkr4BTdem2Kw= +cloud.google.com/go/eventarc v1.11.0/go.mod h1:PyUjsUKPWoRBCHeOxZd/lbOOjahV41icXyUY5kSTvVY= cloud.google.com/go/filestore v1.3.0/go.mod h1:+qbvHGvXU1HaKX2nD0WEPo92TP/8AQuCVEBXNY9z0+w= cloud.google.com/go/filestore v1.4.0/go.mod h1:PaG5oDfo9r224f8OYXURtAsY+Fbyq/bLYoINEK8XQAI= +cloud.google.com/go/filestore v1.5.0/go.mod h1:FqBXDWBp4YLHqRnVGveOkHDf8svj9r5+mUDLupOWEDs= +cloud.google.com/go/filestore v1.6.0/go.mod h1:di5unNuss/qfZTw2U9nhFqo8/ZDSc466dre85Kydllg= cloud.google.com/go/firestore v1.9.0/go.mod h1:HMkjKHNTtRyZNiMzu7YAsLr9K3X2udY2AMwDaMEQiiE= cloud.google.com/go/functions v1.6.0/go.mod h1:3H1UA3qiIPRWD7PeZKLvHZ9SaQhR26XIJcC0A5GbvAk= cloud.google.com/go/functions v1.7.0/go.mod h1:+d+QBcWM+RsrgZfV9xo6KfA1GlzJfxcfZcRPEhDDfzg= cloud.google.com/go/functions v1.8.0/go.mod h1:RTZ4/HsQjIqIYP9a9YPbU+QFoQsAlYgrwOXJWHn1POY= cloud.google.com/go/functions v1.9.0/go.mod h1:Y+Dz8yGguzO3PpIjhLTbnqV1CWmgQ5UwtlpzoyquQ08= +cloud.google.com/go/functions v1.10.0/go.mod h1:0D3hEOe3DbEvCXtYOZHQZmD+SzYsi1YbI7dGvHfldXw= +cloud.google.com/go/functions v1.12.0/go.mod h1:AXWGrF3e2C/5ehvwYo/GH6O5s09tOPksiKhz+hH8WkA= +cloud.google.com/go/functions v1.13.0/go.mod h1:EU4O007sQm6Ef/PwRsI8N2umygGqPBS/IZQKBQBcJ3c= cloud.google.com/go/gaming v1.5.0/go.mod h1:ol7rGcxP/qHTRQE/RO4bxkXq+Fix0j6D4LFPzYTIrDM= cloud.google.com/go/gaming v1.6.0/go.mod h1:YMU1GEvA39Qt3zWGyAVA9bpYz/yAhTvaQ1t2sK4KPUA= cloud.google.com/go/gaming v1.7.0/go.mod h1:LrB8U7MHdGgFG851iHAfqUdLcKBdQ55hzXy9xBJz0+w= cloud.google.com/go/gaming v1.8.0/go.mod h1:xAqjS8b7jAVW0KFYeRUxngo9My3f33kFmua++Pi+ggM= +cloud.google.com/go/gaming v1.9.0/go.mod h1:Fc7kEmCObylSWLO334NcO+O9QMDyz+TKC4v1D7X+Bc0= cloud.google.com/go/gkebackup v0.2.0/go.mod h1:XKvv/4LfG829/B8B7xRkk8zRrOEbKtEam6yNfuQNH60= cloud.google.com/go/gkebackup v0.3.0/go.mod h1:n/E671i1aOQvUxT541aTkCwExO/bTer2HDlj4TsBRAo= +cloud.google.com/go/gkebackup v0.4.0/go.mod h1:byAyBGUwYGEEww7xsbnUTBHIYcOPy/PgUWUtOeRm9Vg= cloud.google.com/go/gkeconnect v0.5.0/go.mod h1:c5lsNAg5EwAy7fkqX/+goqFsU1Da/jQFqArp+wGNr/o= cloud.google.com/go/gkeconnect v0.6.0/go.mod h1:Mln67KyU/sHJEBY8kFZ0xTeyPtzbq9StAVvEULYK16A= +cloud.google.com/go/gkeconnect v0.7.0/go.mod h1:SNfmVqPkaEi3bF/B3CNZOAYPYdg7sU+obZ+QTky2Myw= cloud.google.com/go/gkehub v0.9.0/go.mod h1:WYHN6WG8w9bXU0hqNxt8rm5uxnk8IH+lPY9J2TV7BK0= cloud.google.com/go/gkehub v0.10.0/go.mod h1:UIPwxI0DsrpsVoWpLB0stwKCP+WFVG9+y977wO+hBH0= +cloud.google.com/go/gkehub v0.11.0/go.mod h1:JOWHlmN+GHyIbuWQPl47/C2RFhnFKH38jH9Ascu3n0E= +cloud.google.com/go/gkehub v0.12.0/go.mod h1:djiIwwzTTBrF5NaXCGv3mf7klpEMcST17VBTVVDcuaw= cloud.google.com/go/gkemulticloud v0.3.0/go.mod h1:7orzy7O0S+5kq95e4Hpn7RysVA7dPs8W/GgfUtsPbrA= cloud.google.com/go/gkemulticloud v0.4.0/go.mod h1:E9gxVBnseLWCk24ch+P9+B2CoDFJZTyIgLKSalC7tuI= +cloud.google.com/go/gkemulticloud v0.5.0/go.mod h1:W0JDkiyi3Tqh0TJr//y19wyb1yf8llHVto2Htf2Ja3Y= cloud.google.com/go/grafeas v0.2.0/go.mod h1:KhxgtF2hb0P191HlY5besjYm6MqTSTj3LSI+M+ByZHc= cloud.google.com/go/gsuiteaddons v1.3.0/go.mod h1:EUNK/J1lZEZO8yPtykKxLXI6JSVN2rg9bN8SXOa0bgM= cloud.google.com/go/gsuiteaddons v1.4.0/go.mod h1:rZK5I8hht7u7HxFQcFei0+AtfS9uSushomRlg+3ua1o= +cloud.google.com/go/gsuiteaddons v1.5.0/go.mod h1:TFCClYLd64Eaa12sFVmUyG62tk4mdIsI7pAnSXRkcFo= cloud.google.com/go/iam v0.1.0/go.mod h1:vcUNEa0pEm0qRVpmWepWaFMIAI8/hjB9mO8rNCJtF6c= cloud.google.com/go/iam v0.3.0/go.mod h1:XzJPvDayI+9zsASAFO68Hk07u3z+f+JrT2xXNdp4bnY= cloud.google.com/go/iam v0.5.0/go.mod h1:wPU9Vt0P4UmCux7mqtRu6jcpPAb74cP1fh50J3QpkUc= cloud.google.com/go/iam v0.6.0/go.mod h1:+1AH33ueBne5MzYccyMHtEKqLE4/kJOibtffMHDMFMc= cloud.google.com/go/iam v0.7.0/go.mod h1:H5Br8wRaDGNc8XP3keLc4unfUUZeyH3Sfl9XpQEYOeg= +cloud.google.com/go/iam v0.8.0/go.mod h1:lga0/y3iH6CX7sYqypWJ33hf7kkfXJag67naqGESjkE= +cloud.google.com/go/iam v0.11.0/go.mod h1:9PiLDanza5D+oWFZiH1uG+RnRCfEGKoyl6yo4cgWZGY= +cloud.google.com/go/iam v0.12.0/go.mod h1:knyHGviacl11zrtZUoDuYpDgLjvr28sLQaG0YB2GYAY= +cloud.google.com/go/iam v0.13.0/go.mod h1:ljOg+rcNfzZ5d6f1nAUJ8ZIxOaZUVoS14bKCtaLZ/D0= cloud.google.com/go/iap v1.4.0/go.mod h1:RGFwRJdihTINIe4wZ2iCP0zF/qu18ZwyKxrhMhygBEc= cloud.google.com/go/iap v1.5.0/go.mod h1:UH/CGgKd4KyohZL5Pt0jSKE4m3FR51qg6FKQ/z/Ix9A= +cloud.google.com/go/iap v1.6.0/go.mod h1:NSuvI9C/j7UdjGjIde7t7HBz+QTwBcapPE07+sSRcLk= +cloud.google.com/go/iap v1.7.0/go.mod h1:beqQx56T9O1G1yNPph+spKpNibDlYIiIixiqsQXxLIo= +cloud.google.com/go/iap v1.7.1/go.mod h1:WapEwPc7ZxGt2jFGB/C/bm+hP0Y6NXzOYGjpPnmMS74= cloud.google.com/go/ids v1.1.0/go.mod h1:WIuwCaYVOzHIj2OhN9HAwvW+DBdmUAdcWlFxRl+KubM= cloud.google.com/go/ids v1.2.0/go.mod h1:5WXvp4n25S0rA/mQWAg1YEEBBq6/s+7ml1RDCW1IrcY= +cloud.google.com/go/ids v1.3.0/go.mod h1:JBdTYwANikFKaDP6LtW5JAi4gubs57SVNQjemdt6xV4= cloud.google.com/go/iot v1.3.0/go.mod h1:r7RGh2B61+B8oz0AGE+J72AhA0G7tdXItODWsaA2oLs= cloud.google.com/go/iot v1.4.0/go.mod h1:dIDxPOn0UvNDUMD8Ger7FIaTuvMkj+aGk94RPP0iV+g= +cloud.google.com/go/iot v1.5.0/go.mod h1:mpz5259PDl3XJthEmh9+ap0affn/MqNSP4My77Qql9o= +cloud.google.com/go/iot v1.6.0/go.mod h1:IqdAsmE2cTYYNO1Fvjfzo9po179rAtJeVGUvkLN3rLE= cloud.google.com/go/kms v1.4.0/go.mod h1:fajBHndQ+6ubNw6Ss2sSd+SWvjL26RNo/dr7uxsnnOA= cloud.google.com/go/kms v1.5.0/go.mod h1:QJS2YY0eJGBg3mnDfuaCyLauWwBJiHRboYxJ++1xJNg= cloud.google.com/go/kms v1.6.0/go.mod h1:Jjy850yySiasBUDi6KFUwUv2n1+o7QZFyuUJg6OgjA0= +cloud.google.com/go/kms v1.8.0/go.mod h1:4xFEhYFqvW+4VMELtZyxomGSYtSQKzM178ylFW4jMAg= +cloud.google.com/go/kms v1.9.0/go.mod h1:qb1tPTgfF9RQP8e1wq4cLFErVuTJv7UsSC915J8dh3w= +cloud.google.com/go/kms v1.10.0/go.mod h1:ng3KTUtQQU9bPX3+QGLsflZIHlkbn8amFAMY63m8d24= +cloud.google.com/go/kms v1.10.1/go.mod h1:rIWk/TryCkR59GMC3YtHtXeLzd634lBbKenvyySAyYI= cloud.google.com/go/language v1.4.0/go.mod h1:F9dRpNFQmJbkaop6g0JhSBXCNlO90e1KWx5iDdxbWic= cloud.google.com/go/language v1.6.0/go.mod h1:6dJ8t3B+lUYfStgls25GusK04NLh3eDLQnWM3mdEbhI= cloud.google.com/go/language v1.7.0/go.mod h1:DJ6dYN/W+SQOjF8e1hLQXMF21AkH2w9wiPzPCJa2MIE= cloud.google.com/go/language v1.8.0/go.mod h1:qYPVHf7SPoNNiCL2Dr0FfEFNil1qi3pQEyygwpgVKB8= +cloud.google.com/go/language v1.9.0/go.mod h1:Ns15WooPM5Ad/5no/0n81yUetis74g3zrbeJBE+ptUY= cloud.google.com/go/lifesciences v0.5.0/go.mod h1:3oIKy8ycWGPUyZDR/8RNnTOYevhaMLqh5vLUXs9zvT8= cloud.google.com/go/lifesciences v0.6.0/go.mod h1:ddj6tSX/7BOnhxCSd3ZcETvtNr8NZ6t/iPhY2Tyfu08= +cloud.google.com/go/lifesciences v0.8.0/go.mod h1:lFxiEOMqII6XggGbOnKiyZ7IBwoIqA84ClvoezaA/bo= cloud.google.com/go/logging v1.6.1/go.mod h1:5ZO0mHHbvm8gEmeEUHrmDlTDSu5imF6MUP9OfilNXBw= +cloud.google.com/go/logging v1.7.0/go.mod h1:3xjP2CjkM3ZkO73aj4ASA5wRPGGCRrPIAeNqVNkzY8M= cloud.google.com/go/longrunning v0.1.1/go.mod h1:UUFxuDWkv22EuY93jjmDMFT5GPQKeFVJBIF6QlTqdsE= cloud.google.com/go/longrunning v0.3.0/go.mod h1:qth9Y41RRSUE69rDcOn6DdK3HfQfsUI0YSmW3iIlLJc= +cloud.google.com/go/longrunning v0.4.1/go.mod h1:4iWDqhBZ70CvZ6BfETbvam3T8FMvLK+eFj0E6AaRQTo= cloud.google.com/go/managedidentities v1.3.0/go.mod h1:UzlW3cBOiPrzucO5qWkNkh0w33KFtBJU281hacNvsdE= cloud.google.com/go/managedidentities v1.4.0/go.mod h1:NWSBYbEMgqmbZsLIyKvxrYbtqOsxY1ZrGM+9RgDqInM= +cloud.google.com/go/managedidentities v1.5.0/go.mod h1:+dWcZ0JlUmpuxpIDfyP5pP5y0bLdRwOS4Lp7gMni/LA= +cloud.google.com/go/maps v0.1.0/go.mod h1:BQM97WGyfw9FWEmQMpZ5T6cpovXXSd1cGmFma94eubI= +cloud.google.com/go/maps v0.6.0/go.mod h1:o6DAMMfb+aINHz/p/jbcY+mYeXBoZoxTfdSQ8VAJaCw= +cloud.google.com/go/maps v0.7.0/go.mod h1:3GnvVl3cqeSvgMcpRlQidXsPYuDGQ8naBis7MVzpXsY= cloud.google.com/go/mediatranslation v0.5.0/go.mod h1:jGPUhGTybqsPQn91pNXw0xVHfuJ3leR1wj37oU3y1f4= cloud.google.com/go/mediatranslation v0.6.0/go.mod h1:hHdBCTYNigsBxshbznuIMFNe5QXEowAuNmmC7h8pu5w= +cloud.google.com/go/mediatranslation v0.7.0/go.mod h1:LCnB/gZr90ONOIQLgSXagp8XUW1ODs2UmUMvcgMfI2I= cloud.google.com/go/memcache v1.4.0/go.mod h1:rTOfiGZtJX1AaFUrOgsMHX5kAzaTQ8azHiuDoTPzNsE= cloud.google.com/go/memcache v1.5.0/go.mod h1:dk3fCK7dVo0cUU2c36jKb4VqKPS22BTkf81Xq617aWM= cloud.google.com/go/memcache v1.6.0/go.mod h1:XS5xB0eQZdHtTuTF9Hf8eJkKtR3pVRCcvJwtm68T3rA= cloud.google.com/go/memcache v1.7.0/go.mod h1:ywMKfjWhNtkQTxrWxCkCFkoPjLHPW6A7WOTVI8xy3LY= +cloud.google.com/go/memcache v1.9.0/go.mod h1:8oEyzXCu+zo9RzlEaEjHl4KkgjlNDaXbCQeQWlzNFJM= cloud.google.com/go/metastore v1.5.0/go.mod h1:2ZNrDcQwghfdtCwJ33nM0+GrBGlVuh8rakL3vdPY3XY= cloud.google.com/go/metastore v1.6.0/go.mod h1:6cyQTls8CWXzk45G55x57DVQ9gWg7RiH65+YgPsNh9s= cloud.google.com/go/metastore v1.7.0/go.mod h1:s45D0B4IlsINu87/AsWiEVYbLaIMeUSoxlKKDqBGFS8= cloud.google.com/go/metastore v1.8.0/go.mod h1:zHiMc4ZUpBiM7twCIFQmJ9JMEkDSyZS9U12uf7wHqSI= +cloud.google.com/go/metastore v1.10.0/go.mod h1:fPEnH3g4JJAk+gMRnrAnoqyv2lpUCqJPWOodSaf45Eo= cloud.google.com/go/monitoring v1.7.0/go.mod h1:HpYse6kkGo//7p6sT0wsIC6IBDET0RhIsnmlA53dvEk= cloud.google.com/go/monitoring v1.8.0/go.mod h1:E7PtoMJ1kQXWxPjB6mv2fhC5/15jInuulFdYYtlcvT4= +cloud.google.com/go/monitoring v1.12.0/go.mod h1:yx8Jj2fZNEkL/GYZyTLS4ZtZEZN8WtDEiEqG4kLK50w= +cloud.google.com/go/monitoring v1.13.0/go.mod h1:k2yMBAB1H9JT/QETjNkgdCGD9bPF712XiLTVr+cBrpw= cloud.google.com/go/networkconnectivity v1.4.0/go.mod h1:nOl7YL8odKyAOtzNX73/M5/mGZgqqMeryi6UPZTk/rA= cloud.google.com/go/networkconnectivity v1.5.0/go.mod h1:3GzqJx7uhtlM3kln0+x5wyFvuVH1pIBJjhCpjzSt75o= cloud.google.com/go/networkconnectivity v1.6.0/go.mod h1:OJOoEXW+0LAxHh89nXd64uGG+FbQoeH8DtxCHVOMlaM= cloud.google.com/go/networkconnectivity v1.7.0/go.mod h1:RMuSbkdbPwNMQjB5HBWD5MpTBnNm39iAVpC3TmsExt8= +cloud.google.com/go/networkconnectivity v1.10.0/go.mod h1:UP4O4sWXJG13AqrTdQCD9TnLGEbtNRqjuaaA7bNjF5E= +cloud.google.com/go/networkconnectivity v1.11.0/go.mod h1:iWmDD4QF16VCDLXUqvyspJjIEtBR/4zq5hwnY2X3scM= cloud.google.com/go/networkmanagement v1.4.0/go.mod h1:Q9mdLLRn60AsOrPc8rs8iNV6OHXaGcDdsIQe1ohekq8= cloud.google.com/go/networkmanagement v1.5.0/go.mod h1:ZnOeZ/evzUdUsnvRt792H0uYEnHQEMaz+REhhzJRcf4= +cloud.google.com/go/networkmanagement v1.6.0/go.mod h1:5pKPqyXjB/sgtvB5xqOemumoQNB7y95Q7S+4rjSOPYY= cloud.google.com/go/networksecurity v0.5.0/go.mod h1:xS6fOCoqpVC5zx15Z/MqkfDwH4+m/61A3ODiDV1xmiQ= cloud.google.com/go/networksecurity v0.6.0/go.mod h1:Q5fjhTr9WMI5mbpRYEbiexTzROf7ZbDzvzCrNl14nyU= +cloud.google.com/go/networksecurity v0.7.0/go.mod h1:mAnzoxx/8TBSyXEeESMy9OOYwo1v+gZ5eMRnsT5bC8k= +cloud.google.com/go/networksecurity v0.8.0/go.mod h1:B78DkqsxFG5zRSVuwYFRZ9Xz8IcQ5iECsNrPn74hKHU= cloud.google.com/go/notebooks v1.2.0/go.mod h1:9+wtppMfVPUeJ8fIWPOq1UnATHISkGXGqTkxeieQ6UY= cloud.google.com/go/notebooks v1.3.0/go.mod h1:bFR5lj07DtCPC7YAAJ//vHskFBxA5JzYlH68kXVdk34= cloud.google.com/go/notebooks v1.4.0/go.mod h1:4QPMngcwmgb6uw7Po99B2xv5ufVoIQ7nOGDyL4P8AgA= cloud.google.com/go/notebooks v1.5.0/go.mod h1:q8mwhnP9aR8Hpfnrc5iN5IBhrXUy8S2vuYs+kBJ/gu0= +cloud.google.com/go/notebooks v1.7.0/go.mod h1:PVlaDGfJgj1fl1S3dUwhFMXFgfYGhYQt2164xOMONmE= +cloud.google.com/go/notebooks v1.8.0/go.mod h1:Lq6dYKOYOWUCTvw5t2q1gp1lAp0zxAxRycayS0iJcqQ= cloud.google.com/go/optimization v1.1.0/go.mod h1:5po+wfvX5AQlPznyVEZjGJTMr4+CAkJf2XSTQOOl9l4= cloud.google.com/go/optimization v1.2.0/go.mod h1:Lr7SOHdRDENsh+WXVmQhQTrzdu9ybg0NecjHidBq6xs= +cloud.google.com/go/optimization v1.3.1/go.mod h1:IvUSefKiwd1a5p0RgHDbWCIbDFgKuEdB+fPPuP0IDLI= cloud.google.com/go/orchestration v1.3.0/go.mod h1:Sj5tq/JpWiB//X/q3Ngwdl5K7B7Y0KZ7bfv0wL6fqVA= cloud.google.com/go/orchestration v1.4.0/go.mod h1:6W5NLFWs2TlniBphAViZEVhrXRSMgUGDfW7vrWKvsBk= +cloud.google.com/go/orchestration v1.6.0/go.mod h1:M62Bevp7pkxStDfFfTuCOaXgaaqRAga1yKyoMtEoWPQ= cloud.google.com/go/orgpolicy v1.4.0/go.mod h1:xrSLIV4RePWmP9P3tBl8S93lTmlAxjm06NSm2UTmKvE= cloud.google.com/go/orgpolicy v1.5.0/go.mod h1:hZEc5q3wzwXJaKrsx5+Ewg0u1LxJ51nNFlext7Tanwc= +cloud.google.com/go/orgpolicy v1.10.0/go.mod h1:w1fo8b7rRqlXlIJbVhOMPrwVljyuW5mqssvBtU18ONc= cloud.google.com/go/osconfig v1.7.0/go.mod h1:oVHeCeZELfJP7XLxcBGTMBvRO+1nQ5tFG9VQTmYS2Fs= cloud.google.com/go/osconfig v1.8.0/go.mod h1:EQqZLu5w5XA7eKizepumcvWx+m8mJUhEwiPqWiZeEdg= cloud.google.com/go/osconfig v1.9.0/go.mod h1:Yx+IeIZJ3bdWmzbQU4fxNl8xsZ4amB+dygAwFPlvnNo= cloud.google.com/go/osconfig v1.10.0/go.mod h1:uMhCzqC5I8zfD9zDEAfvgVhDS8oIjySWh+l4WK6GnWw= +cloud.google.com/go/osconfig v1.11.0/go.mod h1:aDICxrur2ogRd9zY5ytBLV89KEgT2MKB2L/n6x1ooPw= cloud.google.com/go/oslogin v1.4.0/go.mod h1:YdgMXWRaElXz/lDk1Na6Fh5orF7gvmJ0FGLIs9LId4E= cloud.google.com/go/oslogin v1.5.0/go.mod h1:D260Qj11W2qx/HVF29zBg+0fd6YCSjSqLUkY/qEenQU= cloud.google.com/go/oslogin v1.6.0/go.mod h1:zOJ1O3+dTU8WPlGEkFSh7qeHPPSoxrcMbbK1Nm2iX70= cloud.google.com/go/oslogin v1.7.0/go.mod h1:e04SN0xO1UNJ1M5GP0vzVBFicIe4O53FOfcixIqTyXo= +cloud.google.com/go/oslogin v1.9.0/go.mod h1:HNavntnH8nzrn8JCTT5fj18FuJLFJc4NaZJtBnQtKFs= cloud.google.com/go/phishingprotection v0.5.0/go.mod h1:Y3HZknsK9bc9dMi+oE8Bim0lczMU6hrX0UpADuMefr0= cloud.google.com/go/phishingprotection v0.6.0/go.mod h1:9Y3LBLgy0kDTcYET8ZH3bq/7qni15yVUoAxiFxnlSUA= +cloud.google.com/go/phishingprotection v0.7.0/go.mod h1:8qJI4QKHoda/sb/7/YmMQ2omRLSLYSu9bU0EKCNI+Lk= cloud.google.com/go/policytroubleshooter v1.3.0/go.mod h1:qy0+VwANja+kKrjlQuOzmlvscn4RNsAc0e15GGqfMxg= cloud.google.com/go/policytroubleshooter v1.4.0/go.mod h1:DZT4BcRw3QoO8ota9xw/LKtPa8lKeCByYeKTIf/vxdE= +cloud.google.com/go/policytroubleshooter v1.5.0/go.mod h1:Rz1WfV+1oIpPdN2VvvuboLVRsB1Hclg3CKQ53j9l8vw= +cloud.google.com/go/policytroubleshooter v1.6.0/go.mod h1:zYqaPTsmfvpjm5ULxAyD/lINQxJ0DDsnWOP/GZ7xzBc= cloud.google.com/go/privatecatalog v0.5.0/go.mod h1:XgosMUvvPyxDjAVNDYxJ7wBW8//hLDDYmnsNcMGq1K0= cloud.google.com/go/privatecatalog v0.6.0/go.mod h1:i/fbkZR0hLN29eEWiiwue8Pb+GforiEIBnV9yrRUOKI= +cloud.google.com/go/privatecatalog v0.7.0/go.mod h1:2s5ssIFO69F5csTXcwBP7NPFTZvps26xGzvQ2PQaBYg= +cloud.google.com/go/privatecatalog v0.8.0/go.mod h1:nQ6pfaegeDAq/Q5lrfCQzQLhubPiZhSaNhIgfJlnIXs= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= cloud.google.com/go/pubsub v1.26.0/go.mod h1:QgBH3U/jdJy/ftjPhTkyXNj543Tin1pRYcdcPRnFIRI= cloud.google.com/go/pubsub v1.27.1/go.mod h1:hQN39ymbV9geqBnfQq6Xf63yNhUAhv9CZhzp5O6qsW0= +cloud.google.com/go/pubsub v1.28.0/go.mod h1:vuXFpwaVoIPQMGXqRyUQigu/AX1S3IWugR9xznmcXX8= +cloud.google.com/go/pubsub v1.30.0/go.mod h1:qWi1OPS0B+b5L+Sg6Gmc9zD1Y+HaM0MdUr7LsupY1P4= cloud.google.com/go/pubsublite v1.5.0/go.mod h1:xapqNQ1CuLfGi23Yda/9l4bBCKz/wC3KIJ5gKcxveZg= +cloud.google.com/go/pubsublite v1.6.0/go.mod h1:1eFCS0U11xlOuMFV/0iBqw3zP12kddMeCbj/F3FSj9k= +cloud.google.com/go/pubsublite v1.7.0/go.mod h1:8hVMwRXfDfvGm3fahVbtDbiLePT3gpoiJYJY+vxWxVM= cloud.google.com/go/recaptchaenterprise v1.3.1/go.mod h1:OdD+q+y4XGeAlxRaMn1Y7/GveP6zmq76byL6tjPE7d4= cloud.google.com/go/recaptchaenterprise/v2 v2.1.0/go.mod h1:w9yVqajwroDNTfGuhmOjPDN//rZGySaf6PtFVcSCa7o= cloud.google.com/go/recaptchaenterprise/v2 v2.2.0/go.mod h1:/Zu5jisWGeERrd5HnlS3EUGb/D335f9k51B/FVil0jk= cloud.google.com/go/recaptchaenterprise/v2 v2.3.0/go.mod h1:O9LwGCjrhGHBQET5CA7dd5NwwNQUErSgEDit1DLNTdo= cloud.google.com/go/recaptchaenterprise/v2 v2.4.0/go.mod h1:Am3LHfOuBstrLrNCBrlI5sbwx9LBg3te2N6hGvHn2mE= cloud.google.com/go/recaptchaenterprise/v2 v2.5.0/go.mod h1:O8LzcHXN3rz0j+LBC91jrwI3R+1ZSZEWrfL7XHgNo9U= +cloud.google.com/go/recaptchaenterprise/v2 v2.6.0/go.mod h1:RPauz9jeLtB3JVzg6nCbe12qNoaa8pXc4d/YukAmcnA= +cloud.google.com/go/recaptchaenterprise/v2 v2.7.0/go.mod h1:19wVj/fs5RtYtynAPJdDTb69oW0vNHYDBTbB4NvMD9c= cloud.google.com/go/recommendationengine v0.5.0/go.mod h1:E5756pJcVFeVgaQv3WNpImkFP8a+RptV6dDLGPILjvg= cloud.google.com/go/recommendationengine v0.6.0/go.mod h1:08mq2umu9oIqc7tDy8sx+MNJdLG0fUi3vaSVbztHgJ4= +cloud.google.com/go/recommendationengine v0.7.0/go.mod h1:1reUcE3GIu6MeBz/h5xZJqNLuuVjNg1lmWMPyjatzac= cloud.google.com/go/recommender v1.5.0/go.mod h1:jdoeiBIVrJe9gQjwd759ecLJbxCDED4A6p+mqoqDvTg= cloud.google.com/go/recommender v1.6.0/go.mod h1:+yETpm25mcoiECKh9DEScGzIRyDKpZ0cEhWGo+8bo+c= cloud.google.com/go/recommender v1.7.0/go.mod h1:XLHs/W+T8olwlGOgfQenXBTbIseGclClff6lhFVe9Bs= cloud.google.com/go/recommender v1.8.0/go.mod h1:PkjXrTT05BFKwxaUxQmtIlrtj0kph108r02ZZQ5FE70= +cloud.google.com/go/recommender v1.9.0/go.mod h1:PnSsnZY7q+VL1uax2JWkt/UegHssxjUVVCrX52CuEmQ= cloud.google.com/go/redis v1.7.0/go.mod h1:V3x5Jq1jzUcg+UNsRvdmsfuFnit1cfe3Z/PGyq/lm4Y= cloud.google.com/go/redis v1.8.0/go.mod h1:Fm2szCDavWzBk2cDKxrkmWBqoCiL1+Ctwq7EyqBCA/A= cloud.google.com/go/redis v1.9.0/go.mod h1:HMYQuajvb2D0LvMgZmLDZW8V5aOC/WxstZHiy4g8OiA= cloud.google.com/go/redis v1.10.0/go.mod h1:ThJf3mMBQtW18JzGgh41/Wld6vnDDc/F/F35UolRZPM= +cloud.google.com/go/redis v1.11.0/go.mod h1:/X6eicana+BWcUda5PpwZC48o37SiFVTFSs0fWAJ7uQ= cloud.google.com/go/resourcemanager v1.3.0/go.mod h1:bAtrTjZQFJkiWTPDb1WBjzvc6/kifjj4QBYuKCCoqKA= cloud.google.com/go/resourcemanager v1.4.0/go.mod h1:MwxuzkumyTX7/a3n37gmsT3py7LIXwrShilPh3P1tR0= +cloud.google.com/go/resourcemanager v1.5.0/go.mod h1:eQoXNAiAvCf5PXxWxXjhKQoTMaUSNrEfg+6qdf/wots= +cloud.google.com/go/resourcemanager v1.6.0/go.mod h1:YcpXGRs8fDzcUl1Xw8uOVmI8JEadvhRIkoXXUNVYcVo= +cloud.google.com/go/resourcemanager v1.7.0/go.mod h1:HlD3m6+bwhzj9XCouqmeiGuni95NTrExfhoSrkC/3EI= cloud.google.com/go/resourcesettings v1.3.0/go.mod h1:lzew8VfESA5DQ8gdlHwMrqZs1S9V87v3oCnKCWoOuQU= cloud.google.com/go/resourcesettings v1.4.0/go.mod h1:ldiH9IJpcrlC3VSuCGvjR5of/ezRrOxFtpJoJo5SmXg= +cloud.google.com/go/resourcesettings v1.5.0/go.mod h1:+xJF7QSG6undsQDfsCJyqWXyBwUoJLhetkRMDRnIoXA= cloud.google.com/go/retail v1.8.0/go.mod h1:QblKS8waDmNUhghY2TI9O3JLlFk8jybHeV4BF19FrE4= cloud.google.com/go/retail v1.9.0/go.mod h1:g6jb6mKuCS1QKnH/dpu7isX253absFl6iE92nHwlBUY= cloud.google.com/go/retail v1.10.0/go.mod h1:2gDk9HsL4HMS4oZwz6daui2/jmKvqShXKQuB2RZ+cCc= cloud.google.com/go/retail v1.11.0/go.mod h1:MBLk1NaWPmh6iVFSz9MeKG/Psyd7TAgm6y/9L2B4x9Y= +cloud.google.com/go/retail v1.12.0/go.mod h1:UMkelN/0Z8XvKymXFbD4EhFJlYKRx1FGhQkVPU5kF14= cloud.google.com/go/run v0.2.0/go.mod h1:CNtKsTA1sDcnqqIFR3Pb5Tq0usWxJJvsWOCPldRU3Do= cloud.google.com/go/run v0.3.0/go.mod h1:TuyY1+taHxTjrD0ZFk2iAR+xyOXEA0ztb7U3UNA0zBo= +cloud.google.com/go/run v0.8.0/go.mod h1:VniEnuBwqjigv0A7ONfQUaEItaiCRVujlMqerPPiktM= +cloud.google.com/go/run v0.9.0/go.mod h1:Wwu+/vvg8Y+JUApMwEDfVfhetv30hCG4ZwDR/IXl2Qg= cloud.google.com/go/scheduler v1.4.0/go.mod h1:drcJBmxF3aqZJRhmkHQ9b3uSSpQoltBPGPxGAWROx6s= cloud.google.com/go/scheduler v1.5.0/go.mod h1:ri073ym49NW3AfT6DZi21vLZrG07GXr5p3H1KxN5QlI= cloud.google.com/go/scheduler v1.6.0/go.mod h1:SgeKVM7MIwPn3BqtcBntpLyrIJftQISRrYB5ZtT+KOk= cloud.google.com/go/scheduler v1.7.0/go.mod h1:jyCiBqWW956uBjjPMMuX09n3x37mtyPJegEWKxRsn44= +cloud.google.com/go/scheduler v1.8.0/go.mod h1:TCET+Y5Gp1YgHT8py4nlg2Sew8nUHMqcpousDgXJVQc= +cloud.google.com/go/scheduler v1.9.0/go.mod h1:yexg5t+KSmqu+njTIh3b7oYPheFtBWGcbVUYF1GGMIc= cloud.google.com/go/secretmanager v1.6.0/go.mod h1:awVa/OXF6IiyaU1wQ34inzQNc4ISIDIrId8qE5QGgKA= cloud.google.com/go/secretmanager v1.8.0/go.mod h1:hnVgi/bN5MYHd3Gt0SPuTPPp5ENina1/LxM+2W9U9J4= cloud.google.com/go/secretmanager v1.9.0/go.mod h1:b71qH2l1yHmWQHt9LC80akm86mX8AL6X1MA01dW8ht4= +cloud.google.com/go/secretmanager v1.10.0/go.mod h1:MfnrdvKMPNra9aZtQFvBcvRU54hbPD8/HayQdlUgJpU= cloud.google.com/go/security v1.5.0/go.mod h1:lgxGdyOKKjHL4YG3/YwIL2zLqMFCKs0UbQwgyZmfJl4= cloud.google.com/go/security v1.7.0/go.mod h1:mZklORHl6Bg7CNnnjLH//0UlAlaXqiG7Lb9PsPXLfD0= cloud.google.com/go/security v1.8.0/go.mod h1:hAQOwgmaHhztFhiQ41CjDODdWP0+AE1B3sX4OFlq+GU= cloud.google.com/go/security v1.9.0/go.mod h1:6Ta1bO8LXI89nZnmnsZGp9lVoVWXqsVbIq/t9dzI+2Q= cloud.google.com/go/security v1.10.0/go.mod h1:QtOMZByJVlibUT2h9afNDWRZ1G96gVywH8T5GUSb9IA= +cloud.google.com/go/security v1.12.0/go.mod h1:rV6EhrpbNHrrxqlvW0BWAIawFWq3X90SduMJdFwtLB8= +cloud.google.com/go/security v1.13.0/go.mod h1:Q1Nvxl1PAgmeW0y3HTt54JYIvUdtcpYKVfIB8AOMZ+0= cloud.google.com/go/securitycenter v1.13.0/go.mod h1:cv5qNAqjY84FCN6Y9z28WlkKXyWsgLO832YiWwkCWcU= cloud.google.com/go/securitycenter v1.14.0/go.mod h1:gZLAhtyKv85n52XYWt6RmeBdydyxfPeTrpToDPw4Auc= cloud.google.com/go/securitycenter v1.15.0/go.mod h1:PeKJ0t8MoFmmXLXWm41JidyzI3PJjd8sXWaVqg43WWk= cloud.google.com/go/securitycenter v1.16.0/go.mod h1:Q9GMaLQFUD+5ZTabrbujNWLtSLZIZF7SAR0wWECrjdk= +cloud.google.com/go/securitycenter v1.18.1/go.mod h1:0/25gAzCM/9OL9vVx4ChPeM/+DlfGQJDwBy/UC8AKK0= +cloud.google.com/go/securitycenter v1.19.0/go.mod h1:LVLmSg8ZkkyaNy4u7HCIshAngSQ8EcIRREP3xBnyfag= cloud.google.com/go/servicecontrol v1.4.0/go.mod h1:o0hUSJ1TXJAmi/7fLJAedOovnujSEvjKCAFNXPQ1RaU= cloud.google.com/go/servicecontrol v1.5.0/go.mod h1:qM0CnXHhyqKVuiZnGKrIurvVImCs8gmqWsDoqe9sU1s= +cloud.google.com/go/servicecontrol v1.10.0/go.mod h1:pQvyvSRh7YzUF2efw7H87V92mxU8FnFDawMClGCNuAA= +cloud.google.com/go/servicecontrol v1.11.0/go.mod h1:kFmTzYzTUIuZs0ycVqRHNaNhgR+UMUpw9n02l/pY+mc= +cloud.google.com/go/servicecontrol v1.11.1/go.mod h1:aSnNNlwEFBY+PWGQ2DoM0JJ/QUXqV5/ZD9DOLB7SnUk= cloud.google.com/go/servicedirectory v1.4.0/go.mod h1:gH1MUaZCgtP7qQiI+F+A+OpeKF/HQWgtAddhTbhL2bs= cloud.google.com/go/servicedirectory v1.5.0/go.mod h1:QMKFL0NUySbpZJ1UZs3oFAmdvVxhhxB6eJ/Vlp73dfg= cloud.google.com/go/servicedirectory v1.6.0/go.mod h1:pUlbnWsLH9c13yGkxCmfumWEPjsRs1RlmJ4pqiNjVL4= cloud.google.com/go/servicedirectory v1.7.0/go.mod h1:5p/U5oyvgYGYejufvxhgwjL8UVXjkuw7q5XcG10wx1U= +cloud.google.com/go/servicedirectory v1.8.0/go.mod h1:srXodfhY1GFIPvltunswqXpVxFPpZjf8nkKQT7XcXaY= +cloud.google.com/go/servicedirectory v1.9.0/go.mod h1:29je5JjiygNYlmsGz8k6o+OZ8vd4f//bQLtvzkPPT/s= cloud.google.com/go/servicemanagement v1.4.0/go.mod h1:d8t8MDbezI7Z2R1O/wu8oTggo3BI2GKYbdG4y/SJTco= cloud.google.com/go/servicemanagement v1.5.0/go.mod h1:XGaCRe57kfqu4+lRxaFEAuqmjzF0r+gWHjWqKqBvKFo= +cloud.google.com/go/servicemanagement v1.6.0/go.mod h1:aWns7EeeCOtGEX4OvZUWCCJONRZeFKiptqKf1D0l/Jc= +cloud.google.com/go/servicemanagement v1.8.0/go.mod h1:MSS2TDlIEQD/fzsSGfCdJItQveu9NXnUniTrq/L8LK4= cloud.google.com/go/serviceusage v1.3.0/go.mod h1:Hya1cozXM4SeSKTAgGXgj97GlqUvF5JaoXacR1JTP/E= cloud.google.com/go/serviceusage v1.4.0/go.mod h1:SB4yxXSaYVuUBYUml6qklyONXNLt83U0Rb+CXyhjEeU= +cloud.google.com/go/serviceusage v1.5.0/go.mod h1:w8U1JvqUqwJNPEOTQjrMHkw3IaIFLoLsPLvsE3xueec= +cloud.google.com/go/serviceusage v1.6.0/go.mod h1:R5wwQcbOWsyuOfbP9tGdAnCAc6B9DRwPG1xtWMDeuPA= cloud.google.com/go/shell v1.3.0/go.mod h1:VZ9HmRjZBsjLGXusm7K5Q5lzzByZmJHf1d0IWHEN5X4= cloud.google.com/go/shell v1.4.0/go.mod h1:HDxPzZf3GkDdhExzD/gs8Grqk+dmYcEjGShZgYa9URw= +cloud.google.com/go/shell v1.6.0/go.mod h1:oHO8QACS90luWgxP3N9iZVuEiSF84zNyLytb+qE2f9A= cloud.google.com/go/spanner v1.41.0/go.mod h1:MLYDBJR/dY4Wt7ZaMIQ7rXOTLjYrmxLE/5ve9vFfWos= +cloud.google.com/go/spanner v1.44.0/go.mod h1:G8XIgYdOK+Fbcpbs7p2fiprDw4CaZX63whnSMLVBxjk= +cloud.google.com/go/spanner v1.45.0/go.mod h1:FIws5LowYz8YAE1J8fOS7DJup8ff7xJeetWEo5REA2M= cloud.google.com/go/speech v1.6.0/go.mod h1:79tcr4FHCimOp56lwC01xnt/WPJZc4v3gzyT7FoBkCM= cloud.google.com/go/speech v1.7.0/go.mod h1:KptqL+BAQIhMsj1kOP2la5DSEEerPDuOP/2mmkhHhZQ= cloud.google.com/go/speech v1.8.0/go.mod h1:9bYIl1/tjsAnMgKGHKmBZzXKEkGgtU+MpdDPTE9f7y0= cloud.google.com/go/speech v1.9.0/go.mod h1:xQ0jTcmnRFFM2RfX/U+rk6FQNUF6DQlydUSyoooSpco= +cloud.google.com/go/speech v1.14.1/go.mod h1:gEosVRPJ9waG7zqqnsHpYTOoAS4KouMRLDFMekpJ0J0= +cloud.google.com/go/speech v1.15.0/go.mod h1:y6oH7GhqCaZANH7+Oe0BhgIogsNInLlz542tg3VqeYI= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= cloud.google.com/go/storage v1.22.1/go.mod h1:S8N1cAStu7BOeFfE8KAQzmyyLkK8p/vmRq6kuBTW58Y= cloud.google.com/go/storage v1.23.0/go.mod h1:vOEEDNFnciUMhBeT6hsJIn3ieU5cFRmzeLgDvXzfIXc= cloud.google.com/go/storage v1.27.0/go.mod h1:x9DOL8TK/ygDUMieqwfhdpQryTeEkhGKMi80i/iqR2s= +cloud.google.com/go/storage v1.28.1/go.mod h1:Qnisd4CqDdo6BGs2AD5LLnEsmSQ80wQ5ogcBBKhU86Y= +cloud.google.com/go/storage v1.29.0/go.mod h1:4puEjyTKnku6gfKoTfNOU/W+a9JyuVNxjpS5GBrB8h4= cloud.google.com/go/storagetransfer v1.5.0/go.mod h1:dxNzUopWy7RQevYFHewchb29POFv3/AaBgnhqzqiK0w= cloud.google.com/go/storagetransfer v1.6.0/go.mod h1:y77xm4CQV/ZhFZH75PLEXY0ROiS7Gh6pSKrM8dJyg6I= +cloud.google.com/go/storagetransfer v1.7.0/go.mod h1:8Giuj1QNb1kfLAiWM1bN6dHzfdlDAVC9rv9abHot2W4= +cloud.google.com/go/storagetransfer v1.8.0/go.mod h1:JpegsHHU1eXg7lMHkvf+KE5XDJ7EQu0GwNJbbVGanEw= cloud.google.com/go/talent v1.1.0/go.mod h1:Vl4pt9jiHKvOgF9KoZo6Kob9oV4lwd/ZD5Cto54zDRw= cloud.google.com/go/talent v1.2.0/go.mod h1:MoNF9bhFQbiJ6eFD3uSsg0uBALw4n4gaCaEjBw9zo8g= cloud.google.com/go/talent v1.3.0/go.mod h1:CmcxwJ/PKfRgd1pBjQgU6W3YBwiewmUzQYH5HHmSCmM= cloud.google.com/go/talent v1.4.0/go.mod h1:ezFtAgVuRf8jRsvyE6EwmbTK5LKciD4KVnHuDEFmOOA= +cloud.google.com/go/talent v1.5.0/go.mod h1:G+ODMj9bsasAEJkQSzO2uHQWXHHXUomArjWQQYkqK6c= cloud.google.com/go/texttospeech v1.4.0/go.mod h1:FX8HQHA6sEpJ7rCMSfXuzBcysDAuWusNNNvN9FELDd8= cloud.google.com/go/texttospeech v1.5.0/go.mod h1:oKPLhR4n4ZdQqWKURdwxMy0uiTS1xU161C8W57Wkea4= +cloud.google.com/go/texttospeech v1.6.0/go.mod h1:YmwmFT8pj1aBblQOI3TfKmwibnsfvhIBzPXcW4EBovc= cloud.google.com/go/tpu v1.3.0/go.mod h1:aJIManG0o20tfDQlRIej44FcwGGl/cD0oiRyMKG19IQ= cloud.google.com/go/tpu v1.4.0/go.mod h1:mjZaX8p0VBgllCzF6wcU2ovUXN9TONFLd7iz227X2Xg= +cloud.google.com/go/tpu v1.5.0/go.mod h1:8zVo1rYDFuW2l4yZVY0R0fb/v44xLh3llq7RuV61fPM= cloud.google.com/go/trace v1.3.0/go.mod h1:FFUE83d9Ca57C+K8rDl/Ih8LwOzWIV1krKgxg6N0G28= cloud.google.com/go/trace v1.4.0/go.mod h1:UG0v8UBqzusp+z63o7FK74SdFE+AXpCLdFb1rshXG+Y= +cloud.google.com/go/trace v1.8.0/go.mod h1:zH7vcsbAhklH8hWFig58HvxcxyQbaIqMarMg9hn5ECA= +cloud.google.com/go/trace v1.9.0/go.mod h1:lOQqpE5IaWY0Ixg7/r2SjixMuc6lfTFeO4QGM4dQWOk= cloud.google.com/go/translate v1.3.0/go.mod h1:gzMUwRjvOqj5i69y/LYLd8RrNQk+hOmIXTi9+nb3Djs= cloud.google.com/go/translate v1.4.0/go.mod h1:06Dn/ppvLD6WvA5Rhdp029IX2Mi3Mn7fpMRLPvXT5Wg= +cloud.google.com/go/translate v1.5.0/go.mod h1:29YDSYveqqpA1CQFD7NQuP49xymq17RXNaUDdc0mNu0= +cloud.google.com/go/translate v1.6.0/go.mod h1:lMGRudH1pu7I3n3PETiOB2507gf3HnfLV8qlkHZEyos= +cloud.google.com/go/translate v1.7.0/go.mod h1:lMGRudH1pu7I3n3PETiOB2507gf3HnfLV8qlkHZEyos= cloud.google.com/go/video v1.8.0/go.mod h1:sTzKFc0bUSByE8Yoh8X0mn8bMymItVGPfTuUBUyRgxk= cloud.google.com/go/video v1.9.0/go.mod h1:0RhNKFRF5v92f8dQt0yhaHrEuH95m068JYOvLZYnJSw= +cloud.google.com/go/video v1.12.0/go.mod h1:MLQew95eTuaNDEGriQdcYn0dTwf9oWiA4uYebxM5kdg= +cloud.google.com/go/video v1.13.0/go.mod h1:ulzkYlYgCp15N2AokzKjy7MQ9ejuynOJdf1tR5lGthk= +cloud.google.com/go/video v1.14.0/go.mod h1:SkgaXwT+lIIAKqWAJfktHT/RbgjSuY6DobxEp0C5yTQ= +cloud.google.com/go/video v1.15.0/go.mod h1:SkgaXwT+lIIAKqWAJfktHT/RbgjSuY6DobxEp0C5yTQ= cloud.google.com/go/videointelligence v1.6.0/go.mod h1:w0DIDlVRKtwPCn/C4iwZIJdvC69yInhW0cfi+p546uU= cloud.google.com/go/videointelligence v1.7.0/go.mod h1:k8pI/1wAhjznARtVT9U1llUaFNPh7muw8QyOUpavru4= cloud.google.com/go/videointelligence v1.8.0/go.mod h1:dIcCn4gVDdS7yte/w+koiXn5dWVplOZkE+xwG9FgK+M= cloud.google.com/go/videointelligence v1.9.0/go.mod h1:29lVRMPDYHikk3v8EdPSaL8Ku+eMzDljjuvRs105XoU= +cloud.google.com/go/videointelligence v1.10.0/go.mod h1:LHZngX1liVtUhZvi2uNS0VQuOzNi2TkY1OakiuoUOjU= cloud.google.com/go/vision v1.2.0/go.mod h1:SmNwgObm5DpFBme2xpyOyasvBc1aPdjvMk2bBk0tKD0= cloud.google.com/go/vision/v2 v2.2.0/go.mod h1:uCdV4PpN1S0jyCyq8sIM42v2Y6zOLkZs+4R9LrGYwFo= cloud.google.com/go/vision/v2 v2.3.0/go.mod h1:UO61abBx9QRMFkNBbf1D8B1LXdS2cGiiCRx0vSpZoUo= cloud.google.com/go/vision/v2 v2.4.0/go.mod h1:VtI579ll9RpVTrdKdkMzckdnwMyX2JILb+MhPqRbPsY= cloud.google.com/go/vision/v2 v2.5.0/go.mod h1:MmaezXOOE+IWa+cS7OhRRLK2cNv1ZL98zhqFFZaaH2E= +cloud.google.com/go/vision/v2 v2.6.0/go.mod h1:158Hes0MvOS9Z/bDMSFpjwsUrZ5fPrdwuyyvKSGAGMY= +cloud.google.com/go/vision/v2 v2.7.0/go.mod h1:H89VysHy21avemp6xcf9b9JvZHVehWbET0uT/bcuY/0= cloud.google.com/go/vmmigration v1.2.0/go.mod h1:IRf0o7myyWFSmVR1ItrBSFLFD/rJkfDCUTO4vLlJvsE= cloud.google.com/go/vmmigration v1.3.0/go.mod h1:oGJ6ZgGPQOFdjHuocGcLqX4lc98YQ7Ygq8YQwHh9A7g= +cloud.google.com/go/vmmigration v1.5.0/go.mod h1:E4YQ8q7/4W9gobHjQg4JJSgXXSgY21nA5r8swQV+Xxc= +cloud.google.com/go/vmmigration v1.6.0/go.mod h1:bopQ/g4z+8qXzichC7GW1w2MjbErL54rk3/C843CjfY= +cloud.google.com/go/vmwareengine v0.1.0/go.mod h1:RsdNEf/8UDvKllXhMz5J40XxDrNJNN4sagiox+OI208= +cloud.google.com/go/vmwareengine v0.2.2/go.mod h1:sKdctNJxb3KLZkE/6Oui94iw/xs9PRNC2wnNLXsHvH8= +cloud.google.com/go/vmwareengine v0.3.0/go.mod h1:wvoyMvNWdIzxMYSpH/R7y2h5h3WFkx6d+1TIsP39WGY= cloud.google.com/go/vpcaccess v1.4.0/go.mod h1:aQHVbTWDYUR1EbTApSVvMq1EnT57ppDmQzZ3imqIk4w= cloud.google.com/go/vpcaccess v1.5.0/go.mod h1:drmg4HLk9NkZpGfCmZ3Tz0Bwnm2+DKqViEpeEpOq0m8= +cloud.google.com/go/vpcaccess v1.6.0/go.mod h1:wX2ILaNhe7TlVa4vC5xce1bCnqE3AeH27RV31lnmZes= cloud.google.com/go/webrisk v1.4.0/go.mod h1:Hn8X6Zr+ziE2aNd8SliSDWpEnSS1u4R9+xXZmFiHmGE= cloud.google.com/go/webrisk v1.5.0/go.mod h1:iPG6fr52Tv7sGk0H6qUFzmL3HHZev1htXuWDEEsqMTg= cloud.google.com/go/webrisk v1.6.0/go.mod h1:65sW9V9rOosnc9ZY7A7jsy1zoHS5W9IAXv6dGqhMQMc= cloud.google.com/go/webrisk v1.7.0/go.mod h1:mVMHgEYH0r337nmt1JyLthzMr6YxwN1aAIEc2fTcq7A= +cloud.google.com/go/webrisk v1.8.0/go.mod h1:oJPDuamzHXgUc+b8SiHRcVInZQuybnvEW72PqTc7sSg= cloud.google.com/go/websecurityscanner v1.3.0/go.mod h1:uImdKm2wyeXQevQJXeh8Uun/Ym1VqworNDlBXQevGMo= cloud.google.com/go/websecurityscanner v1.4.0/go.mod h1:ebit/Fp0a+FWu5j4JOmJEV8S8CzdTkAS77oDsiSqYWQ= +cloud.google.com/go/websecurityscanner v1.5.0/go.mod h1:Y6xdCPy81yi0SQnDY1xdNTNpfY1oAgXUlcfN3B3eSng= cloud.google.com/go/workflows v1.6.0/go.mod h1:6t9F5h/unJz41YqfBmqSASJSXccBLtD1Vwf+KmJENM0= cloud.google.com/go/workflows v1.7.0/go.mod h1:JhSrZuVZWuiDfKEFxU0/F1PQjmpnpcoISEXH2bcHC3M= cloud.google.com/go/workflows v1.8.0/go.mod h1:ysGhmEajwZxGn1OhGOGKsTXc5PyxOc0vfKf5Af+to4M= cloud.google.com/go/workflows v1.9.0/go.mod h1:ZGkj1aFIOd9c8Gerkjjq7OW7I5+l6cSvT3ujaO/WwSA= +cloud.google.com/go/workflows v1.10.0/go.mod h1:fZ8LmRmZQWacon9UCX1r/g/DfAXx5VcPALq2CxzdePw= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= +gioui.org v0.0.0-20210308172011-57750fc8a0a6/go.mod h1:RSH6KIUZ0p2xy5zHDxgAM4zumjgTw83q2ge/PI+yyw8= +git.sr.ht/~sbinet/gg v0.3.1/go.mod h1:KGYtlADtqsqANL9ueOFkWymvzUvLMQllU5Ixo+8v3pc= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c/go.mod h1:X0CRv0ky0k6m906ixxpzmDRLvX58TFUKS2eePweuyxk= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/ajstarks/deck v0.0.0-20200831202436-30c9fc6549a9/go.mod h1:JynElWSGnm/4RlzPXRlREEwqTHAN3T56Bv2ITsFT3gY= +github.com/ajstarks/deck/generate v0.0.0-20210309230005-c3f852c02e19/go.mod h1:T13YZdzov6OU0A1+RfKZiZN9ca6VeKdBdyDV+BY97Tk= +github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= +github.com/ajstarks/svgo v0.0.0-20211024235047-1546f124cd8b/go.mod h1:1KcenG0jGWcpt8ov532z81sp/kMMUG485J2InIOyADM= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= +github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= -github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= +github.com/apache/arrow/go/v10 v10.0.1/go.mod h1:YvhnlEePVnBS4+0z3fhPfUy7W1Ikj0Ih0vcRo/gZ1M0= +github.com/apache/arrow/go/v11 v11.0.0/go.mod h1:Eg5OsL5H+e299f7u5ssuXsuHQVEGC4xei5aX110hRiI= +github.com/apache/thrift v0.16.0/go.mod h1:PHK3hniurgQaNMZYaCLEqXKsYK8upmhPbmdP2FXSqgU= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= +github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= +github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/boombuler/barcode v1.0.0/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= +github.com/boombuler/barcode v1.0.1/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/census-instrumentation/opencensus-proto v0.4.1/go.mod h1:4T9NM4+4Vw91VeyqjLS6ao50K5bOcLKN6Q42XnYaRYw= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= -github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cloudfoundry/gosigar v1.3.6/go.mod h1:lNWstu5g5gw59O09Y+wsMNFzBSnU8a0u+Sfx4dq360E= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= +github.com/cncf/udpa/go v0.0.0-20220112060539-c52dc94e7fbe/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20220314180256-7f1daf1720fc/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20230105202645-06c439db220b/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/elastic/gosigar v0.14.2/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs= +github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -422,22 +661,44 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.m github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= +github.com/envoyproxy/go-control-plane v0.10.3/go.mod h1:fJJn/j26vwOu972OllsvAgJJM//w9BV6Fxbg2LuVd34= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/envoyproxy/protoc-gen-validate v0.6.7/go.mod h1:dyJXwwfPK2VSqiB9Klm1J6romD608Ba7Hij42vrOBCo= +github.com/envoyproxy/protoc-gen-validate v0.9.1/go.mod h1:OKNgG7TCp5pF4d6XftA0++PMirau2/yoOwVac3AbF2w= +github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= +github.com/fogleman/gg v1.3.0/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-fonts/dejavu v0.1.0/go.mod h1:4Wt4I4OU2Nq9asgDCteaAaWZOV24E+0/Pwo0gppep4g= +github.com/go-fonts/latin-modern v0.2.0/go.mod h1:rQVLdDMK+mK1xscDwsqM5J8U2jrRa3T0ecnM9pNujks= +github.com/go-fonts/liberation v0.1.1/go.mod h1:K6qoJYypsmfVjWg8KOVDQhLc8UDgIK2HYqyqAO9z7GY= +github.com/go-fonts/liberation v0.2.0/go.mod h1:K6qoJYypsmfVjWg8KOVDQhLc8UDgIK2HYqyqAO9z7GY= +github.com/go-fonts/stix v0.1.0/go.mod h1:w/c1f0ldAUlJmLBvlbkvVXLAD+tAMqobIIQpmnUIzUY= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= +github.com/go-latex/latex v0.0.0-20210118124228-b3d85cf34e07/go.mod h1:CO1AlKB2CSIqUrmQPqA0gdRIlnLEY0gK5JGjh37zN5U= +github.com/go-latex/latex v0.0.0-20210823091927-c0d11ff05a81/go.mod h1:SX0U8uGpxhq9o2S/CELCSUxEWWAuoCUcVCQWv7G2OCk= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-pdf/fpdf v0.5.0/go.mod h1:HzcnA+A23uwogo0tp9yU+l3V+KXhiESpt1PMayhOh5M= +github.com/go-pdf/fpdf v0.6.0/go.mod h1:HzcnA+A23uwogo0tp9yU+l3V+KXhiESpt1PMayhOh5M= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= +github.com/goccy/go-json v0.9.11/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= +github.com/golang/glog v1.1.0/go.mod h1:pfYeQZ3JWZoXTV5sFc986z3HTpwQs9At6P4ImfuP3NQ= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -466,11 +727,14 @@ github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= -github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/flatbuffers v2.0.8+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -492,6 +756,7 @@ github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXi github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= +github.com/google/martian/v3 v3.3.2/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= @@ -501,6 +766,7 @@ github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= @@ -512,6 +778,8 @@ github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/googleapis/enterprise-certificate-proxy v0.0.0-20220520183353-fd19c99a87aa/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= github.com/googleapis/enterprise-certificate-proxy v0.1.0/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= github.com/googleapis/enterprise-certificate-proxy v0.2.0/go.mod h1:8C0jb7/mgJe/9KK8Lm7X9ctZC2t60YyIpYEI16jx0Qg= +github.com/googleapis/enterprise-certificate-proxy v0.2.1/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= +github.com/googleapis/enterprise-certificate-proxy v0.2.3/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= @@ -522,10 +790,16 @@ github.com/googleapis/gax-go/v2 v2.4.0/go.mod h1:XOTVJ59hdnfJLIP/dh8n5CGryZR2LxK github.com/googleapis/gax-go/v2 v2.5.1/go.mod h1:h6B0KMMFNtI2ddbGJn3T3ZbwkeT6yqEF02fYlzkUCyo= github.com/googleapis/gax-go/v2 v2.6.0/go.mod h1:1mjbznJAPHFpesgE5ucqfYEscaz5kMdcIDwU/6+DDoY= github.com/googleapis/gax-go/v2 v2.7.0/go.mod h1:TEop28CZZQ2y+c0VxMUmu1lV+fQx57QpBWsYpwqHJx8= +github.com/googleapis/gax-go/v2 v2.7.1/go.mod h1:4orTrqY6hXxxaUL4LHIPl6lGo8vAE38/qKbhSAKP6QI= github.com/googleapis/go-type-adapters v1.0.0/go.mod h1:zHW75FOG2aur7gAO2B+MLby+cLsWGBF62rFAi7WjWO4= +github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0/go.mod h1:hgWBS7lorOAVIJEQMi4ZsPv9hVvWI6+ch50m39Pf2Ks= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.3/go.mod h1:o//XUCC/F+yRGJoPO/VU0GSB0f8Nhgmxx0VIRUvaC0w= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/influxdata/tdigest v0.0.1 h1:XpFptwYmnEKUqmkcDjrzffswZ3nvNeevbUSLPP/ZzIY= @@ -538,38 +812,69 @@ github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/jung-kurt/gofpdf v1.0.0/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= +github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= +github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE= +github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= +github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/lyft/protoc-gen-star v0.6.0/go.mod h1:TGAoBVkt8w7MPG72TrKIu85MIdXwDuzJYeZuUPFPNwA= +github.com/lyft/protoc-gen-star v0.6.1/go.mod h1:TGAoBVkt8w7MPG72TrKIu85MIdXwDuzJYeZuUPFPNwA= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-sqlite3 v1.14.14/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8/go.mod h1:mC1jAcsrzbxHt8iiaC+zU4b1ylILSosueou12R++wfY= +github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3/go.mod h1:RagcQ7I8IeTMnF8JTXieKnO4Z6JCsikNEzj0DwauVzE= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= +github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= +github.com/phpdave11/gofpdf v1.4.2/go.mod h1:zpO6xFn9yxo3YLyMvW8HcKWVdbNqgIfOOp2dXMnm1mY= +github.com/phpdave11/gofpdi v1.0.12/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk/7bXwjDoI= +github.com/phpdave11/gofpdi v1.0.13/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk/7bXwjDoI= +github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pingcap/errors v0.11.0/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c h1:xpW9bvK+HuuTmyFqUwr+jcCvpVkK7sumiz+ko5H9eq4= github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c/go.mod h1:X2r9ueLEUZgtx2cIogM0v4Zj5uvvzhuuiu7Pn8HzMPg= github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00 h1:C3N3itkduZXDZFh4N3vQ5HEtld3S+Y+StULhWVvumU0= github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00/go.mod h1:4qGtCB0QK0wBzKtFEGDhxXnSnbQApw1gc9siScUl8ew= -github.com/pingcap/kvproto v0.0.0-20230228041042-1e9aca94bab6 h1:bgLRG7gPJCq6aduA65ZV7xWQBThTcuarBB9VdfAzV4g= -github.com/pingcap/kvproto v0.0.0-20230228041042-1e9aca94bab6/go.mod h1:KUrW1FGoznGMMTssYBu0czfAhn6vQcIrHyZoSC6T990= +github.com/pingcap/kvproto v0.0.0-20230727073445-53e1f8730c30 h1:EvqKcDT7ceGLW0mXqM8Cp5Z8DfgQRnwj2YTnlCLj2QI= +github.com/pingcap/kvproto v0.0.0-20230727073445-53e1f8730c30/go.mod h1:r0q/CFcwvyeRhKtoqzmWMBebrtpIziQQ9vR+JKh1knc= github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 h1:HR/ylkkLmGdSSDaD8IDP+SZrdhV1Kibl9KrHxJ9eciw= github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= +github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= @@ -591,13 +896,21 @@ github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsT github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/ruudk/golang-pdf417 v0.0.0-20181029194003-1af4ab5afa58/go.mod h1:6lfFZQK844Gfx8o5WFuvpxWRwnSoipWe/p622j1v06w= +github.com/ruudk/golang-pdf417 v0.0.0-20201230142125-a7e3863a1245/go.mod h1:pQAZKsJ8yyVxGRWYNEm9oFB8ieLgKFnamEyDmSA0BRk= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.3.3/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4= +github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= +github.com/spf13/afero v1.9.2/go.mod h1:iUV7ddyEEZPO5gA3zD4fJt6iStLlL+Lg4m2cihcDf8Y= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= @@ -610,14 +923,18 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= +github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= +github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -627,40 +944,63 @@ go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= +go.opentelemetry.io/proto/otlp v0.15.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= +go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI= go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= -go.uber.org/multierr v1.7.0 h1:zaiO/rmgFjbmCXdSYJWQcdvOCsthmdaHfr3Gm2Kx4Ec= go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= -go.uber.org/zap v1.20.0 h1:N4oPlghZwYG55MlU6LXk/Zp00FVNE9X9wrYO8CEs4lc= -go.uber.org/zap v1.20.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw= +go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= +go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191002040644-a1355ae1e2c3/go.mod h1:NOZ3BPKG0ec/BKJQgnvsSFpcKLM5xXVWnvZS97DWHgE= golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= -golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6 h1:QE6XYQK6naiK1EPAe1g/ILLxN5RBoH5xkJk3CqlMI/Y= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/exp v0.0.0-20220827204233-334a2380cb91/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE= +golang.org/x/exp v0.0.0-20230711005742-c3f37128e5a4 h1:QLureRX3moex6NVu/Lr4MGakp9FdA7sBHGBmvRW7NaM= +golang.org/x/exp v0.0.0-20230711005742-c3f37128e5a4/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= +golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20190910094157-69e4b8554b2a/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20200119044424-58c23975cae1/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20200430140353-33d19683fad8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20200618115811-c13761719519/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20201208152932-35266b937fa6/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20210216034530-4410531fe030/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20210607152325-775e3b0c77b9/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM= +golang.org/x/image v0.0.0-20210628002857-a66eb6448b8d/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM= +golang.org/x/image v0.0.0-20211028202545-6944b10bf410/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM= +golang.org/x/image v0.0.0-20220302094943-723b81ca9867/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -684,9 +1024,16 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.5.0/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= +golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI= +golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.11.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -709,6 +1056,7 @@ golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/ golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= @@ -717,11 +1065,14 @@ golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220325170049-de3da57026de/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= @@ -734,8 +1085,14 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.0.0-20220909164309-bea034e7d591/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.0.0-20221012135044-0b7e1fb9d458/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.0.0-20221014081412-f15817d10f9b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= -golang.org/x/net v0.2.0 h1:sZfSu1wtKLGlWI4ZZayP0ck9Y73K1ynO6gqzTdBVdPU= +golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= +golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= +golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -761,6 +1118,9 @@ golang.org/x/oauth2 v0.0.0-20220822191816-0ebed06d0094/go.mod h1:h4gKUeWbJ4rQPri golang.org/x/oauth2 v0.0.0-20220909003341-f21342109be1/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= golang.org/x/oauth2 v0.0.0-20221006150949-b44042a4b9c1/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= +golang.org/x/oauth2 v0.4.0/go.mod h1:RznEsdpjGAINPTOF0UH/t+xJ75L18YO3Ho6Pyn+uRec= +golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I= +golang.org/x/oauth2 v0.6.0/go.mod h1:ycmewcwgD4Rpr3eZJLSB4Kyyljb3qDh40vJ8STE5HKw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -774,11 +1134,12 @@ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220819030929-7fc1605a5dde/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220929204114-8fcdb60fdcc0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20180810173357-98c5dad5d1a0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -789,11 +1150,15 @@ golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -813,14 +1178,18 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210304124612-50617c2ba197/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -829,8 +1198,11 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -847,11 +1219,24 @@ golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220624220833-87e55d714810/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.2.0 h1:ljd4t30dBnAvMZaQCevtY0xLLD0A+bRZXbgLMLU1F/A= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220829200755-d48e67d00261/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= +golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -862,16 +1247,23 @@ golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= -golang.org/x/text v0.4.0 h1:BrVqGRd7+k1DiOgtnFvAkoQEWQvBc25ouMJM6429SFg= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20220922220347-f3bd1da661af/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.1.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= @@ -884,6 +1276,7 @@ golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190927191325-030b2cf1153e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -915,17 +1308,25 @@ golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201124115921-2c860bdd6e78/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA= +golang.org/x/tools v0.3.0/go.mod h1:/rWhSS2+zyEVwoJf8YAX6L2f0ntZ7Kn/mGgAWcipA5k= +golang.org/x/tools v0.5.0/go.mod h1:N+Kgy78s5I24c24dU8OfWNEotWjutIs8SnJvn5IDq+k= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -934,9 +1335,17 @@ golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= -gonum.org/v1/gonum v0.0.0-20181121035319-3f7ecaa7e8ca h1:PupagGYwj8+I4ubCxcmcBRk3VlUWtTg5huQpZR9flmE= +gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= gonum.org/v1/gonum v0.0.0-20181121035319-3f7ecaa7e8ca/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= +gonum.org/v1/gonum v0.8.2/go.mod h1:oe/vMfY3deqTw+1EZJhuvEW2iwGF1bW9wwu7XCu0+v0= +gonum.org/v1/gonum v0.9.3/go.mod h1:TZumC3NeyVQskjXqmyWt4S3bINhy7B4eYwW69EbyX+0= +gonum.org/v1/gonum v0.11.0 h1:f1IJhK4Km5tBJmaiJXtk/PkL4cdVX6J+tGiM187uT5E= +gonum.org/v1/gonum v0.11.0/go.mod h1:fSG4YDCxxUZQJ7rKsQrj0gMOg00Il0Z96/qMA4bVQhA= gonum.org/v1/netlib v0.0.0-20181029234149-ec6d1f5cefe6/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= +gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= +gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc= +gonum.org/v1/plot v0.9.0/go.mod h1:3Pcqqmp6RHvJI72kgb8fThyUnav364FOsdDo2aGW5lY= +gonum.org/v1/plot v0.10.1/go.mod h1:VZW5OlhkL1mysU9vaqNHnsy86inf6Ot+jB3r+BczCEo= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= @@ -988,6 +1397,12 @@ google.golang.org/api v0.99.0/go.mod h1:1YOf74vkVndF7pG6hIHuINsM7eWwpVTAfNMNiL91 google.golang.org/api v0.100.0/go.mod h1:ZE3Z2+ZOr87Rx7dqFsdRQkRBk36kDtp/h+QpHbB7a70= google.golang.org/api v0.102.0/go.mod h1:3VFl6/fzoA+qNuS1N1/VfXY4LjoXN/wzeIp7TweWwGo= google.golang.org/api v0.103.0/go.mod h1:hGtW6nK1AC+d9si/UBhw8Xli+QMOf6xyNAyJw4qU9w0= +google.golang.org/api v0.106.0/go.mod h1:2Ts0XTHNVWxypznxWOYUeI4g3WdP9Pk2Qk58+a/O9MY= +google.golang.org/api v0.107.0/go.mod h1:2Ts0XTHNVWxypznxWOYUeI4g3WdP9Pk2Qk58+a/O9MY= +google.golang.org/api v0.108.0/go.mod h1:2Ts0XTHNVWxypznxWOYUeI4g3WdP9Pk2Qk58+a/O9MY= +google.golang.org/api v0.110.0/go.mod h1:7FC4Vvx1Mooxh8C5HWjzZHcavuS2f6pmJpZx60ca7iI= +google.golang.org/api v0.111.0/go.mod h1:qtFHvU9mhgTJegR31csQ+rwxyUTHOKFqCKWp1J0fdw0= +google.golang.org/api v0.114.0/go.mod h1:ifYI2ZsFK6/uGddGfAD5BMxlnkBqCmqHSDUVi45N5Yg= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -1030,7 +1445,9 @@ google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= @@ -1063,6 +1480,7 @@ google.golang.org/genproto v0.0.0-20220222213610-43724f9ea8cf/go.mod h1:kGP+zUP2 google.golang.org/genproto v0.0.0-20220304144024-325a89244dc8/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= google.golang.org/genproto v0.0.0-20220310185008-1973136f34c6/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= google.golang.org/genproto v0.0.0-20220324131243-acbaeb5b85eb/go.mod h1:hAL49I2IFola2sVEjAn7MEwsja0xp51I0tlGAf9hz4E= +google.golang.org/genproto v0.0.0-20220329172620-7be39ac1afc7/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= google.golang.org/genproto v0.0.0-20220407144326-9054f6ed7bac/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= google.golang.org/genproto v0.0.0-20220413183235-5e96e2839df9/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= google.golang.org/genproto v0.0.0-20220414192740-2d67ff6cf2b4/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= @@ -1098,11 +1516,33 @@ google.golang.org/genproto v0.0.0-20221014213838-99cd37c6964a/go.mod h1:1vXfmgAz google.golang.org/genproto v0.0.0-20221024153911-1573dae28c9c/go.mod h1:9qHF0xnpdSfF6knlcsnpzUu5y+rpwgbvsyGAZPBMg4s= google.golang.org/genproto v0.0.0-20221024183307-1bc688fe9f3e/go.mod h1:9qHF0xnpdSfF6knlcsnpzUu5y+rpwgbvsyGAZPBMg4s= google.golang.org/genproto v0.0.0-20221027153422-115e99e71e1c/go.mod h1:CGI5F/G+E5bKwmfYo09AXuVN4dD894kIKUFmVbP2/Fo= +google.golang.org/genproto v0.0.0-20221109142239-94d6d90a7d66/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= +google.golang.org/genproto v0.0.0-20221114212237-e4508ebdbee1/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= google.golang.org/genproto v0.0.0-20221117204609-8f9c96812029/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= google.golang.org/genproto v0.0.0-20221118155620-16455021b5e6/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= google.golang.org/genproto v0.0.0-20221201164419-0e50fba7f41c/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= -google.golang.org/genproto v0.0.0-20221202195650-67e5cbc046fd h1:OjndDrsik+Gt+e6fs45z9AxiewiKyLKYpA45W5Kpkks= +google.golang.org/genproto v0.0.0-20221201204527-e3fa12d562f3/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= google.golang.org/genproto v0.0.0-20221202195650-67e5cbc046fd/go.mod h1:cTsE614GARnxrLsqKREzmNYJACSWWpAWdNMwnD7c2BE= +google.golang.org/genproto v0.0.0-20221227171554-f9683d7f8bef/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= +google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= +google.golang.org/genproto v0.0.0-20230112194545-e10362b5ecf9/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= +google.golang.org/genproto v0.0.0-20230113154510-dbe35b8444a5/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= +google.golang.org/genproto v0.0.0-20230123190316-2c411cf9d197/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= +google.golang.org/genproto v0.0.0-20230124163310-31e0e69b6fc2/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= +google.golang.org/genproto v0.0.0-20230125152338-dcaf20b6aeaa/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= +google.golang.org/genproto v0.0.0-20230127162408-596548ed4efa/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= +google.golang.org/genproto v0.0.0-20230209215440-0dfe4f8abfcc/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= +google.golang.org/genproto v0.0.0-20230216225411-c8e22ba71e44/go.mod h1:8B0gmkoRebU8ukX6HP+4wrVQUY1+6PkQ44BSyIlflHA= +google.golang.org/genproto v0.0.0-20230222225845-10f96fb3dbec/go.mod h1:3Dl5ZL0q0isWJt+FVcfpQyirqemEuLAK/iFvg1UP1Hw= +google.golang.org/genproto v0.0.0-20230223222841-637eb2293923/go.mod h1:3Dl5ZL0q0isWJt+FVcfpQyirqemEuLAK/iFvg1UP1Hw= +google.golang.org/genproto v0.0.0-20230303212802-e74f57abe488/go.mod h1:TvhZT5f700eVlTNwND1xoEZQeWTB2RY/65kplwl/bFA= +google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4/go.mod h1:NWraEVixdDnqcqQ30jipen1STv2r/n24Wb7twVTGR4s= +google.golang.org/genproto v0.0.0-20230320184635-7606e756e683/go.mod h1:NWraEVixdDnqcqQ30jipen1STv2r/n24Wb7twVTGR4s= +google.golang.org/genproto v0.0.0-20230323212658-478b75c54725/go.mod h1:UUQDJDOlWu4KYeJZffbWgBkS1YFobzKbLVfK69pe0Ak= +google.golang.org/genproto v0.0.0-20230330154414-c0448cd141ea/go.mod h1:UUQDJDOlWu4KYeJZffbWgBkS1YFobzKbLVfK69pe0Ak= +google.golang.org/genproto v0.0.0-20230331144136-dcfb400f0633/go.mod h1:UUQDJDOlWu4KYeJZffbWgBkS1YFobzKbLVfK69pe0Ak= +google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 h1:KpwkzHKEF7B9Zxg18WzOa7djJ+Ha5DzthMyZYQfEn2A= +google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -1129,6 +1569,7 @@ google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnD google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= +google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= @@ -1138,8 +1579,10 @@ google.golang.org/grpc v1.48.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACu google.golang.org/grpc v1.49.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= google.golang.org/grpc v1.50.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= google.golang.org/grpc v1.50.1/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= -google.golang.org/grpc v1.51.0 h1:E1eGv1FTqoLIdnBCZufiSHgKjlqG6fKFf6pPWtMTh8U= google.golang.org/grpc v1.51.0/go.mod h1:wgNDFcnuBGmxLKI/qn4T+m5BtEBYXJPvibbUPsAIPww= +google.golang.org/grpc v1.53.0/go.mod h1:OnIrk0ipVdj4N5d9IUoFUx72/VlD7+jUsHwZgwSMQpw= +google.golang.org/grpc v1.54.0 h1:EhTqbhiYeixwWQtAEZAxmV9MGqcjEU2mFx52xCzNyag= +google.golang.org/grpc v1.54.0/go.mod h1:PUSEXI6iWghWaB6lXM4knEgpJNu2qUcKfDtNci3EC2g= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= @@ -1155,22 +1598,27 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.29.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= +google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= +gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= +gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= @@ -1183,6 +1631,42 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.1.3/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las= +lukechampine.com/uint128 v1.1.1/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk= +lukechampine.com/uint128 v1.2.0/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk= +modernc.org/cc/v3 v3.36.0/go.mod h1:NFUHyPn4ekoC/JHeZFfZurN6ixxawE1BnVonP/oahEI= +modernc.org/cc/v3 v3.36.2/go.mod h1:NFUHyPn4ekoC/JHeZFfZurN6ixxawE1BnVonP/oahEI= +modernc.org/cc/v3 v3.36.3/go.mod h1:NFUHyPn4ekoC/JHeZFfZurN6ixxawE1BnVonP/oahEI= +modernc.org/ccgo/v3 v3.0.0-20220428102840-41399a37e894/go.mod h1:eI31LL8EwEBKPpNpA4bU1/i+sKOwOrQy8D87zWUcRZc= +modernc.org/ccgo/v3 v3.0.0-20220430103911-bc99d88307be/go.mod h1:bwdAnOoaIt8Ax9YdWGjxWsdkPcZyRPHqrOvJxaKAKGw= +modernc.org/ccgo/v3 v3.16.4/go.mod h1:tGtX0gE9Jn7hdZFeU88slbTh1UtCYKusWOoCJuvkWsQ= +modernc.org/ccgo/v3 v3.16.6/go.mod h1:tGtX0gE9Jn7hdZFeU88slbTh1UtCYKusWOoCJuvkWsQ= +modernc.org/ccgo/v3 v3.16.8/go.mod h1:zNjwkizS+fIFDrDjIAgBSCLkWbJuHF+ar3QRn+Z9aws= +modernc.org/ccgo/v3 v3.16.9/go.mod h1:zNMzC9A9xeNUepy6KuZBbugn3c0Mc9TeiJO4lgvkJDo= +modernc.org/ccorpus v1.11.6/go.mod h1:2gEUTrWqdpH2pXsmTM1ZkjeSrUWDpjMu2T6m29L/ErQ= +modernc.org/httpfs v1.0.6/go.mod h1:7dosgurJGp0sPaRanU53W4xZYKh14wfzX420oZADeHM= +modernc.org/libc v0.0.0-20220428101251-2d5f3daf273b/go.mod h1:p7Mg4+koNjc8jkqwcoFBJx7tXkpj00G77X7A72jXPXA= +modernc.org/libc v1.16.0/go.mod h1:N4LD6DBE9cf+Dzf9buBlzVJndKr/iJHG97vGLHYnb5A= +modernc.org/libc v1.16.1/go.mod h1:JjJE0eu4yeK7tab2n4S1w8tlWd9MxXLRzheaRnAKymU= +modernc.org/libc v1.16.17/go.mod h1:hYIV5VZczAmGZAnG15Vdngn5HSF5cSkbvfz2B7GRuVU= +modernc.org/libc v1.16.19/go.mod h1:p7Mg4+koNjc8jkqwcoFBJx7tXkpj00G77X7A72jXPXA= +modernc.org/libc v1.17.0/go.mod h1:XsgLldpP4aWlPlsjqKRdHPqCxCjISdHfM/yeWC5GyW0= +modernc.org/libc v1.17.1/go.mod h1:FZ23b+8LjxZs7XtFMbSzL/EhPxNbfZbErxEHc7cbD9s= +modernc.org/mathutil v1.2.2/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= +modernc.org/mathutil v1.4.1/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= +modernc.org/mathutil v1.5.0/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= +modernc.org/memory v1.1.1/go.mod h1:/0wo5ibyrQiaoUoH7f9D8dnglAmILJ5/cxZlRECf+Nw= +modernc.org/memory v1.2.0/go.mod h1:/0wo5ibyrQiaoUoH7f9D8dnglAmILJ5/cxZlRECf+Nw= +modernc.org/memory v1.2.1/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU= +modernc.org/opt v0.1.1/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0= +modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0= +modernc.org/sqlite v1.18.1/go.mod h1:6ho+Gow7oX5V+OiOQ6Tr4xeqbx13UZ6t+Fw9IRUG4d4= +modernc.org/strutil v1.1.1/go.mod h1:DE+MQQ/hjKBZS2zNInV5hhcipt5rLPWkmpbGeW5mmdw= +modernc.org/strutil v1.1.3/go.mod h1:MEHNA7PdEnEwLvspRMtWTNnp2nnyvMfkimT1NKNAGbw= +modernc.org/tcl v1.13.1/go.mod h1:XOLfOwzhkljL4itZkK6T72ckMgvj0BDsnKNdZVUOecw= +modernc.org/token v1.0.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= +modernc.org/z v1.5.1/go.mod h1:eWFB510QWW5Th9YGZT81s+LwvaAs3Q2yr4sP0rmLkv8= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/tools/pd-tso-bench/main.go b/tools/pd-tso-bench/main.go index d01758a473e..236e78c7808 100644 --- a/tools/pd-tso-bench/main.go +++ b/tools/pd-tso-bench/main.go @@ -19,6 +19,7 @@ import ( "flag" "fmt" "io" + "math/rand" "net/http" "net/http/httptest" "os" @@ -33,24 +34,35 @@ import ( "github.com/prometheus/client_golang/prometheus/promhttp" pd "github.com/tikv/pd/client" "go.uber.org/zap" + "google.golang.org/grpc" + "google.golang.org/grpc/keepalive" +) + +const ( + keepaliveTime = 10 * time.Second + keepaliveTimeout = 3 * time.Second ) var ( - pdAddrs = flag.String("pd", "127.0.0.1:2379", "pd address") - microservice = flag.Bool("m", false, "talks to the tso microservice") - clientNumber = flag.Int("client", 1, "the number of pd clients involved in each benchmark") - concurrency = flag.Int("c", 1000, "concurrency") - count = flag.Int("count", 1, "the count number that the test will run") - duration = flag.Duration("duration", 60*time.Second, "how many seconds the test will last") - dcLocation = flag.String("dc", "global", "which dc-location this bench will request") - verbose = flag.Bool("v", false, "output statistics info every interval and output metrics info at the end") - interval = flag.Duration("interval", time.Second, "interval to output the statistics") - caPath = flag.String("cacert", "", "path of file that contains list of trusted SSL CAs") - certPath = flag.String("cert", "", "path of file that contains X509 certificate in PEM format") - keyPath = flag.String("key", "", "path of file that contains X509 key in PEM format") - maxBatchWaitInterval = flag.Duration("batch-interval", 0, "the max batch wait interval") - enableTSOFollowerProxy = flag.Bool("enable-tso-follower-proxy", false, "whether enable the TSO Follower Proxy") - wg sync.WaitGroup + pdAddrs = flag.String("pd", "127.0.0.1:2379", "pd address") + clientNumber = flag.Int("client", 1, "the number of pd clients involved in each benchmark") + concurrency = flag.Int("c", 1000, "concurrency") + count = flag.Int("count", 1, "the count number that the test will run") + duration = flag.Duration("duration", 60*time.Second, "how many seconds the test will last") + dcLocation = flag.String("dc", "global", "which dc-location this bench will request") + verbose = flag.Bool("v", false, "output statistics info every interval and output metrics info at the end") + interval = flag.Duration("interval", time.Second, "interval to output the statistics") + caPath = flag.String("cacert", "", "path of file that contains list of trusted SSL CAs") + certPath = flag.String("cert", "", "path of file that contains X509 certificate in PEM format") + keyPath = flag.String("key", "", "path of file that contains X509 key in PEM format") + maxBatchWaitInterval = flag.Duration("batch-interval", 0, "the max batch wait interval") + enableTSOFollowerProxy = flag.Bool("enable-tso-follower-proxy", false, "whether enable the TSO Follower Proxy") + enableFaultInjection = flag.Bool("enable-fault-injection", false, "whether enable fault injection") + faultInjectionRate = flag.Float64("fault-injection-rate", 0.01, "the failure rate [0.0001, 1]. 0.01 means 1% failure rate") + maxTSOSendIntervalMilliseconds = flag.Int("max-send-interval-ms", 0, "max tso send interval in milliseconds, 60s by default") + keyspaceID = flag.Uint("keyspace-id", 0, "the id of the keyspace to access") + keyspaceName = flag.String("keyspace-name", "", "the name of the keyspace to access") + wg sync.WaitGroup ) var promServer *httptest.Server @@ -91,27 +103,7 @@ func bench(mainCtx context.Context) { fmt.Printf("Create %d client(s) for benchmark\n", *clientNumber) pdClients := make([]pd.Client, *clientNumber) for idx := range pdClients { - var ( - pdCli pd.Client - err error - ) - - if *microservice { - pdCli, err = pd.NewTSOClientWithContext(mainCtx, 0, []string{*pdAddrs}, pd.SecurityOption{ - CAPath: *caPath, - CertPath: *certPath, - KeyPath: *keyPath, - }) - } else { - pdCli, err = pd.NewClientWithContext(mainCtx, []string{*pdAddrs}, pd.SecurityOption{ - CAPath: *caPath, - CertPath: *certPath, - KeyPath: *keyPath, - }) - } - - pdCli.UpdateOption(pd.MaxTSOBatchWaitInterval, *maxBatchWaitInterval) - pdCli.UpdateOption(pd.EnableTSOFollowerProxy, *enableTSOFollowerProxy) + pdCli, err := createPDClient(mainCtx) if err != nil { log.Fatal(fmt.Sprintf("create pd client #%d failed: %v", idx, err)) } @@ -129,10 +121,18 @@ func bench(mainCtx context.Context) { durCh := make(chan time.Duration, 2*(*concurrency)*(*clientNumber)) - wg.Add((*concurrency) * (*clientNumber)) - for _, pdCli := range pdClients { - for i := 0; i < *concurrency; i++ { - go reqWorker(ctx, pdCli, durCh) + if *enableFaultInjection { + fmt.Printf("Enable fault injection, failure rate: %f\n", *faultInjectionRate) + wg.Add(*clientNumber) + for i := 0; i < *clientNumber; i++ { + go reqWorker(ctx, pdClients, i, durCh) + } + } else { + wg.Add((*concurrency) * (*clientNumber)) + for i := 0; i < *clientNumber; i++ { + for j := 0; j < *concurrency; j++ { + go reqWorker(ctx, pdClients, i, durCh) + } } } @@ -347,22 +347,51 @@ func (s *stats) calculate(count int) float64 { return float64(count) * 100 / float64(s.count) } -func reqWorker(ctx context.Context, pdCli pd.Client, durCh chan time.Duration) { +func reqWorker(ctx context.Context, pdClients []pd.Client, clientIdx int, durCh chan time.Duration) { defer wg.Done() reqCtx, cancel := context.WithCancel(ctx) defer cancel() + var ( + err error + maxRetryTime int = 120 + sleepIntervalOnFailure time.Duration = 1000 * time.Millisecond + totalSleepBeforeGetTS time.Duration + ) + pdCli := pdClients[clientIdx] for { + if pdCli == nil || (*enableFaultInjection && shouldInjectFault()) { + if pdCli != nil { + pdCli.Close() + } + pdCli, err = createPDClient(ctx) + if err != nil { + log.Error(fmt.Sprintf("re-create pd client #%d failed: %v", clientIdx, err)) + select { + case <-reqCtx.Done(): + case <-time.After(100 * time.Millisecond): + } + continue + } + pdClients[clientIdx] = pdCli + } + + totalSleepBeforeGetTS = 0 start := time.Now() - var ( - i int32 - err error - maxRetryTime int32 = 50 - sleepIntervalOnFailure time.Duration = 100 * time.Millisecond - ) + i := 0 for ; i < maxRetryTime; i++ { + if *maxTSOSendIntervalMilliseconds > 0 { + sleepBeforeGetTS := time.Duration(rand.Intn(*maxTSOSendIntervalMilliseconds)) * time.Millisecond + ticker := time.NewTicker(sleepBeforeGetTS) + defer ticker.Stop() + select { + case <-reqCtx.Done(): + case <-ticker.C: + totalSleepBeforeGetTS += sleepBeforeGetTS + } + } _, _, err = pdCli.GetLocalTS(reqCtx, *dcLocation) if errors.Cause(err) == context.Canceled { return @@ -376,7 +405,7 @@ func reqWorker(ctx context.Context, pdCli pd.Client, durCh chan time.Duration) { if err != nil { log.Fatal(fmt.Sprintf("%v", err)) } - dur := time.Since(start) - time.Duration(i)*sleepIntervalOnFailure + dur := time.Since(start) - time.Duration(i)*sleepIntervalOnFailure - totalSleepBeforeGetTS select { case <-reqCtx.Done(): @@ -385,3 +414,44 @@ func reqWorker(ctx context.Context, pdCli pd.Client, durCh chan time.Duration) { } } } + +func createPDClient(ctx context.Context) (pd.Client, error) { + var ( + pdCli pd.Client + err error + ) + + opts := make([]pd.ClientOption, 0) + opts = append(opts, pd.WithGRPCDialOptions( + grpc.WithKeepaliveParams(keepalive.ClientParameters{ + Time: keepaliveTime, + Timeout: keepaliveTimeout, + }), + )) + + if len(*keyspaceName) > 0 { + apiCtx := pd.NewAPIContextV2(*keyspaceName) + pdCli, err = pd.NewClientWithAPIContext(ctx, apiCtx, []string{*pdAddrs}, pd.SecurityOption{ + CAPath: *caPath, + CertPath: *certPath, + KeyPath: *keyPath, + }, opts...) + } else { + pdCli, err = pd.NewClientWithKeyspace(ctx, uint32(*keyspaceID), []string{*pdAddrs}, pd.SecurityOption{ + CAPath: *caPath, + CertPath: *certPath, + KeyPath: *keyPath, + }, opts...) + } + if err != nil { + return nil, err + } + + pdCli.UpdateOption(pd.MaxTSOBatchWaitInterval, *maxBatchWaitInterval) + pdCli.UpdateOption(pd.EnableTSOFollowerProxy, *enableTSOFollowerProxy) + return pdCli, err +} + +func shouldInjectFault() bool { + return rand.Intn(10000) < int(*faultInjectionRate*10000) +}