From cc2900c42e1dfd3f3eda210e1e51715cac491053 Mon Sep 17 00:00:00 2001 From: lhy1024 Date: Thu, 15 Aug 2024 13:56:40 +0800 Subject: [PATCH 01/18] *: upgrade etcd to v3.5.15 (#8441) close tikv/pd#3877, ref tikv/pd#5134 Signed-off-by: lhy1024 --- go.mod | 30 ++++++-- go.sum | 77 +++++++++++++++---- pkg/autoscaling/calculation.go | 2 +- pkg/autoscaling/types.go | 2 +- pkg/basicserver/basic_server.go | 2 +- pkg/election/leadership.go | 4 +- pkg/election/leadership_test.go | 4 +- pkg/election/lease.go | 2 +- pkg/encryption/key_manager.go | 4 +- pkg/encryption/key_manager_test.go | 2 +- pkg/id/id.go | 2 +- pkg/keyspace/tso_keyspace_group.go | 4 +- pkg/mcs/discovery/discover.go | 2 +- pkg/mcs/discovery/register.go | 2 +- pkg/mcs/discovery/register_test.go | 4 +- pkg/mcs/metastorage/server/grpc_service.go | 2 +- pkg/mcs/metastorage/server/manager.go | 2 +- pkg/mcs/scheduling/server/config/watcher.go | 4 +- pkg/mcs/scheduling/server/meta/watcher.go | 4 +- pkg/mcs/scheduling/server/rule/watcher.go | 4 +- .../scheduling/server/rule/watcher_test.go | 4 +- pkg/mcs/server/server.go | 2 +- pkg/mcs/utils/expected_primary.go | 2 +- pkg/mcs/utils/util.go | 6 +- pkg/member/member.go | 4 +- pkg/member/participant.go | 2 +- pkg/storage/endpoint/config.go | 2 +- pkg/storage/endpoint/gc_safe_point.go | 2 +- pkg/storage/endpoint/keyspace.go | 2 +- pkg/storage/endpoint/safepoint_v2.go | 2 +- pkg/storage/endpoint/tso.go | 2 +- pkg/storage/endpoint/tso_keyspace_group.go | 2 +- pkg/storage/endpoint/util.go | 2 +- pkg/storage/etcd_backend.go | 2 +- pkg/storage/kv/etcd_kv.go | 2 +- pkg/storage/kv/kv_test.go | 2 +- pkg/storage/storage.go | 2 +- pkg/storage/storage_test.go | 2 +- pkg/tso/allocator_manager.go | 2 +- pkg/tso/keyspace_group_manager.go | 4 +- pkg/tso/keyspace_group_manager_test.go | 4 +- pkg/tso/local_allocator.go | 2 +- pkg/tso/tso.go | 2 +- pkg/utils/etcdutil/etcdutil.go | 26 ++++--- pkg/utils/etcdutil/etcdutil_test.go | 36 ++++++--- pkg/utils/etcdutil/health_checker.go | 2 +- pkg/utils/etcdutil/testutil.go | 15 +++- pkg/utils/grpcutil/grpcutil.go | 2 +- server/cluster/cluster.go | 2 +- server/config/config.go | 7 +- server/config/persist_options.go | 2 +- server/gc_service.go | 2 +- server/grpc_service.go | 2 +- server/join/join.go | 4 +- server/keyspace_service.go | 4 +- server/server.go | 14 ++-- server/server_test.go | 6 +- server/testutil.go | 2 +- tests/cluster.go | 2 +- tests/integrations/client/client_test.go | 2 +- tests/integrations/client/client_tls_test.go | 2 +- tests/integrations/client/gc_client_test.go | 2 +- tests/integrations/go.mod | 29 +++++-- tests/integrations/go.sum | 77 +++++++++++++++---- .../mcs/scheduling/server_test.go | 3 + tests/integrations/mcs/tso/server_test.go | 2 +- tests/server/tso/manager_test.go | 2 +- tools/go.mod | 29 +++++-- tools/go.sum | 77 +++++++++++++++---- tools/pd-api-bench/cases/cases.go | 2 +- tools/pd-api-bench/cases/controller.go | 2 +- tools/pd-api-bench/main.go | 2 +- tools/pd-backup/main.go | 4 +- tools/pd-backup/pdbackup/backup.go | 2 +- tools/pd-backup/pdbackup/backup_test.go | 4 +- tools/pd-backup/tests/backup_test.go | 2 +- tools/pd-ctl/pdctl/command/global.go | 2 +- tools/pd-ctl/tests/health/health_test.go | 2 +- tools/pd-ctl/tests/store/store_test.go | 2 +- tools/pd-heartbeat-bench/main.go | 2 +- tools/pd-heartbeat-bench/metrics/util.go | 2 +- tools/pd-recover/main.go | 4 +- tools/pd-simulator/simulator/drive.go | 2 +- tools/regions-dump/main.go | 4 +- tools/stores-dump/main.go | 4 +- 85 files changed, 417 insertions(+), 195 deletions(-) diff --git a/go.mod b/go.mod index aee8cc7afad..9dc5fa9f249 100644 --- a/go.mod +++ b/go.mod @@ -37,7 +37,7 @@ require ( github.com/pingcap/kvproto v0.0.0-20240716095229-5f7ffec83ea7 github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21 - github.com/pingcap/tidb-dashboard v0.0.0-20240718034516-e6e78c7c120b + github.com/pingcap/tidb-dashboard v0.0.0-20240815045040-4d89bc193a0c github.com/prometheus/client_golang v1.19.0 github.com/prometheus/common v0.51.1 github.com/sasha-s/go-deadlock v0.2.0 @@ -52,7 +52,10 @@ require ( github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965 github.com/unrolled/render v1.0.1 github.com/urfave/negroni v0.3.0 - go.etcd.io/etcd v0.5.0-alpha.5.0.20240320135013-950cd5fbe6ca + go.etcd.io/etcd/api/v3 v3.5.15 + go.etcd.io/etcd/client/pkg/v3 v3.5.15 + go.etcd.io/etcd/client/v3 v3.5.15 + go.etcd.io/etcd/server/v3 v3.5.15 go.uber.org/atomic v1.10.0 go.uber.org/goleak v1.3.0 go.uber.org/zap v1.27.0 @@ -84,11 +87,10 @@ require ( github.com/bitly/go-simplejson v0.5.0 // indirect github.com/breeswish/gin-jwt/v2 v2.6.4-jwt-patch // indirect github.com/bytedance/sonic v1.9.1 // indirect - github.com/cenkalti/backoff/v4 v4.0.2 // indirect + github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect - github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf // indirect - github.com/coreos/pkg v0.0.0-20240122114842-bbd7aa9bf6fb // indirect + github.com/coreos/go-systemd/v22 v22.3.2 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.3 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dnephin/pflag v1.0.7 // indirect @@ -99,6 +101,8 @@ require ( github.com/gabriel-vasile/mimetype v1.4.2 // indirect github.com/ghodss/yaml v1.0.0 // indirect github.com/gin-contrib/sse v0.1.0 // indirect + github.com/go-logr/logr v1.3.0 // indirect + github.com/go-logr/stdr v1.2.2 // indirect github.com/go-ole/go-ole v1.2.6 // indirect github.com/go-openapi/jsonpointer v0.19.5 // indirect github.com/go-openapi/jsonreference v0.19.6 // indirect @@ -112,6 +116,7 @@ require ( github.com/goccy/go-graphviz v0.1.3 // indirect github.com/goccy/go-json v0.10.2 // indirect github.com/golang-jwt/jwt v3.2.2+incompatible // indirect + github.com/golang-jwt/jwt/v4 v4.4.2 // indirect github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/golang/snappy v0.0.4 // indirect @@ -121,6 +126,7 @@ require ( github.com/gorilla/websocket v1.5.1 // indirect github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 // indirect github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 // indirect github.com/gtank/cryptopasta v0.0.0-20170601214702-1f550f6f2f69 // indirect github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect @@ -169,8 +175,18 @@ require ( github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect github.com/xiang90/probing v0.0.0-20221125231312-a49e3df8f510 // indirect github.com/yusufpapurcu/wmi v1.2.2 // indirect - // Fix panic in unit test with go >= 1.14, ref: etcd-io/bbolt#201 https://github.com/etcd-io/bbolt/pull/201 - go.etcd.io/bbolt v1.3.9 // indirect + go.etcd.io/bbolt v1.3.10 // indirect + go.etcd.io/etcd/client/v2 v2.305.15 // indirect + go.etcd.io/etcd/pkg/v3 v3.5.15 // indirect + go.etcd.io/etcd/raft/v3 v3.5.15 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.0 // indirect + go.opentelemetry.io/otel v1.20.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.20.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.20.0 // indirect + go.opentelemetry.io/otel/metric v1.20.0 // indirect + go.opentelemetry.io/otel/sdk v1.20.0 // indirect + go.opentelemetry.io/otel/trace v1.20.0 // indirect + go.opentelemetry.io/proto/otlp v1.0.0 // indirect go.uber.org/dig v1.9.0 // indirect go.uber.org/fx v1.12.0 // indirect go.uber.org/multierr v1.11.0 diff --git a/go.sum b/go.sum index 3bbddb05476..e3bc1ed175a 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,10 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.112.2 h1:ZaGT6LiG7dBzi6zNOvVZwacaXlmf3lRqnC4DQzqyRQw= +cloud.google.com/go/compute v1.25.1 h1:ZRpHJedLtTpKgr3RV1Fx23NuaAEN1Zfx9hw1u4aJdjU= +cloud.google.com/go/compute v1.25.1/go.mod h1:oopOIR53ly6viBYxaDhBfJwzUAxf1zE//uf3IB011ls= +cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= +cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= github.com/AlekSi/gocov-xml v1.0.0 h1:4QctJBgXEkbzeKz6PJy6bt3JSPNSN4I2mITYW+eKUoQ= github.com/AlekSi/gocov-xml v1.0.0/go.mod h1:J0qYeZ6tDg4oZubW9mAAgxlqw39PDfoEkzB3HXSbEuA= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= @@ -67,8 +72,8 @@ github.com/bytedance/sonic v1.9.1 h1:6iJ6NqdoxCDr6mbY8h18oSO+cShGSMRGCEo7F2h0x8s github.com/bytedance/sonic v1.9.1/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZXU064P/U= github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 h1:BjkPE3785EwPhhyuFkbINB+2a1xATwk8SNDWnJiD41g= github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5/go.mod h1:jtAfVaU/2cu1+wdSRPWE2c1N2qeAA3K4RH9pYgqwets= -github.com/cenkalti/backoff/v4 v4.0.2 h1:JIufpQLbh4DkbQoii76ItQIUFzevQSqOLZca4eamEDs= -github.com/cenkalti/backoff/v4 v4.0.2/go.mod h1:eEew/i+1Q6OrCDZh3WiXYv3+nJwBASZ8Bog/87DQnVg= +github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= +github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -80,14 +85,14 @@ github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5P github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa h1:OaNxuTZr7kxeODyLWsRMC+OD03aFUH+mW6r2d+MWa5Y= -github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= +github.com/cncf/xds/go v0.0.0-20231128003011-0fa0005c9caa h1:jQCWAUqqlij9Pgj2i/PB79y4KOPYVyFYdROxgaCwdTQ= +github.com/cncf/xds/go v0.0.0-20231128003011-0fa0005c9caa/go.mod h1:x/1Gn8zydmfq8dk6e9PdstVsDgu9RuyIIJqAaF//0IM= +github.com/cockroachdb/datadriven v1.0.2 h1:H9MtNqVoVhvd9nCBwOyDjUEdZCREqbIdCJD93PBm/jA= +github.com/cockroachdb/datadriven v1.0.2/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU= github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr4= github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03VsM8rvUec= -github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf h1:iW4rZ826su+pqaw19uhpSCzhj44qo35pNgKFGqzDKkU= -github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/pkg v0.0.0-20240122114842-bbd7aa9bf6fb h1:GIzvVQ9UkUlOhSDlqmrQAAAUd6R3E+caIisNEyWXvNE= -github.com/coreos/pkg v0.0.0-20240122114842-bbd7aa9bf6fb/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/coreos/go-systemd/v22 v22.3.2 h1:D9/bQk5vlXQFZ6Kwuu6zaiXJ9oTPe68++AzAJc1DzSI= +github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/corona10/goimagehash v1.0.2 h1:pUfB0LnsJASMPGEZLj7tGY251vF+qLGqOgEP4rUs6kA= github.com/corona10/goimagehash v1.0.2/go.mod h1:/l9umBhvcHQXVtQO1V6Gp1yD20STawkhRnnX0D1bvVI= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= @@ -111,6 +116,8 @@ github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymF github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/envoyproxy/protoc-gen-validate v1.0.4 h1:gVPz/FMfvh57HdSJQyvBtF00j8JU4zdyUgIUNhlgg0A= +github.com/envoyproxy/protoc-gen-validate v1.0.4/go.mod h1:qys6tmnRsYrQqIhm2bvKZH4Blx/1gTIZ2UKVY1M+Yew= github.com/fatih/color v1.10.0 h1:s36xzo75JdqLaaWoiEHk767eHiwo0598uUxyfiPkDsg= github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= github.com/fogleman/gg v1.3.0 h1:/7zJX8F6AaYQc57WQCyN9cAIz+4bCJGO9B+dyW29am8= @@ -138,6 +145,11 @@ github.com/gin-gonic/gin v1.9.1 h1:4idEAncQnU5cB7BeOkPtxjfCSye0AAm1R0RVIqJ+Jmg= github.com/gin-gonic/gin v1.9.1/go.mod h1:hPrL7YrpYKXt5YId3A/Tnip5kqbEAP+KLuI3SUcPTeU= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY= +github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= @@ -175,6 +187,7 @@ github.com/goccy/go-graphviz v0.1.3/go.mod h1:pMYpbAqJT10V8dzV1JN/g/wUlG/0imKPzn github.com/goccy/go-json v0.9.7/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v0.0.0-20171007142547-342cbe0a0415/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v0.0.0-20180717141946-636bf0302bc9/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= @@ -182,6 +195,8 @@ github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69 github.com/golang-jwt/jwt v3.2.1+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY= github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= +github.com/golang-jwt/jwt/v4 v4.4.2 h1:rcc4lwaZgFMCZ5jxF9ABolDcIHdBytAFgqFPbSJQAYs= +github.com/golang-jwt/jwt/v4 v4.4.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 h1:au07oEsX2xN0ktxqI+Sida1w446QrXBRJ0nee3SNZlA= github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= github.com/golang-sql/sqlexp v0.1.0 h1:ZCD6MBpcuOVfGVqsEmY5/4FtYiKz6tSyUv9LPEDei6A= @@ -189,8 +204,8 @@ github.com/golang-sql/sqlexp v0.1.0/go.mod h1:J4ad9Vo8ZCWQ2GMrC4UCQy1JpCbwU9m3EO github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 h1:DACJavvAHhabrF08vX0COfcOBJRhZ8lUbR+ZWIs0Y5g= github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903 h1:LbsanbbD6LieFkXbj9YNNBupiGHJgFeLpO0j0Fza1h8= -github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/glog v1.2.0 h1:uCdmnmatrKCgMBlM4rMuJZWOkPDqdbZPnrMXDY4gI68= +github.com/golang/glog v1.2.0/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v0.0.0-20180814211427-aa810b61a9c7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -235,6 +250,8 @@ github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92Bcuy github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 h1:YBftPWNWd4WwGqtY2yeZL2ef8rHAxPBD8KFhJpmcqms= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0/go.mod h1:YN5jB8ie0yfIUg6VvR9Kz84aCaG7AsGZnLjhHbUqwPg= github.com/gtank/cryptopasta v0.0.0-20170601214702-1f550f6f2f69 h1:7xsUJsB2NrdcttQPa7JLEaGzvdbk7KvfrjgHZXOQRo0= github.com/gtank/cryptopasta v0.0.0-20170601214702-1f550f6f2f69/go.mod h1:YLEMZOtU+AZ7dhN9T/IpGhXVGly2bvkJQ+zxj3WeVQo= github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= @@ -377,8 +394,8 @@ github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 h1:HR/ylkkLmGdSSDaD8 github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4= github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21 h1:QV6jqlfOkh8hqvEAgwBZa+4bSgO0EeKC7s5c6Luam2I= github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21/go.mod h1:QYnjfA95ZaMefyl1NO8oPtKeb8pYUdnDVhQgf+qdpjM= -github.com/pingcap/tidb-dashboard v0.0.0-20240718034516-e6e78c7c120b h1:MKgJ9yCQxD5ewLERuoiiD9XVOHuqZ2WRZnB20yMiKyo= -github.com/pingcap/tidb-dashboard v0.0.0-20240718034516-e6e78c7c120b/go.mod h1:ucZBRz52icb23T/5Z4CsuUHmarYiin7p2MeiVBe+o8c= +github.com/pingcap/tidb-dashboard v0.0.0-20240815045040-4d89bc193a0c h1:iiVpATdlLLr0NEgKZu+ZnBuYR4J8IgjNE1hNMjMOkYY= +github.com/pingcap/tidb-dashboard v0.0.0-20240815045040-4d89bc193a0c/go.mod h1:AT9vfeojwr/GGCHTURXtA8yZBE9AW8LdIo02/eYdfHU= github.com/pingcap/tipb v0.0.0-20220718022156-3e2483c20a9e h1:FBaTXU8C3xgt/drM58VHxojHo/QoG1oPsgWTGvaSpO4= github.com/pingcap/tipb v0.0.0-20220718022156-3e2483c20a9e/go.mod h1:A7mrd7WHBl1o63LE2bIBGEJMTNWXqhgmYiOvMLxozfs= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= @@ -513,10 +530,38 @@ github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg= github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= -go.etcd.io/bbolt v1.3.9 h1:8x7aARPEXiXbHmtUwAIv7eV2fQFHrLLavdiJ3uzJXoI= -go.etcd.io/bbolt v1.3.9/go.mod h1:zaO32+Ti0PK1ivdPtgMESzuzL2VPoIG1PCQNvOdo/dE= -go.etcd.io/etcd v0.5.0-alpha.5.0.20240320135013-950cd5fbe6ca h1:LCc0GAhfJ+qDqnUbE7ybQ0mTz1dNRn2iiM6e183p/5E= -go.etcd.io/etcd v0.5.0-alpha.5.0.20240320135013-950cd5fbe6ca/go.mod h1:1AyK+XVcIwjbjw5EYrhT+IiMYSgRZTohGb2ceZ0/US8= +go.etcd.io/bbolt v1.3.10 h1:+BqfJTcCzTItrop8mq/lbzL8wSGtj94UO/3U31shqG0= +go.etcd.io/bbolt v1.3.10/go.mod h1:bK3UQLPJZly7IlNmV7uVHJDxfe5aK9Ll93e/74Y9oEQ= +go.etcd.io/etcd/api/v3 v3.5.15 h1:3KpLJir1ZEBrYuV2v+Twaa/e2MdDCEZ/70H+lzEiwsk= +go.etcd.io/etcd/api/v3 v3.5.15/go.mod h1:N9EhGzXq58WuMllgH9ZvnEr7SI9pS0k0+DHZezGp7jM= +go.etcd.io/etcd/client/pkg/v3 v3.5.15 h1:fo0HpWz/KlHGMCC+YejpiCmyWDEuIpnTDzpJLB5fWlA= +go.etcd.io/etcd/client/pkg/v3 v3.5.15/go.mod h1:mXDI4NAOwEiszrHCb0aqfAYNCrZP4e9hRca3d1YK8EU= +go.etcd.io/etcd/client/v2 v2.305.15 h1:VG2xbf8Vz1KJh65Ar2V5eDmfkp1bpzkSEHlhJM3usp8= +go.etcd.io/etcd/client/v2 v2.305.15/go.mod h1:Ad5dRjPVb/n5yXgAWQ/hXzuXXkBk0Y658ocuXYaUU48= +go.etcd.io/etcd/client/v3 v3.5.15 h1:23M0eY4Fd/inNv1ZfU3AxrbbOdW79r9V9Rl62Nm6ip4= +go.etcd.io/etcd/client/v3 v3.5.15/go.mod h1:CLSJxrYjvLtHsrPKsy7LmZEE+DK2ktfd2bN4RhBMwlU= +go.etcd.io/etcd/pkg/v3 v3.5.15 h1:/Iu6Sr3iYaAjy++8sIDoZW9/EfhcwLZwd4FOZX2mMOU= +go.etcd.io/etcd/pkg/v3 v3.5.15/go.mod h1:e3Acf298sPFmTCGTrnGvkClEw9RYIyPtNzi1XM8rets= +go.etcd.io/etcd/raft/v3 v3.5.15 h1:jOA2HJF7zb3wy8H/pL13e8geWqkEa/kUs0waUggZC0I= +go.etcd.io/etcd/raft/v3 v3.5.15/go.mod h1:k3r7P4seEiUcgxOPLp+mloJWV3Q4QLPGNvy/OgC8OtM= +go.etcd.io/etcd/server/v3 v3.5.15 h1:x35jrWnZgsRwMsFsUJIUdT1bvzIz1B+29HjMfRYVN/E= +go.etcd.io/etcd/server/v3 v3.5.15/go.mod h1:l9jX9oa/iuArjqz0RNX/TDbc70dLXxRZo/nmPucrpFo= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.0 h1:PzIubN4/sjByhDRHLviCjJuweBXWFZWhghjg7cS28+M= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.0/go.mod h1:Ct6zzQEuGK3WpJs2n4dn+wfJYzd/+hNnxMRTWjGn30M= +go.opentelemetry.io/otel v1.20.0 h1:vsb/ggIY+hUjD/zCAQHpzTmndPqv/ml2ArbsbfBYTAc= +go.opentelemetry.io/otel v1.20.0/go.mod h1:oUIGj3D77RwJdM6PPZImDpSZGDvkD9fhesHny69JFrs= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.20.0 h1:DeFD0VgTZ+Cj6hxravYYZE2W4GlneVH81iAOPjZkzk8= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.20.0/go.mod h1:GijYcYmNpX1KazD5JmWGsi4P7dDTTTnfv1UbGn84MnU= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.20.0 h1:gvmNvqrPYovvyRmCSygkUDyL8lC5Tl845MLEwqpxhEU= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.20.0/go.mod h1:vNUq47TGFioo+ffTSnKNdob241vePmtNZnAODKapKd0= +go.opentelemetry.io/otel/metric v1.20.0 h1:ZlrO8Hu9+GAhnepmRGhSU7/VkpjrNowxRN9GyKR4wzA= +go.opentelemetry.io/otel/metric v1.20.0/go.mod h1:90DRw3nfK4D7Sm/75yQ00gTJxtkBxX+wu6YaNymbpVM= +go.opentelemetry.io/otel/sdk v1.20.0 h1:5Jf6imeFZlZtKv9Qbo6qt2ZkmWtdWx/wzcCbNUlAWGM= +go.opentelemetry.io/otel/sdk v1.20.0/go.mod h1:rmkSx1cZCm/tn16iWDn1GQbLtsW/LvsdEEFzCSRM6V0= +go.opentelemetry.io/otel/trace v1.20.0 h1:+yxVAPZPbQhbC3OfAkeIVTky6iTFpcr4SiY9om7mXSQ= +go.opentelemetry.io/otel/trace v1.20.0/go.mod h1:HJSK7F/hA5RlzpZ0zKDCHCDHm556LCDtKaAo6JmBFUU= +go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I= +go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= diff --git a/pkg/autoscaling/calculation.go b/pkg/autoscaling/calculation.go index 8c8783dd618..b155c44bf68 100644 --- a/pkg/autoscaling/calculation.go +++ b/pkg/autoscaling/calculation.go @@ -29,7 +29,7 @@ import ( "github.com/tikv/pd/pkg/utils/typeutil" "github.com/tikv/pd/server/cluster" "github.com/tikv/pd/server/config" - "go.etcd.io/etcd/clientv3" + clientv3 "go.etcd.io/etcd/client/v3" "go.uber.org/zap" ) diff --git a/pkg/autoscaling/types.go b/pkg/autoscaling/types.go index 5499d20f510..53c614312b2 100644 --- a/pkg/autoscaling/types.go +++ b/pkg/autoscaling/types.go @@ -21,7 +21,7 @@ import ( "strings" "github.com/tikv/pd/pkg/utils/etcdutil" - "go.etcd.io/etcd/clientv3" + clientv3 "go.etcd.io/etcd/client/v3" ) // Strategy within a HTTP request provides rules and resources to help make decision for auto scaling. diff --git a/pkg/basicserver/basic_server.go b/pkg/basicserver/basic_server.go index 28ba3ad08de..4b7e4eeab50 100644 --- a/pkg/basicserver/basic_server.go +++ b/pkg/basicserver/basic_server.go @@ -18,7 +18,7 @@ import ( "context" "net/http" - "go.etcd.io/etcd/clientv3" + clientv3 "go.etcd.io/etcd/client/v3" ) // Server defines the common basic behaviors of a server diff --git a/pkg/election/leadership.go b/pkg/election/leadership.go index 1361d685b57..ec64a003c53 100644 --- a/pkg/election/leadership.go +++ b/pkg/election/leadership.go @@ -28,8 +28,8 @@ import ( "github.com/tikv/pd/pkg/utils/etcdutil" "github.com/tikv/pd/pkg/utils/grpcutil" "github.com/tikv/pd/pkg/utils/syncutil" - "go.etcd.io/etcd/clientv3" - "go.etcd.io/etcd/mvcc/mvccpb" + "go.etcd.io/etcd/api/v3/mvccpb" + clientv3 "go.etcd.io/etcd/client/v3" "go.uber.org/zap" ) diff --git a/pkg/election/leadership_test.go b/pkg/election/leadership_test.go index e16c5842542..1846d671a61 100644 --- a/pkg/election/leadership_test.go +++ b/pkg/election/leadership_test.go @@ -25,8 +25,8 @@ import ( "github.com/stretchr/testify/require" "github.com/tikv/pd/pkg/utils/etcdutil" "github.com/tikv/pd/pkg/utils/testutil" - "go.etcd.io/etcd/clientv3" - "go.etcd.io/etcd/embed" + clientv3 "go.etcd.io/etcd/client/v3" + "go.etcd.io/etcd/server/v3/embed" ) const defaultLeaseTimeout = 1 diff --git a/pkg/election/lease.go b/pkg/election/lease.go index c2e9eb97117..21bd43018b5 100644 --- a/pkg/election/lease.go +++ b/pkg/election/lease.go @@ -24,7 +24,7 @@ import ( "github.com/tikv/pd/pkg/utils/etcdutil" "github.com/tikv/pd/pkg/utils/logutil" "github.com/tikv/pd/pkg/utils/typeutil" - "go.etcd.io/etcd/clientv3" + clientv3 "go.etcd.io/etcd/client/v3" "go.uber.org/zap" ) diff --git a/pkg/encryption/key_manager.go b/pkg/encryption/key_manager.go index 28a5f04d0f7..5a029847a9d 100644 --- a/pkg/encryption/key_manager.go +++ b/pkg/encryption/key_manager.go @@ -26,8 +26,8 @@ import ( "github.com/tikv/pd/pkg/errs" "github.com/tikv/pd/pkg/utils/etcdutil" "github.com/tikv/pd/pkg/utils/syncutil" - "go.etcd.io/etcd/clientv3" - "go.etcd.io/etcd/mvcc/mvccpb" + "go.etcd.io/etcd/api/v3/mvccpb" + clientv3 "go.etcd.io/etcd/client/v3" "go.uber.org/zap" ) diff --git a/pkg/encryption/key_manager_test.go b/pkg/encryption/key_manager_test.go index 26453eeb5b3..e827a714a3a 100644 --- a/pkg/encryption/key_manager_test.go +++ b/pkg/encryption/key_manager_test.go @@ -30,7 +30,7 @@ import ( "github.com/tikv/pd/pkg/election" "github.com/tikv/pd/pkg/utils/etcdutil" "github.com/tikv/pd/pkg/utils/typeutil" - "go.etcd.io/etcd/clientv3" + clientv3 "go.etcd.io/etcd/client/v3" ) // #nosec G101 diff --git a/pkg/id/id.go b/pkg/id/id.go index ea4a2a54341..0225dedd4f1 100644 --- a/pkg/id/id.go +++ b/pkg/id/id.go @@ -24,7 +24,7 @@ import ( "github.com/tikv/pd/pkg/utils/etcdutil" "github.com/tikv/pd/pkg/utils/syncutil" "github.com/tikv/pd/pkg/utils/typeutil" - "go.etcd.io/etcd/clientv3" + clientv3 "go.etcd.io/etcd/client/v3" "go.uber.org/zap" ) diff --git a/pkg/keyspace/tso_keyspace_group.go b/pkg/keyspace/tso_keyspace_group.go index ffab89f4f0e..5d7137ac3c2 100644 --- a/pkg/keyspace/tso_keyspace_group.go +++ b/pkg/keyspace/tso_keyspace_group.go @@ -37,8 +37,8 @@ import ( "github.com/tikv/pd/pkg/utils/logutil" "github.com/tikv/pd/pkg/utils/syncutil" "github.com/tikv/pd/pkg/utils/typeutil" - "go.etcd.io/etcd/clientv3" - "go.etcd.io/etcd/mvcc/mvccpb" + "go.etcd.io/etcd/api/v3/mvccpb" + clientv3 "go.etcd.io/etcd/client/v3" "go.uber.org/zap" ) diff --git a/pkg/mcs/discovery/discover.go b/pkg/mcs/discovery/discover.go index c219cbc047f..119ce7957d9 100644 --- a/pkg/mcs/discovery/discover.go +++ b/pkg/mcs/discovery/discover.go @@ -23,7 +23,7 @@ import ( "github.com/tikv/pd/pkg/mcs/utils/constant" "github.com/tikv/pd/pkg/storage/kv" "github.com/tikv/pd/pkg/utils/etcdutil" - "go.etcd.io/etcd/clientv3" + clientv3 "go.etcd.io/etcd/client/v3" "go.uber.org/zap" ) diff --git a/pkg/mcs/discovery/register.go b/pkg/mcs/discovery/register.go index 36fb19f2673..ab27387ab91 100644 --- a/pkg/mcs/discovery/register.go +++ b/pkg/mcs/discovery/register.go @@ -22,7 +22,7 @@ import ( "github.com/pingcap/log" "github.com/tikv/pd/pkg/utils/etcdutil" "github.com/tikv/pd/pkg/utils/logutil" - "go.etcd.io/etcd/clientv3" + clientv3 "go.etcd.io/etcd/client/v3" "go.uber.org/zap" ) diff --git a/pkg/mcs/discovery/register_test.go b/pkg/mcs/discovery/register_test.go index 707c251e5fb..3ea0148d75f 100644 --- a/pkg/mcs/discovery/register_test.go +++ b/pkg/mcs/discovery/register_test.go @@ -24,8 +24,8 @@ import ( "github.com/stretchr/testify/require" "github.com/tikv/pd/pkg/utils/etcdutil" "github.com/tikv/pd/pkg/utils/testutil" - "go.etcd.io/etcd/clientv3" - "go.etcd.io/etcd/embed" + clientv3 "go.etcd.io/etcd/client/v3" + "go.etcd.io/etcd/server/v3/embed" ) func TestRegister(t *testing.T) { diff --git a/pkg/mcs/metastorage/server/grpc_service.go b/pkg/mcs/metastorage/server/grpc_service.go index c3ecfa572a1..4f43114ea29 100644 --- a/pkg/mcs/metastorage/server/grpc_service.go +++ b/pkg/mcs/metastorage/server/grpc_service.go @@ -24,7 +24,7 @@ import ( bs "github.com/tikv/pd/pkg/basicserver" "github.com/tikv/pd/pkg/mcs/registry" "github.com/tikv/pd/pkg/utils/apiutil" - "go.etcd.io/etcd/clientv3" + clientv3 "go.etcd.io/etcd/client/v3" "go.uber.org/zap" "google.golang.org/grpc" "google.golang.org/grpc/codes" diff --git a/pkg/mcs/metastorage/server/manager.go b/pkg/mcs/metastorage/server/manager.go index bd082f48202..17fee85c08b 100644 --- a/pkg/mcs/metastorage/server/manager.go +++ b/pkg/mcs/metastorage/server/manager.go @@ -19,7 +19,7 @@ import ( bs "github.com/tikv/pd/pkg/basicserver" "github.com/tikv/pd/pkg/storage/endpoint" "github.com/tikv/pd/pkg/storage/kv" - "go.etcd.io/etcd/clientv3" + clientv3 "go.etcd.io/etcd/client/v3" "go.uber.org/zap" ) diff --git a/pkg/mcs/scheduling/server/config/watcher.go b/pkg/mcs/scheduling/server/config/watcher.go index d1ca99bd36d..46201ba3cd3 100644 --- a/pkg/mcs/scheduling/server/config/watcher.go +++ b/pkg/mcs/scheduling/server/config/watcher.go @@ -29,8 +29,8 @@ import ( "github.com/tikv/pd/pkg/storage" "github.com/tikv/pd/pkg/storage/endpoint" "github.com/tikv/pd/pkg/utils/etcdutil" - "go.etcd.io/etcd/clientv3" - "go.etcd.io/etcd/mvcc/mvccpb" + "go.etcd.io/etcd/api/v3/mvccpb" + clientv3 "go.etcd.io/etcd/client/v3" "go.uber.org/zap" ) diff --git a/pkg/mcs/scheduling/server/meta/watcher.go b/pkg/mcs/scheduling/server/meta/watcher.go index 2daa6766d75..42e9cf054ec 100644 --- a/pkg/mcs/scheduling/server/meta/watcher.go +++ b/pkg/mcs/scheduling/server/meta/watcher.go @@ -26,8 +26,8 @@ import ( "github.com/tikv/pd/pkg/statistics" "github.com/tikv/pd/pkg/storage/endpoint" "github.com/tikv/pd/pkg/utils/etcdutil" - "go.etcd.io/etcd/clientv3" - "go.etcd.io/etcd/mvcc/mvccpb" + "go.etcd.io/etcd/api/v3/mvccpb" + clientv3 "go.etcd.io/etcd/client/v3" "go.uber.org/zap" ) diff --git a/pkg/mcs/scheduling/server/rule/watcher.go b/pkg/mcs/scheduling/server/rule/watcher.go index ea90b9d4e49..118da18376c 100644 --- a/pkg/mcs/scheduling/server/rule/watcher.go +++ b/pkg/mcs/scheduling/server/rule/watcher.go @@ -26,8 +26,8 @@ import ( "github.com/tikv/pd/pkg/schedule/placement" "github.com/tikv/pd/pkg/storage/endpoint" "github.com/tikv/pd/pkg/utils/etcdutil" - "go.etcd.io/etcd/clientv3" - "go.etcd.io/etcd/mvcc/mvccpb" + "go.etcd.io/etcd/api/v3/mvccpb" + clientv3 "go.etcd.io/etcd/client/v3" "go.uber.org/zap" ) diff --git a/pkg/mcs/scheduling/server/rule/watcher_test.go b/pkg/mcs/scheduling/server/rule/watcher_test.go index 37fce0a0ded..11b721b4e9e 100644 --- a/pkg/mcs/scheduling/server/rule/watcher_test.go +++ b/pkg/mcs/scheduling/server/rule/watcher_test.go @@ -28,8 +28,8 @@ import ( "github.com/tikv/pd/pkg/storage/endpoint" "github.com/tikv/pd/pkg/storage/kv" "github.com/tikv/pd/pkg/utils/etcdutil" - "go.etcd.io/etcd/clientv3" - "go.etcd.io/etcd/embed" + clientv3 "go.etcd.io/etcd/client/v3" + "go.etcd.io/etcd/server/v3/embed" ) const ( diff --git a/pkg/mcs/server/server.go b/pkg/mcs/server/server.go index 31b6c2c76cb..fef05a85012 100644 --- a/pkg/mcs/server/server.go +++ b/pkg/mcs/server/server.go @@ -26,7 +26,7 @@ import ( "github.com/pingcap/log" "github.com/tikv/pd/pkg/mcs/utils/constant" "github.com/tikv/pd/pkg/utils/grpcutil" - "go.etcd.io/etcd/clientv3" + clientv3 "go.etcd.io/etcd/client/v3" "google.golang.org/grpc" ) diff --git a/pkg/mcs/utils/expected_primary.go b/pkg/mcs/utils/expected_primary.go index d44b2eae436..c65d0a1cc5e 100644 --- a/pkg/mcs/utils/expected_primary.go +++ b/pkg/mcs/utils/expected_primary.go @@ -29,7 +29,7 @@ import ( "github.com/tikv/pd/pkg/storage/endpoint" "github.com/tikv/pd/pkg/storage/kv" "github.com/tikv/pd/pkg/utils/etcdutil" - "go.etcd.io/etcd/clientv3" + clientv3 "go.etcd.io/etcd/client/v3" "go.uber.org/zap" ) diff --git a/pkg/mcs/utils/util.go b/pkg/mcs/utils/util.go index fa76aadc1e8..dfcfc0e312d 100644 --- a/pkg/mcs/utils/util.go +++ b/pkg/mcs/utils/util.go @@ -40,8 +40,8 @@ import ( "github.com/tikv/pd/pkg/utils/grpcutil" "github.com/tikv/pd/pkg/utils/logutil" "github.com/tikv/pd/pkg/versioninfo" - "go.etcd.io/etcd/clientv3" - "go.etcd.io/etcd/pkg/types" + etcdtypes "go.etcd.io/etcd/client/pkg/v3/types" + clientv3 "go.etcd.io/etcd/client/v3" "go.uber.org/zap" "google.golang.org/grpc" "google.golang.org/grpc/keepalive" @@ -123,7 +123,7 @@ func InitClient(s server) error { if err != nil { return err } - backendUrls, err := types.NewURLs(strings.Split(s.GetBackendEndpoints(), ",")) + backendUrls, err := etcdtypes.NewURLs(strings.Split(s.GetBackendEndpoints(), ",")) if err != nil { return err } diff --git a/pkg/member/member.go b/pkg/member/member.go index 9c61a832b5f..8bf50e6d310 100644 --- a/pkg/member/member.go +++ b/pkg/member/member.go @@ -34,8 +34,8 @@ import ( "github.com/tikv/pd/pkg/errs" "github.com/tikv/pd/pkg/storage/kv" "github.com/tikv/pd/pkg/utils/etcdutil" - "go.etcd.io/etcd/clientv3" - "go.etcd.io/etcd/embed" + clientv3 "go.etcd.io/etcd/client/v3" + "go.etcd.io/etcd/server/v3/embed" "go.uber.org/zap" ) diff --git a/pkg/member/participant.go b/pkg/member/participant.go index 5e35d127bf7..599e56387d6 100644 --- a/pkg/member/participant.go +++ b/pkg/member/participant.go @@ -30,7 +30,7 @@ import ( "github.com/tikv/pd/pkg/errs" "github.com/tikv/pd/pkg/mcs/utils/constant" "github.com/tikv/pd/pkg/utils/etcdutil" - "go.etcd.io/etcd/clientv3" + clientv3 "go.etcd.io/etcd/client/v3" "go.uber.org/zap" ) diff --git a/pkg/storage/endpoint/config.go b/pkg/storage/endpoint/config.go index 820778e36ff..71ae8cb5248 100644 --- a/pkg/storage/endpoint/config.go +++ b/pkg/storage/endpoint/config.go @@ -19,7 +19,7 @@ import ( "strings" "github.com/tikv/pd/pkg/errs" - "go.etcd.io/etcd/clientv3" + clientv3 "go.etcd.io/etcd/client/v3" ) // ConfigStorage defines the storage operations on the config. diff --git a/pkg/storage/endpoint/gc_safe_point.go b/pkg/storage/endpoint/gc_safe_point.go index 85b29e0b47e..8d59d827fa4 100644 --- a/pkg/storage/endpoint/gc_safe_point.go +++ b/pkg/storage/endpoint/gc_safe_point.go @@ -23,7 +23,7 @@ import ( "github.com/pingcap/errors" "github.com/pingcap/log" "github.com/tikv/pd/pkg/errs" - "go.etcd.io/etcd/clientv3" + clientv3 "go.etcd.io/etcd/client/v3" ) // ServiceSafePoint is the safepoint for a specific service diff --git a/pkg/storage/endpoint/keyspace.go b/pkg/storage/endpoint/keyspace.go index 30540e49a2e..b892250c463 100644 --- a/pkg/storage/endpoint/keyspace.go +++ b/pkg/storage/endpoint/keyspace.go @@ -22,7 +22,7 @@ import ( "github.com/pingcap/kvproto/pkg/keyspacepb" "github.com/tikv/pd/pkg/errs" "github.com/tikv/pd/pkg/storage/kv" - "go.etcd.io/etcd/clientv3" + clientv3 "go.etcd.io/etcd/client/v3" ) const ( diff --git a/pkg/storage/endpoint/safepoint_v2.go b/pkg/storage/endpoint/safepoint_v2.go index 8d690d07261..2072f9c12ec 100644 --- a/pkg/storage/endpoint/safepoint_v2.go +++ b/pkg/storage/endpoint/safepoint_v2.go @@ -22,7 +22,7 @@ import ( "github.com/pingcap/errors" "github.com/pingcap/log" "github.com/tikv/pd/pkg/errs" - "go.etcd.io/etcd/clientv3" + clientv3 "go.etcd.io/etcd/client/v3" "go.uber.org/zap" ) diff --git a/pkg/storage/endpoint/tso.go b/pkg/storage/endpoint/tso.go index f0aa9d8cde0..30aceb88fa8 100644 --- a/pkg/storage/endpoint/tso.go +++ b/pkg/storage/endpoint/tso.go @@ -23,7 +23,7 @@ import ( "github.com/pingcap/log" "github.com/tikv/pd/pkg/storage/kv" "github.com/tikv/pd/pkg/utils/typeutil" - "go.etcd.io/etcd/clientv3" + clientv3 "go.etcd.io/etcd/client/v3" "go.uber.org/zap" ) diff --git a/pkg/storage/endpoint/tso_keyspace_group.go b/pkg/storage/endpoint/tso_keyspace_group.go index 5b6a7481176..9c7b12fff5f 100644 --- a/pkg/storage/endpoint/tso_keyspace_group.go +++ b/pkg/storage/endpoint/tso_keyspace_group.go @@ -21,7 +21,7 @@ import ( "github.com/tikv/pd/pkg/slice" "github.com/tikv/pd/pkg/storage/kv" "github.com/tikv/pd/pkg/utils/typeutil" - "go.etcd.io/etcd/clientv3" + clientv3 "go.etcd.io/etcd/client/v3" ) // UserKind represents the user kind. diff --git a/pkg/storage/endpoint/util.go b/pkg/storage/endpoint/util.go index 06535490206..39aa6240f5a 100644 --- a/pkg/storage/endpoint/util.go +++ b/pkg/storage/endpoint/util.go @@ -23,7 +23,7 @@ import ( "github.com/tikv/pd/pkg/errs" "github.com/tikv/pd/pkg/storage/kv" "github.com/tikv/pd/pkg/utils/etcdutil" - "go.etcd.io/etcd/clientv3" + clientv3 "go.etcd.io/etcd/client/v3" ) func (se *StorageEndpoint) loadProto(key string, msg proto.Message) (bool, error) { diff --git a/pkg/storage/etcd_backend.go b/pkg/storage/etcd_backend.go index f354c1bb070..e9af5fc67f6 100644 --- a/pkg/storage/etcd_backend.go +++ b/pkg/storage/etcd_backend.go @@ -17,7 +17,7 @@ package storage import ( "github.com/tikv/pd/pkg/storage/endpoint" "github.com/tikv/pd/pkg/storage/kv" - "go.etcd.io/etcd/clientv3" + clientv3 "go.etcd.io/etcd/client/v3" ) // etcdBackend is a storage backend that stores data in etcd, diff --git a/pkg/storage/kv/etcd_kv.go b/pkg/storage/kv/etcd_kv.go index e2eb8c979eb..5945990c51b 100644 --- a/pkg/storage/kv/etcd_kv.go +++ b/pkg/storage/kv/etcd_kv.go @@ -25,7 +25,7 @@ import ( "github.com/pingcap/log" "github.com/tikv/pd/pkg/errs" "github.com/tikv/pd/pkg/utils/etcdutil" - "go.etcd.io/etcd/clientv3" + clientv3 "go.etcd.io/etcd/client/v3" "go.uber.org/zap" ) diff --git a/pkg/storage/kv/kv_test.go b/pkg/storage/kv/kv_test.go index 93359934da1..3bc3120af95 100644 --- a/pkg/storage/kv/kv_test.go +++ b/pkg/storage/kv/kv_test.go @@ -23,7 +23,7 @@ import ( "github.com/stretchr/testify/require" "github.com/tikv/pd/pkg/utils/etcdutil" - "go.etcd.io/etcd/clientv3" + clientv3 "go.etcd.io/etcd/client/v3" ) func TestEtcd(t *testing.T) { diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go index 5e006133d22..dce2f1712b8 100644 --- a/pkg/storage/storage.go +++ b/pkg/storage/storage.go @@ -24,7 +24,7 @@ import ( "github.com/tikv/pd/pkg/storage/endpoint" "github.com/tikv/pd/pkg/storage/kv" "github.com/tikv/pd/pkg/utils/syncutil" - "go.etcd.io/etcd/clientv3" + clientv3 "go.etcd.io/etcd/client/v3" ) // Storage is the interface for the backend storage of the PD. diff --git a/pkg/storage/storage_test.go b/pkg/storage/storage_test.go index 460489ecd10..d5f27e34714 100644 --- a/pkg/storage/storage_test.go +++ b/pkg/storage/storage_test.go @@ -30,7 +30,7 @@ import ( "github.com/stretchr/testify/require" "github.com/tikv/pd/pkg/core" "github.com/tikv/pd/pkg/storage/endpoint" - "go.etcd.io/etcd/clientv3" + clientv3 "go.etcd.io/etcd/client/v3" ) func TestBasic(t *testing.T) { diff --git a/pkg/tso/allocator_manager.go b/pkg/tso/allocator_manager.go index 9cd5ae9c743..1f5bce04583 100644 --- a/pkg/tso/allocator_manager.go +++ b/pkg/tso/allocator_manager.go @@ -40,7 +40,7 @@ import ( "github.com/tikv/pd/pkg/utils/grpcutil" "github.com/tikv/pd/pkg/utils/logutil" "github.com/tikv/pd/pkg/utils/syncutil" - "go.etcd.io/etcd/clientv3" + clientv3 "go.etcd.io/etcd/client/v3" "go.uber.org/zap" "google.golang.org/grpc" ) diff --git a/pkg/tso/keyspace_group_manager.go b/pkg/tso/keyspace_group_manager.go index 6db935cdce8..0e62fc0346c 100644 --- a/pkg/tso/keyspace_group_manager.go +++ b/pkg/tso/keyspace_group_manager.go @@ -45,8 +45,8 @@ import ( "github.com/tikv/pd/pkg/utils/syncutil" "github.com/tikv/pd/pkg/utils/tsoutil" "github.com/tikv/pd/pkg/utils/typeutil" - "go.etcd.io/etcd/clientv3" - "go.etcd.io/etcd/mvcc/mvccpb" + "go.etcd.io/etcd/api/v3/mvccpb" + clientv3 "go.etcd.io/etcd/client/v3" "go.uber.org/zap" ) diff --git a/pkg/tso/keyspace_group_manager_test.go b/pkg/tso/keyspace_group_manager_test.go index b891aeb246d..fb4ac92a59c 100644 --- a/pkg/tso/keyspace_group_manager_test.go +++ b/pkg/tso/keyspace_group_manager_test.go @@ -40,8 +40,8 @@ import ( "github.com/tikv/pd/pkg/utils/testutil" "github.com/tikv/pd/pkg/utils/tsoutil" "github.com/tikv/pd/pkg/utils/typeutil" - "go.etcd.io/etcd/clientv3" - "go.etcd.io/etcd/mvcc/mvccpb" + "go.etcd.io/etcd/api/v3/mvccpb" + clientv3 "go.etcd.io/etcd/client/v3" "go.uber.org/goleak" ) diff --git a/pkg/tso/local_allocator.go b/pkg/tso/local_allocator.go index e9019bf2bf3..3bbaa661dd6 100644 --- a/pkg/tso/local_allocator.go +++ b/pkg/tso/local_allocator.go @@ -30,7 +30,7 @@ import ( "github.com/tikv/pd/pkg/utils/logutil" "github.com/tikv/pd/pkg/utils/tsoutil" "github.com/tikv/pd/pkg/utils/typeutil" - "go.etcd.io/etcd/clientv3" + clientv3 "go.etcd.io/etcd/client/v3" "go.uber.org/zap" ) diff --git a/pkg/tso/tso.go b/pkg/tso/tso.go index bcb3169e73c..d04710ac378 100644 --- a/pkg/tso/tso.go +++ b/pkg/tso/tso.go @@ -31,7 +31,7 @@ import ( "github.com/tikv/pd/pkg/utils/syncutil" "github.com/tikv/pd/pkg/utils/tsoutil" "github.com/tikv/pd/pkg/utils/typeutil" - "go.etcd.io/etcd/clientv3" + clientv3 "go.etcd.io/etcd/client/v3" "go.uber.org/zap" ) diff --git a/pkg/utils/etcdutil/etcdutil.go b/pkg/utils/etcdutil/etcdutil.go index 2b89c8b4da6..3eb1afabeac 100644 --- a/pkg/utils/etcdutil/etcdutil.go +++ b/pkg/utils/etcdutil/etcdutil.go @@ -33,11 +33,12 @@ import ( "github.com/tikv/pd/pkg/utils/logutil" "github.com/tikv/pd/pkg/utils/syncutil" "github.com/tikv/pd/pkg/utils/typeutil" - "go.etcd.io/etcd/clientv3" - "go.etcd.io/etcd/etcdserver" - "go.etcd.io/etcd/etcdserver/api/v3rpc/rpctypes" - "go.etcd.io/etcd/mvcc/mvccpb" - "go.etcd.io/etcd/pkg/types" + "go.etcd.io/etcd/api/v3/etcdserverpb" + "go.etcd.io/etcd/api/v3/mvccpb" + "go.etcd.io/etcd/api/v3/v3rpc/rpctypes" + etcdtypes "go.etcd.io/etcd/client/pkg/v3/types" + clientv3 "go.etcd.io/etcd/client/v3" + "go.etcd.io/etcd/server/v3/etcdserver" "go.uber.org/zap" "google.golang.org/grpc/codes" ) @@ -74,7 +75,7 @@ const ( // CheckClusterID checks etcd cluster ID, returns an error if mismatch. // This function will never block even quorum is not satisfied. -func CheckClusterID(localClusterID types.ID, um types.URLsMap, tlsConfig *tls.Config) error { +func CheckClusterID(localClusterID etcdtypes.ID, um etcdtypes.URLsMap, tlsConfig *tls.Config) error { if len(um) == 0 { return nil } @@ -88,7 +89,7 @@ func CheckClusterID(localClusterID types.ID, um types.URLsMap, tlsConfig *tls.Co trp := &http.Transport{ TLSClientConfig: tlsConfig, } - remoteCluster, gerr := etcdserver.GetClusterFromRemotePeers(nil, []string{u}, trp, true) + remoteCluster, gerr := etcdserver.GetClusterFromRemotePeers(nil, []string{u}, trp) trp.CloseIdleConnections() if gerr != nil { // Do not return error, because other members may be not ready. @@ -119,12 +120,17 @@ func ListEtcdMembers(ctx context.Context, client *clientv3.Client) (*clientv3.Me time.Sleep(time.Duration(d) * time.Second) }) newCtx, cancel := context.WithTimeout(ctx, DefaultRequestTimeout) - listResp, err := client.MemberList(newCtx) + // After the etcd server is upgraded to v3.5.x, the MemberList API will return the member list in a linearizable way by default. + // It is introduced by https://github.com/etcd-io/etcd/pull/11639 + // If Linearizable is set to false, the member list will be returned with server's local data. + // If Linearizable is set to true, it is served with linearizable guarantee. If the server is disconnected from quorum, `MemberList` call will fail. + c := clientv3.RetryClusterClient(client) + resp, err := c.MemberList(newCtx, &etcdserverpb.MemberListRequest{Linearizable: false}) cancel() if err != nil { - return listResp, errs.ErrEtcdMemberList.Wrap(err).GenWithStackByCause() + return (*clientv3.MemberListResponse)(resp), errs.ErrEtcdMemberList.Wrap(err).GenWithStackByCause() } - return listResp, nil + return (*clientv3.MemberListResponse)(resp), nil } // RemoveEtcdMember removes a member by the given id. diff --git a/pkg/utils/etcdutil/etcdutil_test.go b/pkg/utils/etcdutil/etcdutil_test.go index c402081fa2f..99f71ffde05 100644 --- a/pkg/utils/etcdutil/etcdutil_test.go +++ b/pkg/utils/etcdutil/etcdutil_test.go @@ -34,11 +34,11 @@ import ( "github.com/tikv/pd/pkg/utils/syncutil" "github.com/tikv/pd/pkg/utils/tempurl" "github.com/tikv/pd/pkg/utils/testutil" - "go.etcd.io/etcd/clientv3" - "go.etcd.io/etcd/embed" - "go.etcd.io/etcd/etcdserver/etcdserverpb" - "go.etcd.io/etcd/mvcc/mvccpb" - "go.etcd.io/etcd/pkg/types" + "go.etcd.io/etcd/api/v3/etcdserverpb" + "go.etcd.io/etcd/api/v3/mvccpb" + etcdtypes "go.etcd.io/etcd/client/pkg/v3/types" + clientv3 "go.etcd.io/etcd/client/v3" + "go.etcd.io/etcd/server/v3/embed" "go.uber.org/goleak" ) @@ -46,6 +46,16 @@ func TestMain(m *testing.M) { goleak.VerifyTestMain(m, testutil.LeakOptions...) } +func TestAddMember(t *testing.T) { + re := require.New(t) + servers, client1, clean := NewTestEtcdCluster(t, 1) + defer clean() + etcd1, cfg1 := servers[0], servers[0].Config() + etcd2 := MustAddEtcdMember(t, &cfg1, client1) + defer etcd2.Close() + checkMembers(re, client1, []*embed.Etcd{etcd1, etcd2}) +} + func TestMemberHelpers(t *testing.T) { re := require.New(t) servers, client1, clean := NewTestEtcdCluster(t, 1) @@ -65,7 +75,7 @@ func TestMemberHelpers(t *testing.T) { checkMembers(re, client1, []*embed.Etcd{etcd1, etcd2}) // Test CheckClusterID - urlsMap, err := types.NewURLsMap(etcd2.Config().InitialCluster) + urlsMap, err := etcdtypes.NewURLsMap(etcd2.Config().InitialCluster) re.NoError(err) err = CheckClusterID(etcd1.Server.Cluster().ID(), urlsMap, &tls.Config{MinVersion: tls.VersionTLS12}) re.NoError(err) @@ -172,7 +182,6 @@ func TestEtcdClientSync(t *testing.T) { servers, client1, clean := NewTestEtcdCluster(t, 1) defer clean() etcd1, cfg1 := servers[0], servers[0].Config() - defer etcd1.Close() // Add a new member. etcd2 := MustAddEtcdMember(t, &cfg1, client1) @@ -303,7 +312,7 @@ func checkEtcdWithHangLeader(t *testing.T) error { go proxyWithDiscard(ctx, re, cfg1.ListenClientUrls[0].String(), proxyAddr, &enableDiscard) // Create an etcd client with etcd1 as endpoint. - urls, err := types.NewURLs([]string{proxyAddr}) + urls, err := etcdtypes.NewURLs([]string{proxyAddr}) re.NoError(err) client1, err := CreateEtcdClient(nil, urls) re.NoError(err) @@ -771,7 +780,16 @@ func (suite *loopWatcherTestSuite) startEtcd(re *require.Assertions) { suite.etcd = etcd1 <-etcd1.Server.ReadyNotify() suite.cleans = append(suite.cleans, func() { - suite.etcd.Close() + if suite.etcd.Server != nil { + select { + case _, ok := <-suite.etcd.Err(): + if !ok { + return + } + default: + } + suite.etcd.Close() + } }) } diff --git a/pkg/utils/etcdutil/health_checker.go b/pkg/utils/etcdutil/health_checker.go index c5ece5a0804..a933279ff2b 100644 --- a/pkg/utils/etcdutil/health_checker.go +++ b/pkg/utils/etcdutil/health_checker.go @@ -26,7 +26,7 @@ import ( "github.com/tikv/pd/pkg/errs" "github.com/tikv/pd/pkg/utils/logutil" "github.com/tikv/pd/pkg/utils/typeutil" - "go.etcd.io/etcd/clientv3" + clientv3 "go.etcd.io/etcd/client/v3" "go.uber.org/zap" ) diff --git a/pkg/utils/etcdutil/testutil.go b/pkg/utils/etcdutil/testutil.go index 13c10260a40..e67ae415736 100644 --- a/pkg/utils/etcdutil/testutil.go +++ b/pkg/utils/etcdutil/testutil.go @@ -24,9 +24,9 @@ import ( "github.com/stretchr/testify/require" "github.com/tikv/pd/pkg/utils/tempurl" "github.com/tikv/pd/pkg/utils/testutil" - "go.etcd.io/etcd/clientv3" - "go.etcd.io/etcd/embed" - "go.etcd.io/etcd/etcdserver/etcdserverpb" + "go.etcd.io/etcd/api/v3/etcdserverpb" + clientv3 "go.etcd.io/etcd/client/v3" + "go.etcd.io/etcd/server/v3/embed" ) // NewTestSingleConfig is used to create a etcd config for the unit test purpose. @@ -86,7 +86,14 @@ func NewTestEtcdCluster(t *testing.T, count int) (servers []*embed.Etcd, etcdCli clean = func() { etcdClient.Close() for _, server := range servers { - if server != nil { + if server.Server != nil { + select { + case _, ok := <-server.Err(): + if !ok { + return + } + default: + } server.Close() } } diff --git a/pkg/utils/grpcutil/grpcutil.go b/pkg/utils/grpcutil/grpcutil.go index 5633533ae4a..5f852ff7359 100644 --- a/pkg/utils/grpcutil/grpcutil.go +++ b/pkg/utils/grpcutil/grpcutil.go @@ -27,7 +27,7 @@ import ( "github.com/pingcap/log" "github.com/tikv/pd/pkg/errs" "github.com/tikv/pd/pkg/utils/logutil" - "go.etcd.io/etcd/pkg/transport" + "go.etcd.io/etcd/client/pkg/v3/transport" "go.uber.org/zap" "google.golang.org/grpc" "google.golang.org/grpc/backoff" diff --git a/server/cluster/cluster.go b/server/cluster/cluster.go index a2a365bb4c1..9e12b158f42 100644 --- a/server/cluster/cluster.go +++ b/server/cluster/cluster.go @@ -65,7 +65,7 @@ import ( "github.com/tikv/pd/pkg/utils/typeutil" "github.com/tikv/pd/pkg/versioninfo" "github.com/tikv/pd/server/config" - "go.etcd.io/etcd/clientv3" + clientv3 "go.etcd.io/etcd/client/v3" "go.uber.org/zap" ) diff --git a/server/config/config.go b/server/config/config.go index 81da5f6d9db..8c4f6eaacc8 100644 --- a/server/config/config.go +++ b/server/config/config.go @@ -39,8 +39,8 @@ import ( "github.com/tikv/pd/pkg/utils/metricutil" "github.com/tikv/pd/pkg/utils/typeutil" "github.com/tikv/pd/pkg/versioninfo" - "go.etcd.io/etcd/embed" - "go.etcd.io/etcd/pkg/transport" + "go.etcd.io/etcd/client/pkg/v3/transport" + "go.etcd.io/etcd/server/v3/embed" "go.uber.org/zap" ) @@ -739,6 +739,9 @@ func (c *Config) GenEmbedEtcdConfig() (*embed.Config, error) { cfg.PeerTLSInfo.TrustedCAFile = c.Security.CAPath cfg.PeerTLSInfo.CertFile = c.Security.CertPath cfg.PeerTLSInfo.KeyFile = c.Security.KeyPath + // TODO: After https://github.com/etcd-io/etcd/pull/18015, AllowedCN is Deprecated. + // It will be replaced by AllowedCNs in the future to support multi cn. + // nolint:staticcheck cfg.PeerTLSInfo.AllowedCN = allowedCN cfg.ForceNewCluster = c.ForceNewCluster cfg.ZapLoggerBuilder = embed.NewZapCoreLoggerBuilder(c.Logger, c.Logger.Core(), c.LogProps.Syncer) diff --git a/server/config/persist_options.go b/server/config/persist_options.go index b6963a6645a..19f0ef0d475 100644 --- a/server/config/persist_options.go +++ b/server/config/persist_options.go @@ -38,7 +38,7 @@ import ( "github.com/tikv/pd/pkg/storage/endpoint" "github.com/tikv/pd/pkg/utils/etcdutil" "github.com/tikv/pd/pkg/utils/typeutil" - "go.etcd.io/etcd/clientv3" + clientv3 "go.etcd.io/etcd/client/v3" "go.uber.org/zap" ) diff --git a/server/gc_service.go b/server/gc_service.go index 8c967d542ee..d31e047b1ea 100644 --- a/server/gc_service.go +++ b/server/gc_service.go @@ -28,7 +28,7 @@ import ( "github.com/tikv/pd/pkg/storage/endpoint" "github.com/tikv/pd/pkg/utils/etcdutil" "github.com/tikv/pd/pkg/utils/tsoutil" - "go.etcd.io/etcd/clientv3" + clientv3 "go.etcd.io/etcd/client/v3" "go.uber.org/zap" "google.golang.org/grpc" ) diff --git a/server/grpc_service.go b/server/grpc_service.go index 0f241986a9b..f7a0ea9443e 100644 --- a/server/grpc_service.go +++ b/server/grpc_service.go @@ -47,7 +47,7 @@ import ( "github.com/tikv/pd/pkg/utils/tsoutil" "github.com/tikv/pd/pkg/versioninfo" "github.com/tikv/pd/server/cluster" - "go.etcd.io/etcd/clientv3" + clientv3 "go.etcd.io/etcd/client/v3" "go.uber.org/multierr" "go.uber.org/zap" "google.golang.org/grpc" diff --git a/server/join/join.go b/server/join/join.go index 8da90b7201b..bdc2704cd47 100644 --- a/server/join/join.go +++ b/server/join/join.go @@ -27,8 +27,8 @@ import ( "github.com/tikv/pd/pkg/errs" "github.com/tikv/pd/pkg/utils/etcdutil" "github.com/tikv/pd/server/config" - "go.etcd.io/etcd/clientv3" - "go.etcd.io/etcd/embed" + clientv3 "go.etcd.io/etcd/client/v3" + "go.etcd.io/etcd/server/v3/embed" "go.uber.org/zap" ) diff --git a/server/keyspace_service.go b/server/keyspace_service.go index 44973a83061..afc5b41b1b6 100644 --- a/server/keyspace_service.go +++ b/server/keyspace_service.go @@ -25,8 +25,8 @@ import ( "github.com/tikv/pd/pkg/keyspace" "github.com/tikv/pd/pkg/storage/endpoint" "github.com/tikv/pd/pkg/utils/etcdutil" - "go.etcd.io/etcd/clientv3" - "go.etcd.io/etcd/mvcc/mvccpb" + "go.etcd.io/etcd/api/v3/mvccpb" + clientv3 "go.etcd.io/etcd/client/v3" ) // KeyspaceServer wraps GrpcServer to provide keyspace service. diff --git a/server/server.go b/server/server.go index 6e0179798e8..205d220180f 100644 --- a/server/server.go +++ b/server/server.go @@ -80,10 +80,10 @@ import ( "github.com/tikv/pd/pkg/versioninfo" "github.com/tikv/pd/server/cluster" "github.com/tikv/pd/server/config" - "go.etcd.io/etcd/clientv3" - "go.etcd.io/etcd/embed" - "go.etcd.io/etcd/mvcc/mvccpb" - "go.etcd.io/etcd/pkg/types" + "go.etcd.io/etcd/api/v3/mvccpb" + etcdtypes "go.etcd.io/etcd/client/pkg/v3/types" + clientv3 "go.etcd.io/etcd/client/v3" + "go.etcd.io/etcd/server/v3/embed" "go.uber.org/zap" "google.golang.org/grpc" ) @@ -331,7 +331,7 @@ func (s *Server) startEtcd(ctx context.Context) error { } // Check cluster ID - urlMap, err := types.NewURLsMap(s.cfg.InitialCluster) + urlMap, err := etcdtypes.NewURLsMap(s.cfg.InitialCluster) if err != nil { return errs.ErrEtcdURLMap.Wrap(err).GenWithStackByCause() } @@ -1667,8 +1667,8 @@ func (s *Server) leaderLoop() { log.Info("the pd leader is lost for a long time, try to re-campaign a pd leader with resign etcd leader", zap.Duration("timeout", randomTimeout), zap.Time("last-updated", lastUpdated), - zap.String("current-leader-member-id", types.ID(etcdLeader).String()), - zap.String("transferee-member-id", types.ID(s.member.ID()).String()), + zap.String("current-leader-member-id", etcdtypes.ID(etcdLeader).String()), + zap.String("transferee-member-id", etcdtypes.ID(s.member.ID()).String()), ) if err := s.member.MoveEtcdLeader(s.ctx, etcdLeader, s.member.ID()); err != nil { log.Error("failed to move etcd leader", errs.ZapError(err)) diff --git a/server/server_test.go b/server/server_test.go index 80771721a60..50c116083e0 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -30,8 +30,8 @@ import ( "github.com/tikv/pd/pkg/utils/etcdutil" "github.com/tikv/pd/pkg/utils/testutil" "github.com/tikv/pd/server/config" - "go.etcd.io/etcd/embed" - "go.etcd.io/etcd/pkg/types" + etcdtypes "go.etcd.io/etcd/client/pkg/v3/types" + "go.etcd.io/etcd/server/v3/embed" "go.uber.org/goleak" ) @@ -321,7 +321,7 @@ func TestCheckClusterID(t *testing.T) { etcd, err := embed.StartEtcd(svr.etcdCfg) re.NoError(err) - urlsMap, err := types.NewURLsMap(svr.cfg.InitialCluster) + urlsMap, err := etcdtypes.NewURLsMap(svr.cfg.InitialCluster) re.NoError(err) tlsConfig, err := svr.cfg.Security.ToTLSConfig() re.NoError(err) diff --git a/server/testutil.go b/server/testutil.go index be6b2bbebb0..7b159d7d94b 100644 --- a/server/testutil.go +++ b/server/testutil.go @@ -33,7 +33,7 @@ import ( "github.com/tikv/pd/pkg/utils/testutil" "github.com/tikv/pd/pkg/utils/typeutil" "github.com/tikv/pd/server/config" - "go.etcd.io/etcd/embed" + "go.etcd.io/etcd/server/v3/embed" ) // NewTestServer creates a pd server for testing. diff --git a/tests/cluster.go b/tests/cluster.go index ad796226e31..f9cbb99ce25 100644 --- a/tests/cluster.go +++ b/tests/cluster.go @@ -47,7 +47,7 @@ import ( "github.com/tikv/pd/server/cluster" "github.com/tikv/pd/server/config" "github.com/tikv/pd/server/join" - "go.etcd.io/etcd/clientv3" + clientv3 "go.etcd.io/etcd/client/v3" ) // TestServer states. diff --git a/tests/integrations/client/client_test.go b/tests/integrations/client/client_test.go index 79eec0accde..9517444d017 100644 --- a/tests/integrations/client/client_test.go +++ b/tests/integrations/client/client_test.go @@ -56,7 +56,7 @@ import ( "github.com/tikv/pd/server/config" "github.com/tikv/pd/tests" "github.com/tikv/pd/tests/integrations/mcs" - "go.etcd.io/etcd/clientv3" + clientv3 "go.etcd.io/etcd/client/v3" "go.uber.org/goleak" ) diff --git a/tests/integrations/client/client_tls_test.go b/tests/integrations/client/client_tls_test.go index 091fea2a4c8..b915d77b3b0 100644 --- a/tests/integrations/client/client_tls_test.go +++ b/tests/integrations/client/client_tls_test.go @@ -32,7 +32,7 @@ import ( "github.com/tikv/pd/pkg/utils/netutil" "github.com/tikv/pd/server/config" "github.com/tikv/pd/tests" - "go.etcd.io/etcd/pkg/transport" + "go.etcd.io/etcd/client/pkg/v3/transport" "google.golang.org/grpc" ) diff --git a/tests/integrations/client/gc_client_test.go b/tests/integrations/client/gc_client_test.go index 0913579f47e..8494b8edbcf 100644 --- a/tests/integrations/client/gc_client_test.go +++ b/tests/integrations/client/gc_client_test.go @@ -30,7 +30,7 @@ import ( "github.com/tikv/pd/pkg/utils/assertutil" "github.com/tikv/pd/pkg/utils/testutil" "github.com/tikv/pd/server" - "go.etcd.io/etcd/clientv3" + clientv3 "go.etcd.io/etcd/client/v3" "go.uber.org/zap" "google.golang.org/grpc" ) diff --git a/tests/integrations/go.mod b/tests/integrations/go.mod index f56b1cdc32a..d7305379e9f 100644 --- a/tests/integrations/go.mod +++ b/tests/integrations/go.mod @@ -21,7 +21,8 @@ require ( github.com/stretchr/testify v1.8.4 github.com/tikv/pd v0.0.0-00010101000000-000000000000 github.com/tikv/pd/client v0.0.0-00010101000000-000000000000 - go.etcd.io/etcd v0.5.0-alpha.5.0.20240320135013-950cd5fbe6ca + go.etcd.io/etcd/client/pkg/v3 v3.5.15 + go.etcd.io/etcd/client/v3 v3.5.15 go.uber.org/goleak v1.3.0 go.uber.org/zap v1.27.0 google.golang.org/grpc v1.62.1 @@ -57,13 +58,12 @@ require ( github.com/breeswish/gin-jwt/v2 v2.6.4-jwt-patch // indirect github.com/bytedance/sonic v1.9.1 // indirect github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 // indirect - github.com/cenkalti/backoff/v4 v4.0.2 // indirect + github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect github.com/cloudfoundry/gosigar v1.3.6 // indirect github.com/coreos/go-semver v0.3.1 // indirect - github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf // indirect - github.com/coreos/pkg v0.0.0-20240122114842-bbd7aa9bf6fb // indirect + github.com/coreos/go-systemd/v22 v22.3.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/elliotchance/pie/v2 v2.1.0 // indirect @@ -74,6 +74,8 @@ require ( github.com/gin-contrib/pprof v1.4.0 // indirect github.com/gin-contrib/sse v0.1.0 // indirect github.com/gin-gonic/gin v1.9.1 // indirect + github.com/go-logr/logr v1.3.0 // indirect + github.com/go-logr/stdr v1.2.2 // indirect github.com/go-ole/go-ole v1.2.6 // indirect github.com/go-openapi/jsonpointer v0.19.5 // indirect github.com/go-openapi/jsonreference v0.19.6 // indirect @@ -87,6 +89,7 @@ require ( github.com/goccy/go-json v0.10.2 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt v3.2.2+incompatible // indirect + github.com/golang-jwt/jwt/v4 v4.4.2 // indirect github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/golang/snappy v0.0.4 // indirect @@ -98,6 +101,7 @@ require ( github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 // indirect github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 // indirect github.com/gtank/cryptopasta v0.0.0-20170601214702-1f550f6f2f69 // indirect github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect @@ -125,7 +129,7 @@ require ( github.com/phf/go-queue v0.0.0-20170504031614-9abe38d0371d // indirect github.com/pingcap/errcode v0.3.0 // indirect github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21 // indirect - github.com/pingcap/tidb-dashboard v0.0.0-20240718034516-e6e78c7c120b // indirect + github.com/pingcap/tidb-dashboard v0.0.0-20240815045040-4d89bc193a0c // indirect github.com/pingcap/tipb v0.0.0-20220718022156-3e2483c20a9e // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect @@ -159,7 +163,20 @@ require ( github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect github.com/xiang90/probing v0.0.0-20221125231312-a49e3df8f510 // indirect github.com/yusufpapurcu/wmi v1.2.2 // indirect - go.etcd.io/bbolt v1.3.9 // indirect + go.etcd.io/bbolt v1.3.10 // indirect + go.etcd.io/etcd/api/v3 v3.5.15 // indirect + go.etcd.io/etcd/client/v2 v2.305.15 // indirect + go.etcd.io/etcd/pkg/v3 v3.5.15 // indirect + go.etcd.io/etcd/raft/v3 v3.5.15 // indirect + go.etcd.io/etcd/server/v3 v3.5.15 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.0 // indirect + go.opentelemetry.io/otel v1.20.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.20.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.20.0 // indirect + go.opentelemetry.io/otel/metric v1.20.0 // indirect + go.opentelemetry.io/otel/sdk v1.20.0 // indirect + go.opentelemetry.io/otel/trace v1.20.0 // indirect + go.opentelemetry.io/proto/otlp v1.0.0 // indirect go.uber.org/atomic v1.10.0 // indirect go.uber.org/dig v1.9.0 // indirect go.uber.org/fx v1.12.0 // indirect diff --git a/tests/integrations/go.sum b/tests/integrations/go.sum index e23f71d41ce..914ce1ff35c 100644 --- a/tests/integrations/go.sum +++ b/tests/integrations/go.sum @@ -1,5 +1,10 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.112.2 h1:ZaGT6LiG7dBzi6zNOvVZwacaXlmf3lRqnC4DQzqyRQw= +cloud.google.com/go/compute v1.25.1 h1:ZRpHJedLtTpKgr3RV1Fx23NuaAEN1Zfx9hw1u4aJdjU= +cloud.google.com/go/compute v1.25.1/go.mod h1:oopOIR53ly6viBYxaDhBfJwzUAxf1zE//uf3IB011ls= +cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= +cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60= @@ -65,8 +70,8 @@ github.com/bytedance/sonic v1.9.1 h1:6iJ6NqdoxCDr6mbY8h18oSO+cShGSMRGCEo7F2h0x8s github.com/bytedance/sonic v1.9.1/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZXU064P/U= github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 h1:BjkPE3785EwPhhyuFkbINB+2a1xATwk8SNDWnJiD41g= github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5/go.mod h1:jtAfVaU/2cu1+wdSRPWE2c1N2qeAA3K4RH9pYgqwets= -github.com/cenkalti/backoff/v4 v4.0.2 h1:JIufpQLbh4DkbQoii76ItQIUFzevQSqOLZca4eamEDs= -github.com/cenkalti/backoff/v4 v4.0.2/go.mod h1:eEew/i+1Q6OrCDZh3WiXYv3+nJwBASZ8Bog/87DQnVg= +github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= +github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -80,14 +85,14 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cloudfoundry/gosigar v1.3.6 h1:gIc08FbB3QPb+nAQhINIK/qhf5REKkY0FTGgRGXkcVc= github.com/cloudfoundry/gosigar v1.3.6/go.mod h1:lNWstu5g5gw59O09Y+wsMNFzBSnU8a0u+Sfx4dq360E= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa h1:OaNxuTZr7kxeODyLWsRMC+OD03aFUH+mW6r2d+MWa5Y= -github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= +github.com/cncf/xds/go v0.0.0-20231128003011-0fa0005c9caa h1:jQCWAUqqlij9Pgj2i/PB79y4KOPYVyFYdROxgaCwdTQ= +github.com/cncf/xds/go v0.0.0-20231128003011-0fa0005c9caa/go.mod h1:x/1Gn8zydmfq8dk6e9PdstVsDgu9RuyIIJqAaF//0IM= +github.com/cockroachdb/datadriven v1.0.2 h1:H9MtNqVoVhvd9nCBwOyDjUEdZCREqbIdCJD93PBm/jA= +github.com/cockroachdb/datadriven v1.0.2/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU= github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr4= github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03VsM8rvUec= -github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf h1:iW4rZ826su+pqaw19uhpSCzhj44qo35pNgKFGqzDKkU= -github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/pkg v0.0.0-20240122114842-bbd7aa9bf6fb h1:GIzvVQ9UkUlOhSDlqmrQAAAUd6R3E+caIisNEyWXvNE= -github.com/coreos/pkg v0.0.0-20240122114842-bbd7aa9bf6fb/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/coreos/go-systemd/v22 v22.3.2 h1:D9/bQk5vlXQFZ6Kwuu6zaiXJ9oTPe68++AzAJc1DzSI= +github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/corona10/goimagehash v1.0.2 h1:pUfB0LnsJASMPGEZLj7tGY251vF+qLGqOgEP4rUs6kA= github.com/corona10/goimagehash v1.0.2/go.mod h1:/l9umBhvcHQXVtQO1V6Gp1yD20STawkhRnnX0D1bvVI= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= @@ -108,6 +113,8 @@ github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymF github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/envoyproxy/protoc-gen-validate v1.0.4 h1:gVPz/FMfvh57HdSJQyvBtF00j8JU4zdyUgIUNhlgg0A= +github.com/envoyproxy/protoc-gen-validate v1.0.4/go.mod h1:qys6tmnRsYrQqIhm2bvKZH4Blx/1gTIZ2UKVY1M+Yew= github.com/fogleman/gg v1.3.0 h1:/7zJX8F6AaYQc57WQCyN9cAIz+4bCJGO9B+dyW29am8= github.com/fogleman/gg v1.3.0/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= @@ -132,6 +139,11 @@ github.com/gin-gonic/gin v1.9.1 h1:4idEAncQnU5cB7BeOkPtxjfCSye0AAm1R0RVIqJ+Jmg= github.com/gin-gonic/gin v1.9.1/go.mod h1:hPrL7YrpYKXt5YId3A/Tnip5kqbEAP+KLuI3SUcPTeU= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY= +github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= @@ -169,6 +181,7 @@ github.com/goccy/go-graphviz v0.1.3/go.mod h1:pMYpbAqJT10V8dzV1JN/g/wUlG/0imKPzn github.com/goccy/go-json v0.9.7/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v0.0.0-20171007142547-342cbe0a0415/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v0.0.0-20180717141946-636bf0302bc9/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= @@ -177,6 +190,8 @@ github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69 github.com/golang-jwt/jwt v3.2.1+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY= github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= +github.com/golang-jwt/jwt/v4 v4.4.2 h1:rcc4lwaZgFMCZ5jxF9ABolDcIHdBytAFgqFPbSJQAYs= +github.com/golang-jwt/jwt/v4 v4.4.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 h1:au07oEsX2xN0ktxqI+Sida1w446QrXBRJ0nee3SNZlA= github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= github.com/golang-sql/sqlexp v0.1.0 h1:ZCD6MBpcuOVfGVqsEmY5/4FtYiKz6tSyUv9LPEDei6A= @@ -184,8 +199,8 @@ github.com/golang-sql/sqlexp v0.1.0/go.mod h1:J4ad9Vo8ZCWQ2GMrC4UCQy1JpCbwU9m3EO github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 h1:DACJavvAHhabrF08vX0COfcOBJRhZ8lUbR+ZWIs0Y5g= github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903 h1:LbsanbbD6LieFkXbj9YNNBupiGHJgFeLpO0j0Fza1h8= -github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/glog v1.2.0 h1:uCdmnmatrKCgMBlM4rMuJZWOkPDqdbZPnrMXDY4gI68= +github.com/golang/glog v1.2.0/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v0.0.0-20180814211427-aa810b61a9c7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -228,6 +243,8 @@ github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92Bcuy github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 h1:YBftPWNWd4WwGqtY2yeZL2ef8rHAxPBD8KFhJpmcqms= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0/go.mod h1:YN5jB8ie0yfIUg6VvR9Kz84aCaG7AsGZnLjhHbUqwPg= github.com/gtank/cryptopasta v0.0.0-20170601214702-1f550f6f2f69 h1:7xsUJsB2NrdcttQPa7JLEaGzvdbk7KvfrjgHZXOQRo0= github.com/gtank/cryptopasta v0.0.0-20170601214702-1f550f6f2f69/go.mod h1:YLEMZOtU+AZ7dhN9T/IpGhXVGly2bvkJQ+zxj3WeVQo= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= @@ -374,8 +391,8 @@ github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 h1:HR/ylkkLmGdSSDaD8 github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4= github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21 h1:QV6jqlfOkh8hqvEAgwBZa+4bSgO0EeKC7s5c6Luam2I= github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21/go.mod h1:QYnjfA95ZaMefyl1NO8oPtKeb8pYUdnDVhQgf+qdpjM= -github.com/pingcap/tidb-dashboard v0.0.0-20240718034516-e6e78c7c120b h1:MKgJ9yCQxD5ewLERuoiiD9XVOHuqZ2WRZnB20yMiKyo= -github.com/pingcap/tidb-dashboard v0.0.0-20240718034516-e6e78c7c120b/go.mod h1:ucZBRz52icb23T/5Z4CsuUHmarYiin7p2MeiVBe+o8c= +github.com/pingcap/tidb-dashboard v0.0.0-20240815045040-4d89bc193a0c h1:iiVpATdlLLr0NEgKZu+ZnBuYR4J8IgjNE1hNMjMOkYY= +github.com/pingcap/tidb-dashboard v0.0.0-20240815045040-4d89bc193a0c/go.mod h1:AT9vfeojwr/GGCHTURXtA8yZBE9AW8LdIo02/eYdfHU= github.com/pingcap/tipb v0.0.0-20220718022156-3e2483c20a9e h1:FBaTXU8C3xgt/drM58VHxojHo/QoG1oPsgWTGvaSpO4= github.com/pingcap/tipb v0.0.0-20220718022156-3e2483c20a9e/go.mod h1:A7mrd7WHBl1o63LE2bIBGEJMTNWXqhgmYiOvMLxozfs= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= @@ -504,10 +521,38 @@ github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg= github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= -go.etcd.io/bbolt v1.3.9 h1:8x7aARPEXiXbHmtUwAIv7eV2fQFHrLLavdiJ3uzJXoI= -go.etcd.io/bbolt v1.3.9/go.mod h1:zaO32+Ti0PK1ivdPtgMESzuzL2VPoIG1PCQNvOdo/dE= -go.etcd.io/etcd v0.5.0-alpha.5.0.20240320135013-950cd5fbe6ca h1:LCc0GAhfJ+qDqnUbE7ybQ0mTz1dNRn2iiM6e183p/5E= -go.etcd.io/etcd v0.5.0-alpha.5.0.20240320135013-950cd5fbe6ca/go.mod h1:1AyK+XVcIwjbjw5EYrhT+IiMYSgRZTohGb2ceZ0/US8= +go.etcd.io/bbolt v1.3.10 h1:+BqfJTcCzTItrop8mq/lbzL8wSGtj94UO/3U31shqG0= +go.etcd.io/bbolt v1.3.10/go.mod h1:bK3UQLPJZly7IlNmV7uVHJDxfe5aK9Ll93e/74Y9oEQ= +go.etcd.io/etcd/api/v3 v3.5.15 h1:3KpLJir1ZEBrYuV2v+Twaa/e2MdDCEZ/70H+lzEiwsk= +go.etcd.io/etcd/api/v3 v3.5.15/go.mod h1:N9EhGzXq58WuMllgH9ZvnEr7SI9pS0k0+DHZezGp7jM= +go.etcd.io/etcd/client/pkg/v3 v3.5.15 h1:fo0HpWz/KlHGMCC+YejpiCmyWDEuIpnTDzpJLB5fWlA= +go.etcd.io/etcd/client/pkg/v3 v3.5.15/go.mod h1:mXDI4NAOwEiszrHCb0aqfAYNCrZP4e9hRca3d1YK8EU= +go.etcd.io/etcd/client/v2 v2.305.15 h1:VG2xbf8Vz1KJh65Ar2V5eDmfkp1bpzkSEHlhJM3usp8= +go.etcd.io/etcd/client/v2 v2.305.15/go.mod h1:Ad5dRjPVb/n5yXgAWQ/hXzuXXkBk0Y658ocuXYaUU48= +go.etcd.io/etcd/client/v3 v3.5.15 h1:23M0eY4Fd/inNv1ZfU3AxrbbOdW79r9V9Rl62Nm6ip4= +go.etcd.io/etcd/client/v3 v3.5.15/go.mod h1:CLSJxrYjvLtHsrPKsy7LmZEE+DK2ktfd2bN4RhBMwlU= +go.etcd.io/etcd/pkg/v3 v3.5.15 h1:/Iu6Sr3iYaAjy++8sIDoZW9/EfhcwLZwd4FOZX2mMOU= +go.etcd.io/etcd/pkg/v3 v3.5.15/go.mod h1:e3Acf298sPFmTCGTrnGvkClEw9RYIyPtNzi1XM8rets= +go.etcd.io/etcd/raft/v3 v3.5.15 h1:jOA2HJF7zb3wy8H/pL13e8geWqkEa/kUs0waUggZC0I= +go.etcd.io/etcd/raft/v3 v3.5.15/go.mod h1:k3r7P4seEiUcgxOPLp+mloJWV3Q4QLPGNvy/OgC8OtM= +go.etcd.io/etcd/server/v3 v3.5.15 h1:x35jrWnZgsRwMsFsUJIUdT1bvzIz1B+29HjMfRYVN/E= +go.etcd.io/etcd/server/v3 v3.5.15/go.mod h1:l9jX9oa/iuArjqz0RNX/TDbc70dLXxRZo/nmPucrpFo= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.0 h1:PzIubN4/sjByhDRHLviCjJuweBXWFZWhghjg7cS28+M= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.0/go.mod h1:Ct6zzQEuGK3WpJs2n4dn+wfJYzd/+hNnxMRTWjGn30M= +go.opentelemetry.io/otel v1.20.0 h1:vsb/ggIY+hUjD/zCAQHpzTmndPqv/ml2ArbsbfBYTAc= +go.opentelemetry.io/otel v1.20.0/go.mod h1:oUIGj3D77RwJdM6PPZImDpSZGDvkD9fhesHny69JFrs= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.20.0 h1:DeFD0VgTZ+Cj6hxravYYZE2W4GlneVH81iAOPjZkzk8= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.20.0/go.mod h1:GijYcYmNpX1KazD5JmWGsi4P7dDTTTnfv1UbGn84MnU= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.20.0 h1:gvmNvqrPYovvyRmCSygkUDyL8lC5Tl845MLEwqpxhEU= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.20.0/go.mod h1:vNUq47TGFioo+ffTSnKNdob241vePmtNZnAODKapKd0= +go.opentelemetry.io/otel/metric v1.20.0 h1:ZlrO8Hu9+GAhnepmRGhSU7/VkpjrNowxRN9GyKR4wzA= +go.opentelemetry.io/otel/metric v1.20.0/go.mod h1:90DRw3nfK4D7Sm/75yQ00gTJxtkBxX+wu6YaNymbpVM= +go.opentelemetry.io/otel/sdk v1.20.0 h1:5Jf6imeFZlZtKv9Qbo6qt2ZkmWtdWx/wzcCbNUlAWGM= +go.opentelemetry.io/otel/sdk v1.20.0/go.mod h1:rmkSx1cZCm/tn16iWDn1GQbLtsW/LvsdEEFzCSRM6V0= +go.opentelemetry.io/otel/trace v1.20.0 h1:+yxVAPZPbQhbC3OfAkeIVTky6iTFpcr4SiY9om7mXSQ= +go.opentelemetry.io/otel/trace v1.20.0/go.mod h1:HJSK7F/hA5RlzpZ0zKDCHCDHm556LCDtKaAo6JmBFUU= +go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I= +go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= diff --git a/tests/integrations/mcs/scheduling/server_test.go b/tests/integrations/mcs/scheduling/server_test.go index ab6bb93c60c..5c08892a972 100644 --- a/tests/integrations/mcs/scheduling/server_test.go +++ b/tests/integrations/mcs/scheduling/server_test.go @@ -675,11 +675,14 @@ func (suite *multipleServerTestSuite) TestReElectLeader() { newLeaderName := suite.cluster.WaitLeader() re.NotEqual(originLeaderName, newLeaderName) + suite.pdLeader = suite.cluster.GetServer(newLeaderName) suite.pdLeader.ResignLeader() newLeaderName = suite.cluster.WaitLeader() re.Equal(originLeaderName, newLeaderName) + suite.pdLeader = suite.cluster.GetServer(newLeaderName) rc = suite.pdLeader.GetServer().GetRaftCluster() + re.NotNil(rc) rc.IsPrepared() } diff --git a/tests/integrations/mcs/tso/server_test.go b/tests/integrations/mcs/tso/server_test.go index 980f0db6f12..7ab92026197 100644 --- a/tests/integrations/mcs/tso/server_test.go +++ b/tests/integrations/mcs/tso/server_test.go @@ -42,7 +42,7 @@ import ( "github.com/tikv/pd/pkg/utils/tsoutil" "github.com/tikv/pd/tests" "github.com/tikv/pd/tests/integrations/mcs" - "go.etcd.io/etcd/clientv3" + clientv3 "go.etcd.io/etcd/client/v3" "go.uber.org/goleak" "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" diff --git a/tests/server/tso/manager_test.go b/tests/server/tso/manager_test.go index c4f5aa5e3e3..1feb74e6643 100644 --- a/tests/server/tso/manager_test.go +++ b/tests/server/tso/manager_test.go @@ -30,7 +30,7 @@ import ( "github.com/tikv/pd/pkg/utils/testutil" "github.com/tikv/pd/server/config" "github.com/tikv/pd/tests" - "go.etcd.io/etcd/clientv3" + clientv3 "go.etcd.io/etcd/client/v3" ) // TestClusterDCLocations will write different dc-locations to each server diff --git a/tools/go.mod b/tools/go.mod index 7152d25fed9..b1a3c1e3543 100644 --- a/tools/go.mod +++ b/tools/go.mod @@ -31,7 +31,10 @@ require ( github.com/stretchr/testify v1.8.4 github.com/tikv/pd v0.0.0-00010101000000-000000000000 github.com/tikv/pd/client v0.0.0-00010101000000-000000000000 - go.etcd.io/etcd v0.5.0-alpha.5.0.20240320135013-950cd5fbe6ca + go.etcd.io/etcd/client/pkg/v3 v3.5.15 + go.etcd.io/etcd/client/v3 v3.5.15 + go.etcd.io/etcd/pkg/v3 v3.5.15 + go.etcd.io/etcd/server/v3 v3.5.15 go.uber.org/automaxprocs v1.5.3 go.uber.org/goleak v1.3.0 go.uber.org/zap v1.27.0 @@ -65,17 +68,18 @@ require ( github.com/bitly/go-simplejson v0.5.0 // indirect github.com/breeswish/gin-jwt/v2 v2.6.4-jwt-patch // indirect github.com/bytedance/sonic v1.9.1 // indirect - github.com/cenkalti/backoff/v4 v4.0.2 // indirect + github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect - github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf // indirect - github.com/coreos/pkg v0.0.0-20240122114842-bbd7aa9bf6fb // indirect + github.com/coreos/go-systemd/v22 v22.3.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/elliotchance/pie/v2 v2.1.0 // indirect github.com/fogleman/gg v1.3.0 // indirect github.com/gabriel-vasile/mimetype v1.4.2 // indirect github.com/gin-contrib/sse v0.1.0 // indirect + github.com/go-logr/logr v1.3.0 // indirect + github.com/go-logr/stdr v1.2.2 // indirect github.com/go-ole/go-ole v1.2.6 // indirect github.com/go-openapi/jsonpointer v0.19.5 // indirect github.com/go-openapi/jsonreference v0.19.6 // indirect @@ -90,6 +94,7 @@ require ( github.com/goccy/go-json v0.10.2 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt v3.2.2+incompatible // indirect + github.com/golang-jwt/jwt/v4 v4.4.2 // indirect github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/golang/snappy v0.0.4 // indirect @@ -101,6 +106,7 @@ require ( github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 // indirect github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 // indirect github.com/gtank/cryptopasta v0.0.0-20170601214702-1f550f6f2f69 // indirect github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect @@ -127,7 +133,7 @@ require ( github.com/phf/go-queue v0.0.0-20170504031614-9abe38d0371d // indirect github.com/pingcap/errcode v0.3.0 // indirect github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21 // indirect - github.com/pingcap/tidb-dashboard v0.0.0-20240718034516-e6e78c7c120b // indirect + github.com/pingcap/tidb-dashboard v0.0.0-20240815045040-4d89bc193a0c // indirect github.com/pingcap/tipb v0.0.0-20220718022156-3e2483c20a9e // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect @@ -159,7 +165,18 @@ require ( github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect github.com/xiang90/probing v0.0.0-20221125231312-a49e3df8f510 // indirect github.com/yusufpapurcu/wmi v1.2.2 // indirect - go.etcd.io/bbolt v1.3.9 // indirect + go.etcd.io/bbolt v1.3.10 // indirect + go.etcd.io/etcd/api/v3 v3.5.15 // indirect + go.etcd.io/etcd/client/v2 v2.305.15 // indirect + go.etcd.io/etcd/raft/v3 v3.5.15 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.0 // indirect + go.opentelemetry.io/otel v1.20.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.20.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.20.0 // indirect + go.opentelemetry.io/otel/metric v1.20.0 // indirect + go.opentelemetry.io/otel/sdk v1.20.0 // indirect + go.opentelemetry.io/otel/trace v1.20.0 // indirect + go.opentelemetry.io/proto/otlp v1.0.0 // indirect go.uber.org/atomic v1.10.0 // indirect go.uber.org/dig v1.9.0 // indirect go.uber.org/fx v1.12.0 // indirect diff --git a/tools/go.sum b/tools/go.sum index d9bc29aebff..0933f7ceda7 100644 --- a/tools/go.sum +++ b/tools/go.sum @@ -1,5 +1,10 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.112.2 h1:ZaGT6LiG7dBzi6zNOvVZwacaXlmf3lRqnC4DQzqyRQw= +cloud.google.com/go/compute v1.25.1 h1:ZRpHJedLtTpKgr3RV1Fx23NuaAEN1Zfx9hw1u4aJdjU= +cloud.google.com/go/compute v1.25.1/go.mod h1:oopOIR53ly6viBYxaDhBfJwzUAxf1zE//uf3IB011ls= +cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= +cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc= @@ -63,8 +68,8 @@ github.com/bytedance/sonic v1.9.1 h1:6iJ6NqdoxCDr6mbY8h18oSO+cShGSMRGCEo7F2h0x8s github.com/bytedance/sonic v1.9.1/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZXU064P/U= github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 h1:BjkPE3785EwPhhyuFkbINB+2a1xATwk8SNDWnJiD41g= github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5/go.mod h1:jtAfVaU/2cu1+wdSRPWE2c1N2qeAA3K4RH9pYgqwets= -github.com/cenkalti/backoff/v4 v4.0.2 h1:JIufpQLbh4DkbQoii76ItQIUFzevQSqOLZca4eamEDs= -github.com/cenkalti/backoff/v4 v4.0.2/go.mod h1:eEew/i+1Q6OrCDZh3WiXYv3+nJwBASZ8Bog/87DQnVg= +github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= +github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -79,14 +84,14 @@ github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1 h1:q763qf9huN11kDQavWs github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa h1:OaNxuTZr7kxeODyLWsRMC+OD03aFUH+mW6r2d+MWa5Y= -github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= +github.com/cncf/xds/go v0.0.0-20231128003011-0fa0005c9caa h1:jQCWAUqqlij9Pgj2i/PB79y4KOPYVyFYdROxgaCwdTQ= +github.com/cncf/xds/go v0.0.0-20231128003011-0fa0005c9caa/go.mod h1:x/1Gn8zydmfq8dk6e9PdstVsDgu9RuyIIJqAaF//0IM= +github.com/cockroachdb/datadriven v1.0.2 h1:H9MtNqVoVhvd9nCBwOyDjUEdZCREqbIdCJD93PBm/jA= +github.com/cockroachdb/datadriven v1.0.2/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU= github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr4= github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03VsM8rvUec= -github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf h1:iW4rZ826su+pqaw19uhpSCzhj44qo35pNgKFGqzDKkU= -github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/pkg v0.0.0-20240122114842-bbd7aa9bf6fb h1:GIzvVQ9UkUlOhSDlqmrQAAAUd6R3E+caIisNEyWXvNE= -github.com/coreos/pkg v0.0.0-20240122114842-bbd7aa9bf6fb/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/coreos/go-systemd/v22 v22.3.2 h1:D9/bQk5vlXQFZ6Kwuu6zaiXJ9oTPe68++AzAJc1DzSI= +github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/corona10/goimagehash v1.0.2 h1:pUfB0LnsJASMPGEZLj7tGY251vF+qLGqOgEP4rUs6kA= github.com/corona10/goimagehash v1.0.2/go.mod h1:/l9umBhvcHQXVtQO1V6Gp1yD20STawkhRnnX0D1bvVI= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= @@ -107,6 +112,8 @@ github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymF github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/envoyproxy/protoc-gen-validate v1.0.4 h1:gVPz/FMfvh57HdSJQyvBtF00j8JU4zdyUgIUNhlgg0A= +github.com/envoyproxy/protoc-gen-validate v1.0.4/go.mod h1:qys6tmnRsYrQqIhm2bvKZH4Blx/1gTIZ2UKVY1M+Yew= github.com/fogleman/gg v1.3.0 h1:/7zJX8F6AaYQc57WQCyN9cAIz+4bCJGO9B+dyW29am8= github.com/fogleman/gg v1.3.0/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= @@ -131,6 +138,11 @@ github.com/go-echarts/go-echarts v1.0.0 h1:n181E4iXwj4zrU9VYmdM2m8dyhERt2w9k9YhH github.com/go-echarts/go-echarts v1.0.0/go.mod h1:qbmyAb/Rl1f2w7wKba1D4LoNq4U164yO4/wedFbcWyo= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY= +github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= @@ -168,6 +180,7 @@ github.com/goccy/go-graphviz v0.1.3/go.mod h1:pMYpbAqJT10V8dzV1JN/g/wUlG/0imKPzn github.com/goccy/go-json v0.9.7/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v0.0.0-20171007142547-342cbe0a0415/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v0.0.0-20180717141946-636bf0302bc9/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= @@ -175,6 +188,8 @@ github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69 github.com/golang-jwt/jwt v3.2.1+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY= github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= +github.com/golang-jwt/jwt/v4 v4.4.2 h1:rcc4lwaZgFMCZ5jxF9ABolDcIHdBytAFgqFPbSJQAYs= +github.com/golang-jwt/jwt/v4 v4.4.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 h1:au07oEsX2xN0ktxqI+Sida1w446QrXBRJ0nee3SNZlA= github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= github.com/golang-sql/sqlexp v0.1.0 h1:ZCD6MBpcuOVfGVqsEmY5/4FtYiKz6tSyUv9LPEDei6A= @@ -182,8 +197,8 @@ github.com/golang-sql/sqlexp v0.1.0/go.mod h1:J4ad9Vo8ZCWQ2GMrC4UCQy1JpCbwU9m3EO github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 h1:DACJavvAHhabrF08vX0COfcOBJRhZ8lUbR+ZWIs0Y5g= github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903 h1:LbsanbbD6LieFkXbj9YNNBupiGHJgFeLpO0j0Fza1h8= -github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/glog v1.2.0 h1:uCdmnmatrKCgMBlM4rMuJZWOkPDqdbZPnrMXDY4gI68= +github.com/golang/glog v1.2.0/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v0.0.0-20180814211427-aa810b61a9c7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -226,6 +241,8 @@ github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92Bcuy github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 h1:YBftPWNWd4WwGqtY2yeZL2ef8rHAxPBD8KFhJpmcqms= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0/go.mod h1:YN5jB8ie0yfIUg6VvR9Kz84aCaG7AsGZnLjhHbUqwPg= github.com/gtank/cryptopasta v0.0.0-20170601214702-1f550f6f2f69 h1:7xsUJsB2NrdcttQPa7JLEaGzvdbk7KvfrjgHZXOQRo0= github.com/gtank/cryptopasta v0.0.0-20170601214702-1f550f6f2f69/go.mod h1:YLEMZOtU+AZ7dhN9T/IpGhXVGly2bvkJQ+zxj3WeVQo= github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= @@ -371,8 +388,8 @@ github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 h1:HR/ylkkLmGdSSDaD8 github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4= github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21 h1:QV6jqlfOkh8hqvEAgwBZa+4bSgO0EeKC7s5c6Luam2I= github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21/go.mod h1:QYnjfA95ZaMefyl1NO8oPtKeb8pYUdnDVhQgf+qdpjM= -github.com/pingcap/tidb-dashboard v0.0.0-20240718034516-e6e78c7c120b h1:MKgJ9yCQxD5ewLERuoiiD9XVOHuqZ2WRZnB20yMiKyo= -github.com/pingcap/tidb-dashboard v0.0.0-20240718034516-e6e78c7c120b/go.mod h1:ucZBRz52icb23T/5Z4CsuUHmarYiin7p2MeiVBe+o8c= +github.com/pingcap/tidb-dashboard v0.0.0-20240815045040-4d89bc193a0c h1:iiVpATdlLLr0NEgKZu+ZnBuYR4J8IgjNE1hNMjMOkYY= +github.com/pingcap/tidb-dashboard v0.0.0-20240815045040-4d89bc193a0c/go.mod h1:AT9vfeojwr/GGCHTURXtA8yZBE9AW8LdIo02/eYdfHU= github.com/pingcap/tipb v0.0.0-20220718022156-3e2483c20a9e h1:FBaTXU8C3xgt/drM58VHxojHo/QoG1oPsgWTGvaSpO4= github.com/pingcap/tipb v0.0.0-20220718022156-3e2483c20a9e/go.mod h1:A7mrd7WHBl1o63LE2bIBGEJMTNWXqhgmYiOvMLxozfs= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= @@ -504,10 +521,38 @@ github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg= github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= -go.etcd.io/bbolt v1.3.9 h1:8x7aARPEXiXbHmtUwAIv7eV2fQFHrLLavdiJ3uzJXoI= -go.etcd.io/bbolt v1.3.9/go.mod h1:zaO32+Ti0PK1ivdPtgMESzuzL2VPoIG1PCQNvOdo/dE= -go.etcd.io/etcd v0.5.0-alpha.5.0.20240320135013-950cd5fbe6ca h1:LCc0GAhfJ+qDqnUbE7ybQ0mTz1dNRn2iiM6e183p/5E= -go.etcd.io/etcd v0.5.0-alpha.5.0.20240320135013-950cd5fbe6ca/go.mod h1:1AyK+XVcIwjbjw5EYrhT+IiMYSgRZTohGb2ceZ0/US8= +go.etcd.io/bbolt v1.3.10 h1:+BqfJTcCzTItrop8mq/lbzL8wSGtj94UO/3U31shqG0= +go.etcd.io/bbolt v1.3.10/go.mod h1:bK3UQLPJZly7IlNmV7uVHJDxfe5aK9Ll93e/74Y9oEQ= +go.etcd.io/etcd/api/v3 v3.5.15 h1:3KpLJir1ZEBrYuV2v+Twaa/e2MdDCEZ/70H+lzEiwsk= +go.etcd.io/etcd/api/v3 v3.5.15/go.mod h1:N9EhGzXq58WuMllgH9ZvnEr7SI9pS0k0+DHZezGp7jM= +go.etcd.io/etcd/client/pkg/v3 v3.5.15 h1:fo0HpWz/KlHGMCC+YejpiCmyWDEuIpnTDzpJLB5fWlA= +go.etcd.io/etcd/client/pkg/v3 v3.5.15/go.mod h1:mXDI4NAOwEiszrHCb0aqfAYNCrZP4e9hRca3d1YK8EU= +go.etcd.io/etcd/client/v2 v2.305.15 h1:VG2xbf8Vz1KJh65Ar2V5eDmfkp1bpzkSEHlhJM3usp8= +go.etcd.io/etcd/client/v2 v2.305.15/go.mod h1:Ad5dRjPVb/n5yXgAWQ/hXzuXXkBk0Y658ocuXYaUU48= +go.etcd.io/etcd/client/v3 v3.5.15 h1:23M0eY4Fd/inNv1ZfU3AxrbbOdW79r9V9Rl62Nm6ip4= +go.etcd.io/etcd/client/v3 v3.5.15/go.mod h1:CLSJxrYjvLtHsrPKsy7LmZEE+DK2ktfd2bN4RhBMwlU= +go.etcd.io/etcd/pkg/v3 v3.5.15 h1:/Iu6Sr3iYaAjy++8sIDoZW9/EfhcwLZwd4FOZX2mMOU= +go.etcd.io/etcd/pkg/v3 v3.5.15/go.mod h1:e3Acf298sPFmTCGTrnGvkClEw9RYIyPtNzi1XM8rets= +go.etcd.io/etcd/raft/v3 v3.5.15 h1:jOA2HJF7zb3wy8H/pL13e8geWqkEa/kUs0waUggZC0I= +go.etcd.io/etcd/raft/v3 v3.5.15/go.mod h1:k3r7P4seEiUcgxOPLp+mloJWV3Q4QLPGNvy/OgC8OtM= +go.etcd.io/etcd/server/v3 v3.5.15 h1:x35jrWnZgsRwMsFsUJIUdT1bvzIz1B+29HjMfRYVN/E= +go.etcd.io/etcd/server/v3 v3.5.15/go.mod h1:l9jX9oa/iuArjqz0RNX/TDbc70dLXxRZo/nmPucrpFo= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.0 h1:PzIubN4/sjByhDRHLviCjJuweBXWFZWhghjg7cS28+M= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.0/go.mod h1:Ct6zzQEuGK3WpJs2n4dn+wfJYzd/+hNnxMRTWjGn30M= +go.opentelemetry.io/otel v1.20.0 h1:vsb/ggIY+hUjD/zCAQHpzTmndPqv/ml2ArbsbfBYTAc= +go.opentelemetry.io/otel v1.20.0/go.mod h1:oUIGj3D77RwJdM6PPZImDpSZGDvkD9fhesHny69JFrs= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.20.0 h1:DeFD0VgTZ+Cj6hxravYYZE2W4GlneVH81iAOPjZkzk8= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.20.0/go.mod h1:GijYcYmNpX1KazD5JmWGsi4P7dDTTTnfv1UbGn84MnU= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.20.0 h1:gvmNvqrPYovvyRmCSygkUDyL8lC5Tl845MLEwqpxhEU= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.20.0/go.mod h1:vNUq47TGFioo+ffTSnKNdob241vePmtNZnAODKapKd0= +go.opentelemetry.io/otel/metric v1.20.0 h1:ZlrO8Hu9+GAhnepmRGhSU7/VkpjrNowxRN9GyKR4wzA= +go.opentelemetry.io/otel/metric v1.20.0/go.mod h1:90DRw3nfK4D7Sm/75yQ00gTJxtkBxX+wu6YaNymbpVM= +go.opentelemetry.io/otel/sdk v1.20.0 h1:5Jf6imeFZlZtKv9Qbo6qt2ZkmWtdWx/wzcCbNUlAWGM= +go.opentelemetry.io/otel/sdk v1.20.0/go.mod h1:rmkSx1cZCm/tn16iWDn1GQbLtsW/LvsdEEFzCSRM6V0= +go.opentelemetry.io/otel/trace v1.20.0 h1:+yxVAPZPbQhbC3OfAkeIVTky6iTFpcr4SiY9om7mXSQ= +go.opentelemetry.io/otel/trace v1.20.0/go.mod h1:HJSK7F/hA5RlzpZ0zKDCHCDHm556LCDtKaAo6JmBFUU= +go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I= +go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= diff --git a/tools/pd-api-bench/cases/cases.go b/tools/pd-api-bench/cases/cases.go index 6ffa429fdb7..d81e7e0fa9d 100644 --- a/tools/pd-api-bench/cases/cases.go +++ b/tools/pd-api-bench/cases/cases.go @@ -24,7 +24,7 @@ import ( "github.com/pingcap/log" pd "github.com/tikv/pd/client" pdHttp "github.com/tikv/pd/client/http" - "go.etcd.io/etcd/clientv3" + clientv3 "go.etcd.io/etcd/client/v3" "go.uber.org/zap" ) diff --git a/tools/pd-api-bench/cases/controller.go b/tools/pd-api-bench/cases/controller.go index dc48b5280cf..a4e20f25758 100644 --- a/tools/pd-api-bench/cases/controller.go +++ b/tools/pd-api-bench/cases/controller.go @@ -23,7 +23,7 @@ import ( "github.com/pingcap/log" pd "github.com/tikv/pd/client" pdHttp "github.com/tikv/pd/client/http" - "go.etcd.io/etcd/clientv3" + clientv3 "go.etcd.io/etcd/client/v3" "go.uber.org/zap" ) diff --git a/tools/pd-api-bench/main.go b/tools/pd-api-bench/main.go index cb472fb5973..f642ede0069 100644 --- a/tools/pd-api-bench/main.go +++ b/tools/pd-api-bench/main.go @@ -40,7 +40,7 @@ import ( "github.com/tikv/pd/pkg/utils/logutil" "github.com/tikv/pd/tools/pd-api-bench/cases" "github.com/tikv/pd/tools/pd-api-bench/config" - "go.etcd.io/etcd/clientv3" + clientv3 "go.etcd.io/etcd/client/v3" "go.uber.org/zap" "google.golang.org/grpc" "google.golang.org/grpc/keepalive" diff --git a/tools/pd-backup/main.go b/tools/pd-backup/main.go index c424113e451..c0ee07afd6e 100644 --- a/tools/pd-backup/main.go +++ b/tools/pd-backup/main.go @@ -22,8 +22,8 @@ import ( "time" "github.com/tikv/pd/tools/pd-backup/pdbackup" - "go.etcd.io/etcd/clientv3" - "go.etcd.io/etcd/pkg/transport" + "go.etcd.io/etcd/client/pkg/v3/transport" + clientv3 "go.etcd.io/etcd/client/v3" ) var ( diff --git a/tools/pd-backup/pdbackup/backup.go b/tools/pd-backup/pdbackup/backup.go index f0b17e48267..7eea1fb4b0b 100644 --- a/tools/pd-backup/pdbackup/backup.go +++ b/tools/pd-backup/pdbackup/backup.go @@ -29,7 +29,7 @@ import ( "github.com/tikv/pd/pkg/utils/etcdutil" "github.com/tikv/pd/pkg/utils/typeutil" "github.com/tikv/pd/server/config" - "go.etcd.io/etcd/clientv3" + clientv3 "go.etcd.io/etcd/client/v3" ) const ( diff --git a/tools/pd-backup/pdbackup/backup_test.go b/tools/pd-backup/pdbackup/backup_test.go index b67873baf8e..f11cfef8e42 100644 --- a/tools/pd-backup/pdbackup/backup_test.go +++ b/tools/pd-backup/pdbackup/backup_test.go @@ -22,8 +22,8 @@ import ( "github.com/tikv/pd/pkg/utils/testutil" "github.com/tikv/pd/pkg/utils/typeutil" "github.com/tikv/pd/server/config" - "go.etcd.io/etcd/clientv3" - "go.etcd.io/etcd/embed" + clientv3 "go.etcd.io/etcd/client/v3" + "go.etcd.io/etcd/server/v3/embed" "go.uber.org/goleak" ) diff --git a/tools/pd-backup/tests/backup_test.go b/tools/pd-backup/tests/backup_test.go index 83a465e94fe..2a55a790849 100644 --- a/tools/pd-backup/tests/backup_test.go +++ b/tools/pd-backup/tests/backup_test.go @@ -25,7 +25,7 @@ import ( "github.com/stretchr/testify/require" "github.com/tikv/pd/tests" "github.com/tikv/pd/tools/pd-backup/pdbackup" - "go.etcd.io/etcd/clientv3" + clientv3 "go.etcd.io/etcd/client/v3" ) func TestBackup(t *testing.T) { diff --git a/tools/pd-ctl/pdctl/command/global.go b/tools/pd-ctl/pdctl/command/global.go index 7438345127e..c5c633de217 100644 --- a/tools/pd-ctl/pdctl/command/global.go +++ b/tools/pd-ctl/pdctl/command/global.go @@ -29,7 +29,7 @@ import ( "github.com/spf13/cobra" pd "github.com/tikv/pd/client/http" "github.com/tikv/pd/pkg/utils/apiutil" - "go.etcd.io/etcd/pkg/transport" + "go.etcd.io/etcd/client/pkg/v3/transport" ) const ( diff --git a/tools/pd-ctl/tests/health/health_test.go b/tools/pd-ctl/tests/health/health_test.go index 1d6cf884ccf..be9d5027988 100644 --- a/tools/pd-ctl/tests/health/health_test.go +++ b/tools/pd-ctl/tests/health/health_test.go @@ -31,7 +31,7 @@ import ( pdTests "github.com/tikv/pd/tests" ctl "github.com/tikv/pd/tools/pd-ctl/pdctl" "github.com/tikv/pd/tools/pd-ctl/tests" - "go.etcd.io/etcd/pkg/transport" + "go.etcd.io/etcd/client/pkg/v3/transport" ) func TestHealth(t *testing.T) { diff --git a/tools/pd-ctl/tests/store/store_test.go b/tools/pd-ctl/tests/store/store_test.go index ae35839837e..983c86b242a 100644 --- a/tools/pd-ctl/tests/store/store_test.go +++ b/tools/pd-ctl/tests/store/store_test.go @@ -35,7 +35,7 @@ import ( pdTests "github.com/tikv/pd/tests" ctl "github.com/tikv/pd/tools/pd-ctl/pdctl" "github.com/tikv/pd/tools/pd-ctl/tests" - "go.etcd.io/etcd/pkg/transport" + "go.etcd.io/etcd/client/pkg/v3/transport" ) func TestStoreLimitV2(t *testing.T) { diff --git a/tools/pd-heartbeat-bench/main.go b/tools/pd-heartbeat-bench/main.go index 9d71be1129b..cfa0495c31c 100644 --- a/tools/pd-heartbeat-bench/main.go +++ b/tools/pd-heartbeat-bench/main.go @@ -48,7 +48,7 @@ import ( "github.com/tikv/pd/pkg/utils/logutil" "github.com/tikv/pd/tools/pd-heartbeat-bench/config" "github.com/tikv/pd/tools/pd-heartbeat-bench/metrics" - "go.etcd.io/etcd/pkg/report" + "go.etcd.io/etcd/pkg/v3/report" "go.uber.org/zap" ) diff --git a/tools/pd-heartbeat-bench/metrics/util.go b/tools/pd-heartbeat-bench/metrics/util.go index bf5010e73da..e409cf10815 100644 --- a/tools/pd-heartbeat-bench/metrics/util.go +++ b/tools/pd-heartbeat-bench/metrics/util.go @@ -26,7 +26,7 @@ import ( "github.com/prometheus/client_golang/api" v1 "github.com/prometheus/client_golang/api/prometheus/v1" "github.com/prometheus/common/model" - "go.etcd.io/etcd/pkg/report" + "go.etcd.io/etcd/pkg/v3/report" "go.uber.org/zap" ) diff --git a/tools/pd-recover/main.go b/tools/pd-recover/main.go index 9b5d08013db..3423d06bb9f 100644 --- a/tools/pd-recover/main.go +++ b/tools/pd-recover/main.go @@ -28,8 +28,8 @@ import ( "github.com/tikv/pd/pkg/utils/etcdutil" "github.com/tikv/pd/pkg/utils/typeutil" "github.com/tikv/pd/pkg/versioninfo" - "go.etcd.io/etcd/clientv3" - "go.etcd.io/etcd/pkg/transport" + "go.etcd.io/etcd/client/pkg/v3/transport" + clientv3 "go.etcd.io/etcd/client/v3" ) var ( diff --git a/tools/pd-simulator/simulator/drive.go b/tools/pd-simulator/simulator/drive.go index 87b779559ca..e602ba2df65 100644 --- a/tools/pd-simulator/simulator/drive.go +++ b/tools/pd-simulator/simulator/drive.go @@ -38,7 +38,7 @@ import ( "github.com/tikv/pd/tools/pd-simulator/simulator/config" "github.com/tikv/pd/tools/pd-simulator/simulator/info" "github.com/tikv/pd/tools/pd-simulator/simulator/simutil" - "go.etcd.io/etcd/clientv3" + clientv3 "go.etcd.io/etcd/client/v3" "go.uber.org/zap" ) diff --git a/tools/regions-dump/main.go b/tools/regions-dump/main.go index dddbcbba854..5ae4241cdc0 100644 --- a/tools/regions-dump/main.go +++ b/tools/regions-dump/main.go @@ -29,8 +29,8 @@ import ( "github.com/pingcap/kvproto/pkg/metapb" "github.com/tikv/pd/pkg/core" "github.com/tikv/pd/pkg/utils/etcdutil" - "go.etcd.io/etcd/clientv3" - "go.etcd.io/etcd/pkg/transport" + "go.etcd.io/etcd/client/pkg/v3/transport" + clientv3 "go.etcd.io/etcd/client/v3" ) var ( diff --git a/tools/stores-dump/main.go b/tools/stores-dump/main.go index b3d9b00be9f..0409244772f 100644 --- a/tools/stores-dump/main.go +++ b/tools/stores-dump/main.go @@ -28,8 +28,8 @@ import ( "github.com/pingcap/errors" "github.com/pingcap/kvproto/pkg/metapb" "github.com/tikv/pd/pkg/utils/etcdutil" - "go.etcd.io/etcd/clientv3" - "go.etcd.io/etcd/pkg/transport" + "go.etcd.io/etcd/client/pkg/v3/transport" + clientv3 "go.etcd.io/etcd/client/v3" ) var ( From 1b8fc6a950dbe658a0051c7f8228367b85bc9180 Mon Sep 17 00:00:00 2001 From: Ryan Leung Date: Thu, 15 Aug 2024 14:17:40 +0800 Subject: [PATCH 02/18] *: add linters to organize import dependency (#8532) ref tikv/pd#4322 Signed-off-by: Ryan Leung --- .gitignore | 1 + .golangci.yml | 11 +++++++++++ client/go.mod | 2 +- client/http/client_test.go | 7 ++++--- client/resource_group/controller/controller.go | 3 +-- go.mod | 2 +- go.sum | 3 +-- pkg/gctuner/finalizer.go | 3 +-- pkg/gctuner/memory_limit_tuner.go | 16 ++++++---------- pkg/memory/var.go | 13 ++----------- tools/pd-simulator/simulator/cases/hot_write.go | 1 + tools/pd-simulator/simulator/simutil/key_test.go | 4 ++-- 12 files changed, 32 insertions(+), 34 deletions(-) diff --git a/.gitignore b/.gitignore index dc2efa5004f..a9d942fe706 100644 --- a/.gitignore +++ b/.gitignore @@ -28,3 +28,4 @@ coverage go.work* embedded_assets_handler.go *.log +*.bin diff --git a/.golangci.yml b/.golangci.yml index bc1ba393f39..e09011c45be 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -15,6 +15,9 @@ linters: - gofmt - revive - errcheck + - exportloopref + - goimports + - depguard linters-settings: gocritic: # Which checks should be disabled; can't be combined with 'enabled-checks'; default is empty @@ -205,6 +208,14 @@ linters-settings: - (net/http.ResponseWriter).Write - github.com/pingcap/log.Sync - (github.com/tikv/pd/pkg/ratelimit.Runner).RunTask + depguard: + rules: + denied-deps: + deny: + - pkg: go.uber.org/atomic + desc: "Use 'sync/atomic' instead of 'go.uber.org/atomic'" + - pkg: github.com/pkg/errors + desc: "Use 'github.com/pingcap/errors' instead of 'github.com/pkg/errors'" issues: exclude-rules: - path: (_test\.go|pkg/mock/.*\.go|tests/.*\.go) diff --git a/client/go.mod b/client/go.mod index 8dc706a4540..eb537f93ec8 100644 --- a/client/go.mod +++ b/client/go.mod @@ -14,7 +14,6 @@ require ( github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 github.com/prometheus/client_golang v1.18.0 github.com/stretchr/testify v1.8.2 - go.uber.org/atomic v1.10.0 go.uber.org/goleak v1.1.11 go.uber.org/zap v1.24.0 golang.org/x/exp v0.0.0-20230711005742-c3f37128e5a4 @@ -34,6 +33,7 @@ require ( github.com/prometheus/common v0.46.0 // indirect github.com/prometheus/procfs v0.12.0 // indirect github.com/stretchr/objx v0.5.0 // indirect + go.uber.org/atomic v1.10.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/net v0.23.0 // indirect golang.org/x/sys v0.18.0 // indirect diff --git a/client/http/client_test.go b/client/http/client_test.go index 8769fa53f9a..5d63a34df54 100644 --- a/client/http/client_test.go +++ b/client/http/client_test.go @@ -18,13 +18,13 @@ import ( "context" "net/http" "strings" + "sync/atomic" "testing" "time" "github.com/stretchr/testify/require" "github.com/tikv/pd/client/errs" "github.com/tikv/pd/client/retry" - "go.uber.org/atomic" ) func TestPDAllowFollowerHandleHeader(t *testing.T) { @@ -53,7 +53,8 @@ func TestPDAllowFollowerHandleHeader(t *testing.T) { func TestWithCallerID(t *testing.T) { re := require.New(t) checked := 0 - expectedVal := atomic.NewString(defaultCallerID) + var expectedVal atomic.Value + expectedVal.Store(defaultCallerID) httpClient := NewHTTPClientWithRequestChecker(func(req *http.Request) error { val := req.Header.Get(xCallerIDKey) // Exclude the request sent by the inner client. @@ -68,7 +69,7 @@ func TestWithCallerID(t *testing.T) { defer c.Close() c.GetRegions(context.Background()) expectedVal.Store("test") - c.WithCallerID(expectedVal.Load()).GetRegions(context.Background()) + c.WithCallerID(expectedVal.Load().(string)).GetRegions(context.Background()) re.Equal(2, checked) } diff --git a/client/resource_group/controller/controller.go b/client/resource_group/controller/controller.go index 8e32b156a61..e5ad1f81fa5 100644 --- a/client/resource_group/controller/controller.go +++ b/client/resource_group/controller/controller.go @@ -32,7 +32,6 @@ import ( "github.com/prometheus/client_golang/prometheus" pd "github.com/tikv/pd/client" "github.com/tikv/pd/client/errs" - atomicutil "go.uber.org/atomic" "go.uber.org/zap" "golang.org/x/exp/slices" ) @@ -57,7 +56,7 @@ const ( lowToken selectType = 1 ) -var enableControllerTraceLog = atomicutil.NewBool(false) +var enableControllerTraceLog atomic.Bool func logControllerTrace(msg string, fields ...zap.Field) { if enableControllerTraceLog.Load() { diff --git a/go.mod b/go.mod index 9dc5fa9f249..79db4f3940c 100644 --- a/go.mod +++ b/go.mod @@ -56,7 +56,6 @@ require ( go.etcd.io/etcd/client/pkg/v3 v3.5.15 go.etcd.io/etcd/client/v3 v3.5.15 go.etcd.io/etcd/server/v3 v3.5.15 - go.uber.org/atomic v1.10.0 go.uber.org/goleak v1.3.0 go.uber.org/zap v1.27.0 golang.org/x/exp v0.0.0-20230711005742-c3f37128e5a4 @@ -187,6 +186,7 @@ require ( go.opentelemetry.io/otel/sdk v1.20.0 // indirect go.opentelemetry.io/otel/trace v1.20.0 // indirect go.opentelemetry.io/proto/otlp v1.0.0 // indirect + go.uber.org/atomic v1.9.0 // indirect go.uber.org/dig v1.9.0 // indirect go.uber.org/fx v1.12.0 // indirect go.uber.org/multierr v1.11.0 diff --git a/go.sum b/go.sum index e3bc1ed175a..e57fdcbce9c 100644 --- a/go.sum +++ b/go.sum @@ -566,9 +566,8 @@ go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= -go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/dig v1.9.0 h1:pJTDXKEhRqBI8W7rU7kwT5EgyRZuSMVSFcZolOvKK9U= go.uber.org/dig v1.9.0/go.mod h1:X34SnWGr8Fyla9zQNO2GSO2D+TIuqB14OS8JhYocIyw= go.uber.org/fx v1.12.0 h1:+1+3Cz9M0dFMPy9SW9XUIUHye8bnPUm7q7DroNGWYG4= diff --git a/pkg/gctuner/finalizer.go b/pkg/gctuner/finalizer.go index b9a73fa7ce5..a1424980b66 100644 --- a/pkg/gctuner/finalizer.go +++ b/pkg/gctuner/finalizer.go @@ -16,8 +16,7 @@ package gctuner import ( "runtime" - - "go.uber.org/atomic" + "sync/atomic" ) type finalizerCallback func() diff --git a/pkg/gctuner/memory_limit_tuner.go b/pkg/gctuner/memory_limit_tuner.go index 77a8f9db4c6..8a852b191d8 100644 --- a/pkg/gctuner/memory_limit_tuner.go +++ b/pkg/gctuner/memory_limit_tuner.go @@ -17,6 +17,7 @@ package gctuner import ( "math" "runtime/debug" + "sync/atomic" "time" "github.com/pingcap/failpoint" @@ -24,7 +25,6 @@ import ( util "github.com/tikv/pd/pkg/gogc" "github.com/tikv/pd/pkg/memory" "github.com/tikv/pd/pkg/utils/logutil" - atomicutil "go.uber.org/atomic" "go.uber.org/zap" ) @@ -35,10 +35,10 @@ var GlobalMemoryLimitTuner = &memoryLimitTuner{} // So we can change memory limit dynamically to avoid frequent GC when memory usage is greater than the limit. type memoryLimitTuner struct { finalizer *finalizer - isTuning atomicutil.Bool - percentage atomicutil.Float64 - waitingReset atomicutil.Bool - nextGCTriggeredByMemoryLimit atomicutil.Bool + isTuning atomic.Bool + percentage atomic.Value + waitingReset atomic.Bool + nextGCTriggeredByMemoryLimit atomic.Bool } // fallbackPercentage indicates the fallback memory limit percentage when turning. @@ -74,8 +74,6 @@ func (t *memoryLimitTuner) tuning() { if t.nextGCTriggeredByMemoryLimit.Load() && t.waitingReset.CompareAndSwap(false, true) { go func() { defer logutil.LogPanic() - memory.MemoryLimitGCLast.Store(time.Now()) - memory.MemoryLimitGCTotal.Add(1) setMemoryLimit(t.calcMemoryLimit(fallbackPercentage)) resetInterval := 1 * time.Minute // Wait 1 minute and set back, to avoid frequent GC failpoint.Inject("testMemoryLimitTuner", func(val failpoint.Value) { @@ -89,12 +87,10 @@ func (t *memoryLimitTuner) tuning() { continue } }() - memory.TriggerMemoryLimitGC.Store(true) } t.nextGCTriggeredByMemoryLimit.Store(true) } else { t.nextGCTriggeredByMemoryLimit.Store(false) - memory.TriggerMemoryLimitGC.Store(false) } } @@ -117,7 +113,7 @@ func (t *memoryLimitTuner) SetPercentage(percentage float64) { // GetPercentage get the percentage from memory limit tuner. func (t *memoryLimitTuner) GetPercentage() float64 { - return t.percentage.Load() + return t.percentage.Load().(float64) } // UpdateMemoryLimit updates the memory limit. diff --git a/pkg/memory/var.go b/pkg/memory/var.go index fd218183937..1ecc0aa3238 100644 --- a/pkg/memory/var.go +++ b/pkg/memory/var.go @@ -15,19 +15,10 @@ package memory import ( - "time" - - atomicutil "go.uber.org/atomic" + atomic "sync/atomic" ) // Process global variables for memory limit. var ( - ServerMemoryLimitOriginText = atomicutil.NewString("0") - ServerMemoryLimit = atomicutil.NewUint64(0) - ServerMemoryLimitSessMinSize = atomicutil.NewUint64(128 << 20) - - QueryForceDisk = atomicutil.NewInt64(0) - TriggerMemoryLimitGC = atomicutil.NewBool(false) - MemoryLimitGCLast = atomicutil.NewTime(time.Time{}) - MemoryLimitGCTotal = atomicutil.NewInt64(0) + ServerMemoryLimit atomic.Uint64 ) diff --git a/tools/pd-simulator/simulator/cases/hot_write.go b/tools/pd-simulator/simulator/cases/hot_write.go index 8f08264590d..54e944e540d 100644 --- a/tools/pd-simulator/simulator/cases/hot_write.go +++ b/tools/pd-simulator/simulator/cases/hot_write.go @@ -16,6 +16,7 @@ package cases import ( "fmt" + "github.com/docker/go-units" "github.com/pingcap/kvproto/pkg/metapb" "github.com/tikv/pd/pkg/core" diff --git a/tools/pd-simulator/simulator/simutil/key_test.go b/tools/pd-simulator/simulator/simutil/key_test.go index ae00887afa8..9db7c597751 100644 --- a/tools/pd-simulator/simulator/simutil/key_test.go +++ b/tools/pd-simulator/simulator/simutil/key_test.go @@ -15,12 +15,12 @@ package simutil import ( - "github.com/pingcap/kvproto/pkg/metapb" - "github.com/tikv/pd/pkg/core" "testing" + "github.com/pingcap/kvproto/pkg/metapb" "github.com/stretchr/testify/require" "github.com/tikv/pd/pkg/codec" + "github.com/tikv/pd/pkg/core" ) func TestGenerateTableKeys(t *testing.T) { From 4b0878ed38c46eca398ee4857520c8c06c92ec5f Mon Sep 17 00:00:00 2001 From: Hu# Date: Fri, 16 Aug 2024 12:01:11 +0800 Subject: [PATCH 03/18] tests/api: add wait leader for api (#8540) close tikv/pd#8513 tests/api: add wait leader for api Signed-off-by: husharp --- tests/server/api/api_test.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/server/api/api_test.go b/tests/server/api/api_test.go index 31769d91ffc..8d48221784a 100644 --- a/tests/server/api/api_test.go +++ b/tests/server/api/api_test.go @@ -129,6 +129,13 @@ func (suite *middlewareTestSuite) SetupSuite() { suite.cluster = cluster } +func (suite *middlewareTestSuite) SetupTest() { + re := suite.Require() + re.NotEmpty(suite.cluster.WaitLeader()) + leader := suite.cluster.GetLeaderServer() + re.NotNil(leader) +} + func (suite *middlewareTestSuite) TearDownSuite() { re := suite.Require() re.NoError(failpoint.Disable("github.com/tikv/pd/server/api/enableFailpointAPI")) From 10cbdcf0c551c3e337ebc1b885b1f2ccd874b163 Mon Sep 17 00:00:00 2001 From: lhy1024 Date: Fri, 16 Aug 2024 14:50:41 +0800 Subject: [PATCH 04/18] config: disable EnableV2 of etcd (#8536) close tikv/pd#8535 Signed-off-by: lhy1024 Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- server/api/member_test.go | 25 +++++++++++-------------- server/config/config.go | 1 - 2 files changed, 11 insertions(+), 15 deletions(-) diff --git a/server/api/member_test.go b/server/api/member_test.go index d3318081e62..ad4812249c7 100644 --- a/server/api/member_test.go +++ b/server/api/member_test.go @@ -15,9 +15,8 @@ package api import ( - "bytes" + "context" "encoding/json" - "fmt" "io" "math/rand" "net/http" @@ -31,6 +30,7 @@ import ( "github.com/tikv/pd/pkg/utils/testutil" "github.com/tikv/pd/server" "github.com/tikv/pd/server/config" + clientv3 "go.etcd.io/etcd/client/v3" ) type memberTestSuite struct { @@ -124,11 +124,10 @@ func (suite *memberTestSuite) TestChangeLeaderPeerUrls() { var got pdpb.Member re.NoError(json.Unmarshal(buf, &got)) - id := got.GetMemberId() peerUrls := got.GetPeerUrls() newPeerUrls := []string{"http://127.0.0.1:1111"} - suite.changeLeaderPeerUrls(leader, id, newPeerUrls) + suite.changeLeaderPeerUrls(leader, newPeerUrls) addr = suite.cfgs[rand.Intn(len(suite.cfgs))].ClientUrls + apiPrefix + "/api/v1/members" resp, err = testDialClient.Get(addr) re.NoError(err) @@ -141,21 +140,19 @@ func (suite *memberTestSuite) TestChangeLeaderPeerUrls() { re.Equal(newPeerUrls, got1["etcd_leader"].GetPeerUrls()) // reset - suite.changeLeaderPeerUrls(leader, id, peerUrls) + suite.changeLeaderPeerUrls(leader, peerUrls) } -func (suite *memberTestSuite) changeLeaderPeerUrls(leader *pdpb.Member, id uint64, urls []string) { +func (suite *memberTestSuite) changeLeaderPeerUrls(leader *pdpb.Member, urls []string) { re := suite.Require() - data := map[string][]string{"peerURLs": urls} - postData, err := json.Marshal(data) - re.NoError(err) - req, err := http.NewRequest(http.MethodPut, fmt.Sprintf("%s/v2/members/%s", leader.GetClientUrls()[0], fmt.Sprintf("%x", id)), bytes.NewBuffer(postData)) + + cli, err := clientv3.New(clientv3.Config{ + Endpoints: leader.GetClientUrls(), + }) re.NoError(err) - req.Header.Set("Content-Type", "application/json") - resp, err := testDialClient.Do(req) + _, err = cli.MemberUpdate(context.Background(), leader.GetMemberId(), urls) re.NoError(err) - re.Equal(204, resp.StatusCode) - resp.Body.Close() + cli.Close() } func (suite *memberTestSuite) TestResignMyself() { diff --git a/server/config/config.go b/server/config/config.go index 8c4f6eaacc8..c1791be1fd9 100644 --- a/server/config/config.go +++ b/server/config/config.go @@ -746,7 +746,6 @@ func (c *Config) GenEmbedEtcdConfig() (*embed.Config, error) { cfg.ForceNewCluster = c.ForceNewCluster cfg.ZapLoggerBuilder = embed.NewZapCoreLoggerBuilder(c.Logger, c.Logger.Core(), c.LogProps.Syncer) cfg.EnableGRPCGateway = c.EnableGRPCGateway - cfg.EnableV2 = true cfg.Logger = "zap" var err error From a7663514c5ebfc78a99bacc84e030c9b7530a10b Mon Sep 17 00:00:00 2001 From: lhy1024 Date: Mon, 19 Aug 2024 17:55:41 +0800 Subject: [PATCH 05/18] *: fix some typos (#8544) ref tikv/pd#4399 Signed-off-by: lhy1024 --- client/pd_service_discovery.go | 4 ++-- conf/config.toml | 2 +- pkg/autoscaling/calculation.go | 2 +- pkg/gc/safepoint_test.go | 16 ++++++++-------- pkg/member/election_leader.go | 2 +- pkg/movingaverage/weight_allocator.go | 2 +- pkg/replication/replication_mode_test.go | 2 +- pkg/schedule/checker/rule_checker.go | 4 ++-- pkg/schedule/operator/operator.go | 2 +- .../operator/operator_controller_test.go | 2 +- pkg/schedule/plan/plan.go | 18 +++++++++--------- pkg/schedule/scatter/region_scatterer.go | 2 +- pkg/schedule/schedulers/evict_slow_store.go | 2 +- pkg/schedule/schedulers/evict_slow_trend.go | 2 +- pkg/schedule/schedulers/metrics.go | 2 +- pkg/schedule/schedulers/utils.go | 2 +- scripts/dashboard-version | 2 +- scripts/update-dashboard.sh | 8 ++++---- server/api/middleware.go | 2 +- server/cluster/cluster_test.go | 4 ++-- tests/server/apiv2/handlers/keyspace_test.go | 4 ++-- tests/server/cluster/cluster_test.go | 2 +- tools/pd-api-bench/cases/controller.go | 12 ++++++------ tools/pd-ctl/pdctl/command/config_command.go | 18 +++++++++--------- tools/pd-simulator/README.md | 2 +- 25 files changed, 60 insertions(+), 60 deletions(-) diff --git a/client/pd_service_discovery.go b/client/pd_service_discovery.go index e8f4c0d7707..c34a5bebac6 100644 --- a/client/pd_service_discovery.go +++ b/client/pd_service_discovery.go @@ -157,7 +157,7 @@ type pdServiceClient struct { } // NOTE: In the current implementation, the URL passed in is bound to have a scheme, -// because it is processed in `newPDServiceDiscovery`, and the url returned by etcd member owns the sheme. +// because it is processed in `newPDServiceDiscovery`, and the url returned by etcd member owns the scheme. // When testing, the URL is also bound to have a scheme. func newPDServiceClient(url, leaderURL string, conn *grpc.ClientConn, isLeader bool) ServiceClient { cli := &pdServiceClient{ @@ -1074,7 +1074,7 @@ func (c *pdServiceDiscovery) updateServiceClient(members []*pdpb.Member, leader leaderURL := pickMatchedURL(leader.GetClientUrls(), c.tlsCfg) leaderChanged, err := c.switchLeader(leaderURL) followerChanged := c.updateFollowers(members, leader.GetMemberId(), leaderURL) - // don't need to recreate balancer if no changess. + // don't need to recreate balancer if no changes. if !followerChanged && !leaderChanged { return err } diff --git a/conf/config.toml b/conf/config.toml index 0c4acf5fd8c..f2feacf30f7 100644 --- a/conf/config.toml +++ b/conf/config.toml @@ -206,7 +206,7 @@ [keyspace] ## pre-alloc is used to pre-allocate keyspaces during pd bootstrap. -## Its value should be a list of strings, denotting the name of the keyspaces. +## Its value should be a list of strings, denoting the name of the keyspaces. ## Example: ## pre-alloc = ["admin", "user1", "user2"] # pre-alloc = [] diff --git a/pkg/autoscaling/calculation.go b/pkg/autoscaling/calculation.go index b155c44bf68..43aa2972ed8 100644 --- a/pkg/autoscaling/calculation.go +++ b/pkg/autoscaling/calculation.go @@ -431,7 +431,7 @@ func findBestGroupToScaleOut(strategy *Strategy, groups []*Plan, component Compo }, } - // TODO: we can provide different senerios by using options and remove this kind of special judgement. + // TODO: we can provide different scenarios by using options and remove this kind of special judgement. if component == TiKV { group.Labels[filter.SpecialUseKey] = filter.SpecialUseHotRegion } diff --git a/pkg/gc/safepoint_test.go b/pkg/gc/safepoint_test.go index 39cd3660b2b..bc1e551594c 100644 --- a/pkg/gc/safepoint_test.go +++ b/pkg/gc/safepoint_test.go @@ -85,7 +85,7 @@ func TestGCSafePointUpdateCurrently(t *testing.T) { func TestServiceGCSafePointUpdate(t *testing.T) { re := require.New(t) manager := NewSafePointManager(newGCStorage(), config.PDServerConfig{}) - gcworkerServiceID := "gc_worker" + gcWorkerServiceID := "gc_worker" cdcServiceID := "cdc" brServiceID := "br" cdcServiceSafePoint := uint64(10) @@ -101,7 +101,7 @@ func TestServiceGCSafePointUpdate(t *testing.T) { re.NoError(err) re.True(updated) // the service will init the service safepoint to 0(<10 for cdc) for gc_worker. - re.Equal(gcworkerServiceID, min.ServiceID) + re.Equal(gcWorkerServiceID, min.ServiceID) }() // update the safepoint for br to 15 should success @@ -111,24 +111,24 @@ func TestServiceGCSafePointUpdate(t *testing.T) { re.NoError(err) re.True(updated) // the service will init the service safepoint to 0(<10 for cdc) for gc_worker. - re.Equal(gcworkerServiceID, min.ServiceID) + re.Equal(gcWorkerServiceID, min.ServiceID) }() - // update safepoint to 8 for gc_woker should be success + // update safepoint to 8 for gc_worker should be success go func() { defer wg.Done() // update with valid ttl for gc_worker should be success. - min, updated, _ := manager.UpdateServiceGCSafePoint(gcworkerServiceID, gcWorkerSafePoint, math.MaxInt64, time.Now()) + min, updated, _ := manager.UpdateServiceGCSafePoint(gcWorkerServiceID, gcWorkerSafePoint, math.MaxInt64, time.Now()) re.True(updated) // the current min safepoint should be 8 for gc_worker(cdc 10) re.Equal(gcWorkerSafePoint, min.SafePoint) - re.Equal(gcworkerServiceID, min.ServiceID) + re.Equal(gcWorkerServiceID, min.ServiceID) }() go func() { defer wg.Done() // update safepoint of gc_worker's service with ttl not infinity should be failed. - _, updated, err := manager.UpdateServiceGCSafePoint(gcworkerServiceID, 10000, 10, time.Now()) + _, updated, err := manager.UpdateServiceGCSafePoint(gcWorkerServiceID, 10000, 10, time.Now()) re.Error(err) re.False(updated) }() @@ -145,7 +145,7 @@ func TestServiceGCSafePointUpdate(t *testing.T) { wg.Wait() // update safepoint to 15(>10 for cdc) for gc_worker gcWorkerSafePoint = uint64(15) - min, updated, err := manager.UpdateServiceGCSafePoint(gcworkerServiceID, gcWorkerSafePoint, math.MaxInt64, time.Now()) + min, updated, err := manager.UpdateServiceGCSafePoint(gcWorkerServiceID, gcWorkerSafePoint, math.MaxInt64, time.Now()) re.NoError(err) re.True(updated) re.Equal(cdcServiceID, min.ServiceID) diff --git a/pkg/member/election_leader.go b/pkg/member/election_leader.go index 24520bfbe64..81afc5dbd0a 100644 --- a/pkg/member/election_leader.go +++ b/pkg/member/election_leader.go @@ -21,7 +21,7 @@ import ( ) // ElectionLeader defines the common interface of the leader, which is the pdpb.Member -// for in PD/API service or the tsopb.Participant in the microserives. +// for in PD/API service or the tsopb.Participant in the micro services. type ElectionLeader interface { // GetListenUrls returns the listen urls GetListenUrls() []string diff --git a/pkg/movingaverage/weight_allocator.go b/pkg/movingaverage/weight_allocator.go index f63ce377e08..e0427d84645 100644 --- a/pkg/movingaverage/weight_allocator.go +++ b/pkg/movingaverage/weight_allocator.go @@ -20,7 +20,7 @@ package movingaverage // WeightAllocator will divide these items into some segments whose number named as segNum which should great than 0. // And the items at first segment will be assigned more weight that is `segNum` times that of item at last segment. // If you want assign same weights, just input segNum as 1. -// If length is 10 and segNum is 3, it will make the weight arrry as [3,3,3,3,2,2,2,1,1,1], +// If length is 10 and segNum is 3, it will make the weight array as [3,3,3,3,2,2,2,1,1,1], // and then uniform it : [3,3,3,3,2,2,2,1,1,1]/sum(arr)=arr/21, // And the final weight is [0.143,0.143,0.143,0.143,0.095,0.095,0.095,0.047,0.047,0.047]; // If length is 10 and segNum is 1, the weight is [0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1]; diff --git a/pkg/replication/replication_mode_test.go b/pkg/replication/replication_mode_test.go index bbaada98924..243d7f7d8f1 100644 --- a/pkg/replication/replication_mode_test.go +++ b/pkg/replication/replication_mode_test.go @@ -409,7 +409,7 @@ func TestReplicateState(t *testing.T) { rep.tickReplicateStatus() assertLastData(t, replicator.lastData[1], "sync", stateID, nil) - // repliate state to new member + // replicate state to new member replicator.memberIDs = append(replicator.memberIDs, 2, 3) rep.tickReplicateStatus() assertLastData(t, replicator.lastData[2], "sync", stateID, nil) diff --git a/pkg/schedule/checker/rule_checker.go b/pkg/schedule/checker/rule_checker.go index e29cd2bc05b..23018ad7096 100644 --- a/pkg/schedule/checker/rule_checker.go +++ b/pkg/schedule/checker/rule_checker.go @@ -265,7 +265,7 @@ func (c *RuleChecker) replaceUnexpectedRulePeer(region *core.RegionInfo, rf *pla minCount := uint64(math.MaxUint64) for _, p := range region.GetPeers() { count := c.record.getOfflineLeaderCount(p.GetStoreId()) - checkPeerhealth := func() bool { + checkPeerHealth := func() bool { if p.GetId() == peer.GetId() { return true } @@ -274,7 +274,7 @@ func (c *RuleChecker) replaceUnexpectedRulePeer(region *core.RegionInfo, rf *pla } return c.allowLeader(fit, p) } - if minCount > count && checkPeerhealth() { + if minCount > count && checkPeerHealth() { minCount = count newLeader = p } diff --git a/pkg/schedule/operator/operator.go b/pkg/schedule/operator/operator.go index 4d57d4fc6c7..f89f6606412 100644 --- a/pkg/schedule/operator/operator.go +++ b/pkg/schedule/operator/operator.go @@ -45,7 +45,7 @@ var ( EpochNotMatch CancelReasonType = "epoch not match" // AlreadyExist is the cancel reason when the operator is running. AlreadyExist CancelReasonType = "already exist" - // AdminStop is the cancel reason when the operator is stopped by adminer. + // AdminStop is the cancel reason when the operator is stopped by admin. AdminStop CancelReasonType = "admin stop" // NotInRunningState is the cancel reason when the operator is not in running state. NotInRunningState CancelReasonType = "not in running state" diff --git a/pkg/schedule/operator/operator_controller_test.go b/pkg/schedule/operator/operator_controller_test.go index 3894df7e5e7..16ba899db1d 100644 --- a/pkg/schedule/operator/operator_controller_test.go +++ b/pkg/schedule/operator/operator_controller_test.go @@ -523,7 +523,7 @@ func (suite *operatorControllerTestSuite) TestCheckOperatorLightly() { re.Nil(r) re.Equal(reason, RegionNotFound) - // check failed because of verions of region epoch changed + // check failed because of versions of region epoch changed cluster.PutRegion(target) source.GetMeta().RegionEpoch = &metapb.RegionEpoch{ConfVer: 0, Version: 1} r, reason = controller.checkOperatorLightly(ops[0]) diff --git a/pkg/schedule/plan/plan.go b/pkg/schedule/plan/plan.go index 8a389b9b9e8..5eb8a345914 100644 --- a/pkg/schedule/plan/plan.go +++ b/pkg/schedule/plan/plan.go @@ -30,18 +30,18 @@ type Plan interface { SetStatus(*Status) } -// Summary is used to analyse plan simply. +// Summary is used to analyze plan simply. // It will return the status of store. type Summary func([]Plan) (map[uint64]Status, bool, error) -// Collector is a plan collector +// Collector is a plan collector. type Collector struct { basePlan Plan unschedulablePlans []Plan schedulablePlans []Plan } -// NewCollector returns a new Collector +// NewCollector returns a new Collector. func NewCollector(plan Plan) *Collector { return &Collector{ basePlan: plan, @@ -50,7 +50,7 @@ func NewCollector(plan Plan) *Collector { } } -// Collect is used to collect a new Plan and save it into PlanCollector +// Collect is used to collect a new Plan and save it into PlanCollector. func (c *Collector) Collect(opts ...Option) { if c == nil { return @@ -63,7 +63,7 @@ func (c *Collector) Collect(opts ...Option) { } } -// GetPlans returns all plans and the first part plans are schedulable +// GetPlans returns all plans and the first part plans are schedulable. func (c *Collector) GetPlans() []Plan { if c == nil { return nil @@ -71,24 +71,24 @@ func (c *Collector) GetPlans() []Plan { return append(c.schedulablePlans, c.unschedulablePlans...) } -// Option is to do some action for plan +// Option is to do some action for plan. type Option func(plan Plan) -// SetStatus is used to set status for plan +// SetStatus is used to set status for plan. func SetStatus(status *Status) Option { return func(plan Plan) { plan.SetStatus(status) } } -// SetResource is used to generate Resource for plan +// SetResource is used to generate Resource for plan. func SetResource(resource any) Option { return func(plan Plan) { plan.SetResource(resource) } } -// SetResourceWithStep is used to generate Resource for plan +// SetResourceWithStep is used to generate Resource for plan. func SetResourceWithStep(resource any, step int) Option { return func(plan Plan) { plan.SetResourceWithStep(resource, step) diff --git a/pkg/schedule/scatter/region_scatterer.go b/pkg/schedule/scatter/region_scatterer.go index efef5439fed..71b2cd346c7 100644 --- a/pkg/schedule/scatter/region_scatterer.go +++ b/pkg/schedule/scatter/region_scatterer.go @@ -437,7 +437,7 @@ func isSameDistribution(region *core.RegionInfo, targetPeers map[uint64]*metapb. // selectNewPeer return the new peer which pick the fewest picked count. // it keeps the origin peer if the origin store's pick count is equal the fewest pick. -// it can be diveded into three steps: +// it can be divided into three steps: // 1. found the max pick count and the min pick count. // 2. if max pick count equals min pick count, it means all store picked count are some, return the origin peer. // 3. otherwise, select the store which pick count is the min pick count and pass all filter. diff --git a/pkg/schedule/schedulers/evict_slow_store.go b/pkg/schedule/schedulers/evict_slow_store.go index de581f597bb..8f9c8841e04 100644 --- a/pkg/schedule/schedulers/evict_slow_store.go +++ b/pkg/schedule/schedulers/evict_slow_store.go @@ -94,7 +94,7 @@ func (conf *evictSlowStoreSchedulerConfig) evictStore() uint64 { return conf.getStores()[0] } -// readyForRecovery checks whether the last cpatured candidate is ready for recovery. +// readyForRecovery checks whether the last captured candidate is ready for recovery. func (conf *evictSlowStoreSchedulerConfig) readyForRecovery() bool { conf.RLock() defer conf.RUnlock() diff --git a/pkg/schedule/schedulers/evict_slow_trend.go b/pkg/schedule/schedulers/evict_slow_trend.go index 427787016a2..b63b4b4a2e0 100644 --- a/pkg/schedule/schedulers/evict_slow_trend.go +++ b/pkg/schedule/schedulers/evict_slow_trend.go @@ -147,7 +147,7 @@ func (conf *evictSlowTrendSchedulerConfig) lastCandidateCapturedSecs() uint64 { return DurationSinceAsSecs(conf.lastEvictCandidate.captureTS) } -// readyForRecovery checks whether the last cpatured candidate is ready for recovery. +// readyForRecovery checks whether the last captured candidate is ready for recovery. func (conf *evictSlowTrendSchedulerConfig) readyForRecovery() bool { conf.RLock() defer conf.RUnlock() diff --git a/pkg/schedule/schedulers/metrics.go b/pkg/schedule/schedulers/metrics.go index 42170e43818..90b0b0c6bb2 100644 --- a/pkg/schedule/schedulers/metrics.go +++ b/pkg/schedule/schedulers/metrics.go @@ -110,7 +110,7 @@ var ( Namespace: "pd", Subsystem: "scheduler", Name: "store_slow_trend_evicted_status", - Help: "Store evited by slow trend status for schedule", + Help: "Store evicted by slow trend status for schedule", }, []string{"address", "store"}) storeSlowTrendActionStatusGauge = prometheus.NewGaugeVec( diff --git a/pkg/schedule/schedulers/utils.go b/pkg/schedule/schedulers/utils.go index 1e911cf7b06..7cbfe714aa9 100644 --- a/pkg/schedule/schedulers/utils.go +++ b/pkg/schedule/schedulers/utils.go @@ -286,7 +286,7 @@ func sliceLoadCmp(cmps ...storeLoadCmp) storeLoadCmp { } // stLdRankCmp returns a cmp that compares the two loads with discretized data. -// For example, if the rank function discretice data by step 10 , the load 11 and 19 will be considered equal. +// For example, if the rank function discretize data by step 10 , the load 11 and 19 will be considered equal. func stLdRankCmp(dim func(ld *statistics.StoreLoad) float64, rank func(value float64) int64) storeLoadCmp { return func(ld1, ld2 *statistics.StoreLoad) int { return rankCmp(dim(ld1), dim(ld2), rank) diff --git a/scripts/dashboard-version b/scripts/dashboard-version index 42ff906db4e..d8d960683ae 100644 --- a/scripts/dashboard-version +++ b/scripts/dashboard-version @@ -1,3 +1,3 @@ # This file is updated by running scripts/update-dashboard.sh -# Don't edit it manullay +# Don't edit it manually 8.3.0-e6e78c7c diff --git a/scripts/update-dashboard.sh b/scripts/update-dashboard.sh index 7411960f766..6eff7d5f827 100755 --- a/scripts/update-dashboard.sh +++ b/scripts/update-dashboard.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -euo pipefail -CUR_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +CUR_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" BASE_DIR="$(dirname "$CUR_DIR")" DASHBOARD_VERSION_FILE="$BASE_DIR/scripts/dashboard-version" # old version @@ -23,9 +23,9 @@ if [ "$#" -ge 1 ]; then # so that we don't need to modify the embed-dashboard-ui.sh logic TO_FILE_VERSION=${DASHBOARD_VERSION#v} - echo "# This file is updated by running scripts/update-dashboard.sh" > $DASHBOARD_VERSION_FILE - echo "# Don't edit it manullay" >> $DASHBOARD_VERSION_FILE - echo $TO_FILE_VERSION >> $DASHBOARD_VERSION_FILE + echo "# This file is updated by running scripts/update-dashboard.sh" >$DASHBOARD_VERSION_FILE + echo "# Don't edit it manually" >>$DASHBOARD_VERSION_FILE + echo $TO_FILE_VERSION >>$DASHBOARD_VERSION_FILE fi echo "+ Update dashboard version to $DASHBOARD_VERSION" diff --git a/server/api/middleware.go b/server/api/middleware.go index 010889f08ce..fd0d81412ea 100644 --- a/server/api/middleware.go +++ b/server/api/middleware.go @@ -46,7 +46,7 @@ func (s *serviceMiddlewareBuilder) createHandler(next func(http.ResponseWriter, return negroni.New(append(s.handlers, negroni.WrapFunc(next))...) } -// requestInfoMiddleware is used to gather info from requsetInfo +// requestInfoMiddleware is used to gather info from requestInfo type requestInfoMiddleware struct { svr *server.Server } diff --git a/server/cluster/cluster_test.go b/server/cluster/cluster_test.go index 3f01305b3f1..cffefbba444 100644 --- a/server/cluster/cluster_test.go +++ b/server/cluster/cluster_test.go @@ -952,8 +952,8 @@ func TestRegionHeartbeat(t *testing.T) { re.NoError(cluster.processRegionHeartbeat(ctx, overlapRegion)) tracer.OnAllStageFinished() re.Condition(func() bool { - fileds := tracer.LogFields() - return slice.AllOf(fileds, func(i int) bool { return fileds[i].Integer > 0 }) + fields := tracer.LogFields() + return slice.AllOf(fields, func(i int) bool { return fields[i].Integer > 0 }) }, "should have stats") region = &metapb.Region{} ok, err = storage.LoadRegion(regions[n-1].GetID(), region) diff --git a/tests/server/apiv2/handlers/keyspace_test.go b/tests/server/apiv2/handlers/keyspace_test.go index 18466ca0da7..f3aa55bbe43 100644 --- a/tests/server/apiv2/handlers/keyspace_test.go +++ b/tests/server/apiv2/handlers/keyspace_test.go @@ -109,7 +109,7 @@ func (suite *keyspaceTestSuite) TestUpdateKeyspaceState() { success, disabledAgain := sendUpdateStateRequest(re, suite.server, created.Name, &handlers.UpdateStateParam{State: "disabled"}) re.True(success) re.Equal(disabled, disabledAgain) - // Tombstoning a DISABLED keyspace should not be allowed. + // Tombstone a DISABLED keyspace should not be allowed. success, _ = sendUpdateStateRequest(re, suite.server, created.Name, &handlers.UpdateStateParam{State: "tombstone"}) re.False(success) // Archiving a DISABLED keyspace should be allowed. @@ -119,7 +119,7 @@ func (suite *keyspaceTestSuite) TestUpdateKeyspaceState() { // Enabling an ARCHIVED keyspace is not allowed. success, _ = sendUpdateStateRequest(re, suite.server, created.Name, &handlers.UpdateStateParam{State: "enabled"}) re.False(success) - // Tombstoning an ARCHIVED keyspace is allowed. + // Tombstone an ARCHIVED keyspace is allowed. success, tombstone := sendUpdateStateRequest(re, suite.server, created.Name, &handlers.UpdateStateParam{State: "tombstone"}) re.True(success) re.Equal(keyspacepb.KeyspaceState_TOMBSTONE, tombstone.State) diff --git a/tests/server/cluster/cluster_test.go b/tests/server/cluster/cluster_test.go index c95aa50cb3d..27d3300b293 100644 --- a/tests/server/cluster/cluster_test.go +++ b/tests/server/cluster/cluster_test.go @@ -827,7 +827,7 @@ func TestSetScheduleOpt(t *testing.T) { re := require.New(t) ctx, cancel := context.WithCancel(context.Background()) defer cancel() - // TODO: enable placementrules + // TODO: enable placementRules tc, err := tests.NewTestCluster(ctx, 1, func(cfg *config.Config, _ string) { cfg.Replication.EnablePlacementRules = false }) defer tc.Destroy() re.NoError(err) diff --git a/tools/pd-api-bench/cases/controller.go b/tools/pd-api-bench/cases/controller.go index a4e20f25758..e19c79c0f3e 100644 --- a/tools/pd-api-bench/cases/controller.go +++ b/tools/pd-api-bench/cases/controller.go @@ -231,10 +231,10 @@ func (c *httpController) run() { case <-ticker.C: err := c.Do(c.ctx, hCli) if err != nil { - log.Error("meet erorr when doing HTTP request", zap.String("case", c.Name()), zap.Error(err)) + log.Error("meet error when doing HTTP request", zap.String("case", c.Name()), zap.Error(err)) } case <-c.ctx.Done(): - log.Info("Got signal to exit running HTTP case") + log.Info("got signal to exit running HTTP case") return } } @@ -300,10 +300,10 @@ func (c *gRPCController) run() { case <-ticker.C: err := c.Unary(c.ctx, cli) if err != nil { - log.Error("meet erorr when doing gRPC request", zap.String("case", c.Name()), zap.Error(err)) + log.Error("meet error when doing gRPC request", zap.String("case", c.Name()), zap.Error(err)) } case <-c.ctx.Done(): - log.Info("Got signal to exit running gRPC case") + log.Info("got signal to exit running gRPC case") return } } @@ -374,10 +374,10 @@ func (c *etcdController) run() { case <-ticker.C: err := c.Unary(c.ctx, cli) if err != nil { - log.Error("meet erorr when doing etcd request", zap.String("case", c.Name()), zap.Error(err)) + log.Error("meet error when doing etcd request", zap.String("case", c.Name()), zap.Error(err)) } case <-c.ctx.Done(): - log.Info("Got signal to exit running etcd case") + log.Info("got signal to exit running etcd case") return } } diff --git a/tools/pd-ctl/pdctl/command/config_command.go b/tools/pd-ctl/pdctl/command/config_command.go index 6f88e06a0dd..4efaae3fa13 100644 --- a/tools/pd-ctl/pdctl/command/config_command.go +++ b/tools/pd-ctl/pdctl/command/config_command.go @@ -77,7 +77,7 @@ func NewShowConfigCommand() *cobra.Command { sc.AddCommand(NewShowClusterVersionCommand()) sc.AddCommand(newShowReplicationModeCommand()) sc.AddCommand(NewShowServerConfigCommand()) - sc.Flags().Bool(flagFromAPIServer, false, "read data from api server rather than micor service") + sc.Flags().Bool(flagFromAPIServer, false, "read data from api server rather than micro service") return sc } @@ -88,7 +88,7 @@ func NewShowAllConfigCommand() *cobra.Command { Short: "show all config of PD", Run: showAllConfigCommandFunc, } - sc.Flags().Bool(flagFromAPIServer, false, "read data from api server rather than micor service") + sc.Flags().Bool(flagFromAPIServer, false, "read data from api server rather than micro service") return sc } @@ -99,7 +99,7 @@ func NewShowScheduleConfigCommand() *cobra.Command { Short: "show schedule config of PD", Run: showScheduleConfigCommandFunc, } - sc.Flags().Bool(flagFromAPIServer, false, "read data from api server rather than micor service") + sc.Flags().Bool(flagFromAPIServer, false, "read data from api server rather than micro service") return sc } @@ -110,7 +110,7 @@ func NewShowReplicationConfigCommand() *cobra.Command { Short: "show replication config of PD", Run: showReplicationConfigCommandFunc, } - sc.Flags().Bool(flagFromAPIServer, false, "read data from api server rather than micor service") + sc.Flags().Bool(flagFromAPIServer, false, "read data from api server rather than micro service") return sc } @@ -447,7 +447,7 @@ func NewPlacementRulesCommand() *cobra.Command { show.Flags().String("id", "", "rule id") show.Flags().String("region", "", "region id") show.Flags().Bool("detail", false, "detailed match info for region") - show.Flags().Bool(flagFromAPIServer, false, "read data from api server rather than micor service") + show.Flags().Bool(flagFromAPIServer, false, "read data from api server rather than micro service") load := &cobra.Command{ Use: "load", Short: "load placement rules to a file", @@ -457,7 +457,7 @@ func NewPlacementRulesCommand() *cobra.Command { load.Flags().String("id", "", "rule id") load.Flags().String("region", "", "region id") load.Flags().String("out", "rules.json", "the filename contains rules") - load.Flags().Bool(flagFromAPIServer, false, "read data from api server rather than micor service") + load.Flags().Bool(flagFromAPIServer, false, "read data from api server rather than micro service") save := &cobra.Command{ Use: "save", Short: "save rules from file", @@ -473,7 +473,7 @@ func NewPlacementRulesCommand() *cobra.Command { Short: "show rule group configuration(s)", Run: showRuleGroupFunc, } - ruleGroupShow.Flags().Bool(flagFromAPIServer, false, "read data from api server rather than micor service") + ruleGroupShow.Flags().Bool(flagFromAPIServer, false, "read data from api server rather than micro service") ruleGroupSet := &cobra.Command{ Use: "set ", Short: "update rule group configuration", @@ -496,7 +496,7 @@ func NewPlacementRulesCommand() *cobra.Command { Run: getRuleBundle, } ruleBundleGet.Flags().String("out", "", "the output file") - ruleBundleGet.Flags().Bool(flagFromAPIServer, false, "read data from api server rather than micor service") + ruleBundleGet.Flags().Bool(flagFromAPIServer, false, "read data from api server rather than micro service") ruleBundleSet := &cobra.Command{ Use: "set", Short: "set rule group config and its rules from file", @@ -515,7 +515,7 @@ func NewPlacementRulesCommand() *cobra.Command { Run: loadRuleBundle, } ruleBundleLoad.Flags().String("out", "rules.json", "the output file") - ruleBundleLoad.Flags().Bool(flagFromAPIServer, false, "read data from api server rather than micor service") + ruleBundleLoad.Flags().Bool(flagFromAPIServer, false, "read data from api server rather than micro service") ruleBundleSave := &cobra.Command{ Use: "save", Short: "save all group configs and rules from file", diff --git a/tools/pd-simulator/README.md b/tools/pd-simulator/README.md index 107f6c40f64..d0519bfedff 100644 --- a/tools/pd-simulator/README.md +++ b/tools/pd-simulator/README.md @@ -44,7 +44,7 @@ Run a specific case with an external PD: ./pd-simulator -pd="http://127.0.0.1:2379" -case="casename" ``` -Run with tiup playgroudn : +Run with tiup playground: ```shell tiup playground nightly --host 127.0.0.1 --kv.binpath ./pd-simulator --kv=1 --db=0 --kv.config=./tikv.conf ``` From 6bf980de258c8e13c032fa023504ba30929d7f48 Mon Sep 17 00:00:00 2001 From: Hu# Date: Tue, 20 Aug 2024 15:51:12 +0800 Subject: [PATCH 06/18] tests/tso: Create independent clients to prevent interfering with other tests. (#8546) ref tikv/pd#8103 tests/tso: Create independent clients to prevent interfering with other tests. Signed-off-by: husharp Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- tests/integrations/tso/client_test.go | 53 ++++++++++++++------------- 1 file changed, 27 insertions(+), 26 deletions(-) diff --git a/tests/integrations/tso/client_test.go b/tests/integrations/tso/client_test.go index f5f33240d18..a669e093200 100644 --- a/tests/integrations/tso/client_test.go +++ b/tests/integrations/tso/client_test.go @@ -103,17 +103,7 @@ func (suite *tsoClientTestSuite) SetupSuite() { suite.backendEndpoints = suite.pdLeaderServer.GetAddr() suite.keyspaceIDs = make([]uint32, 0) - if suite.legacy { - client, err := pd.NewClientWithContext(suite.ctx, suite.getBackendEndpoints(), pd.SecurityOption{}, pd.WithForwardingOption(true)) - re.NoError(err) - innerClient, ok := client.(interface{ GetServiceDiscovery() pd.ServiceDiscovery }) - re.True(ok) - re.Equal(constant.NullKeyspaceID, innerClient.GetServiceDiscovery().GetKeyspaceID()) - re.Equal(constant.DefaultKeyspaceGroupID, innerClient.GetServiceDiscovery().GetKeyspaceGroupID()) - mcs.WaitForTSOServiceAvailable(suite.ctx, re, client) - suite.clients = make([]pd.Client, 0) - suite.clients = append(suite.clients, client) - } else { + if !suite.legacy { suite.tsoCluster, err = tests.NewTestTSOCluster(suite.ctx, 3, suite.backendEndpoints) re.NoError(err) @@ -148,7 +138,23 @@ func (suite *tsoClientTestSuite) SetupSuite() { }, }) } + } +} +// Create independent clients to prevent interfering with other tests. +func (suite *tsoClientTestSuite) SetupTest() { + re := suite.Require() + if suite.legacy { + client, err := pd.NewClientWithContext(suite.ctx, suite.getBackendEndpoints(), pd.SecurityOption{}, pd.WithForwardingOption(true)) + re.NoError(err) + innerClient, ok := client.(interface{ GetServiceDiscovery() pd.ServiceDiscovery }) + re.True(ok) + re.Equal(constant.NullKeyspaceID, innerClient.GetServiceDiscovery().GetKeyspaceID()) + re.Equal(constant.DefaultKeyspaceGroupID, innerClient.GetServiceDiscovery().GetKeyspaceGroupID()) + mcs.WaitForTSOServiceAvailable(suite.ctx, re, client) + suite.clients = make([]pd.Client, 0) + suite.clients = append(suite.clients, client) + } else { suite.waitForAllKeyspaceGroupsInServing(re) } } @@ -183,15 +189,18 @@ func (suite *tsoClientTestSuite) waitForAllKeyspaceGroupsInServing(re *require.A re.Equal(len(suite.keyspaceIDs), len(suite.clients)) } +func (suite *tsoClientTestSuite) TearDownTest() { + for _, client := range suite.clients { + client.Close() + } +} + func (suite *tsoClientTestSuite) TearDownSuite() { suite.cancel() if !suite.legacy { suite.tsoCluster.Destroy() } suite.cluster.Destroy() - for _, client := range suite.clients { - client.Close() - } } func (suite *tsoClientTestSuite) TestGetTS() { @@ -432,21 +441,13 @@ func (suite *tsoClientTestSuite) TestGetTSWhileResettingTSOClient() { re := suite.Require() re.NoError(failpoint.Enable("github.com/tikv/pd/client/delayDispatchTSORequest", "return(true)")) var ( - clients []pd.Client stopSignal atomic.Bool wg sync.WaitGroup ) - // Create independent clients to prevent interfering with other tests. - if suite.legacy { - client, err := pd.NewClientWithContext(suite.ctx, suite.getBackendEndpoints(), pd.SecurityOption{}, pd.WithForwardingOption(true)) - re.NoError(err) - clients = []pd.Client{client} - } else { - clients = mcs.WaitForMultiKeyspacesTSOAvailable(suite.ctx, re, suite.keyspaceIDs, suite.getBackendEndpoints()) - } - wg.Add(tsoRequestConcurrencyNumber * len(clients)) + + wg.Add(tsoRequestConcurrencyNumber * len(suite.clients)) for i := 0; i < tsoRequestConcurrencyNumber; i++ { - for _, client := range clients { + for _, client := range suite.clients { go func(client pd.Client) { defer wg.Done() var lastTS uint64 @@ -465,7 +466,7 @@ func (suite *tsoClientTestSuite) TestGetTSWhileResettingTSOClient() { } // Reset the TSO clients while requesting TSO concurrently. for i := 0; i < tsoRequestConcurrencyNumber; i++ { - for _, client := range clients { + for _, client := range suite.clients { client.(interface{ ResetTSOClient() }).ResetTSOClient() } } From 2e8f20fef9ce497f9a2d9ed2be3174ecf16fd17e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=90=8D=E7=99=BD?= <251098199@qq.com> Date: Wed, 21 Aug 2024 11:21:11 +0800 Subject: [PATCH 07/18] server: refactor params of `askSplit` call (#8537) close tikv/pd#5014 Signed-off-by: qingfeng777 <251098199@qq.com> Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- server/grpc_service.go | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/server/grpc_service.go b/server/grpc_service.go index f7a0ea9443e..d3db45c8364 100644 --- a/server/grpc_service.go +++ b/server/grpc_service.go @@ -1756,10 +1756,7 @@ func (s *GrpcServer) AskSplit(ctx context.Context, request *pdpb.AskSplitRequest "missing region for split"), }, nil } - req := &pdpb.AskSplitRequest{ - Region: request.Region, - } - split, err := rc.HandleAskSplit(req) + split, err := rc.HandleAskSplit(request) if err != nil { return &pdpb.AskSplitResponse{ Header: s.wrapErrorToHeader(pdpb.ErrorType_UNKNOWN, err.Error()), @@ -1836,11 +1833,7 @@ func (s *GrpcServer) AskBatchSplit(ctx context.Context, request *pdpb.AskBatchSp "missing region for split"), }, nil } - req := &pdpb.AskBatchSplitRequest{ - Region: request.Region, - SplitCount: request.SplitCount, - } - split, err := rc.HandleAskBatchSplit(req) + split, err := rc.HandleAskBatchSplit(request) if err != nil { return &pdpb.AskBatchSplitResponse{ Header: s.wrapErrorToHeader(pdpb.ErrorType_UNKNOWN, err.Error()), From 9c14c63a06a6230eaf1a59fe0aa97342015da1f0 Mon Sep 17 00:00:00 2001 From: lhy1024 Date: Wed, 21 Aug 2024 12:07:42 +0800 Subject: [PATCH 08/18] mcs: fix potential data race in scheduling server (#8539) close tikv/pd#8538 Signed-off-by: lhy1024 Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- pkg/mcs/scheduling/server/server.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/pkg/mcs/scheduling/server/server.go b/pkg/mcs/scheduling/server/server.go index e1753cf2972..9ea369aae9e 100644 --- a/pkg/mcs/scheduling/server/server.go +++ b/pkg/mcs/scheduling/server/server.go @@ -55,6 +55,7 @@ import ( "github.com/tikv/pd/pkg/storage/endpoint" "github.com/tikv/pd/pkg/storage/kv" "github.com/tikv/pd/pkg/utils/apiutil" + "github.com/tikv/pd/pkg/utils/etcdutil" "github.com/tikv/pd/pkg/utils/grpcutil" "github.com/tikv/pd/pkg/utils/logutil" "github.com/tikv/pd/pkg/utils/memberutil" @@ -193,7 +194,7 @@ func (s *Server) updateAPIServerMemberLoop() { if !s.IsServing() { continue } - members, err := s.GetClient().MemberList(ctx) + members, err := etcdutil.ListEtcdMembers(ctx, s.GetClient()) if err != nil { log.Warn("failed to list members", errs.ZapError(err)) continue @@ -212,6 +213,11 @@ func (s *Server) updateAPIServerMemberLoop() { cc, err := s.GetDelegateClient(ctx, s.GetTLSConfig(), ep.ClientURLs[0]) if err != nil { log.Info("failed to get delegate client", errs.ZapError(err)) + continue + } + if !s.IsServing() { + // double check + break } if s.cluster.SwitchAPIServerLeader(pdpb.NewPDClient(cc)) { if status.Leader != curLeader { From ce1d0e800f34130e56677861d1570a47c7aa2a87 Mon Sep 17 00:00:00 2001 From: okJiang <819421878@qq.com> Date: Wed, 21 Aug 2024 13:24:11 +0800 Subject: [PATCH 09/18] scheduler: remove old scheduler name (#8516) ref tikv/pd#8379 Signed-off-by: okJiang <819421878@qq.com> Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- pkg/mcs/scheduling/server/cluster.go | 2 +- pkg/mcs/scheduling/server/config/config.go | 2 +- pkg/mock/mockconfig/mockconfig.go | 2 +- pkg/schedule/checker/merge_checker.go | 2 +- pkg/schedule/checker/replica_checker.go | 2 +- pkg/schedule/checker/rule_checker.go | 2 +- pkg/schedule/config/config.go | 2 +- pkg/schedule/config/config_provider.go | 2 +- pkg/schedule/coordinator.go | 4 +- pkg/schedule/filter/counter_test.go | 2 +- pkg/schedule/handler/handler.go | 4 +- pkg/schedule/operator/metrics.go | 2 +- pkg/schedule/schedulers/balance_leader.go | 4 +- pkg/schedule/schedulers/balance_region.go | 7 +-- pkg/schedule/schedulers/balance_test.go | 2 +- pkg/schedule/schedulers/balance_witness.go | 4 +- .../schedulers/balance_witness_test.go | 2 +- pkg/schedule/schedulers/base_scheduler.go | 2 +- .../schedulers/diagnostic_recorder.go | 36 ++++++++------ pkg/schedule/schedulers/evict_leader.go | 6 +-- pkg/schedule/schedulers/evict_leader_test.go | 2 +- pkg/schedule/schedulers/evict_slow_store.go | 5 +- .../schedulers/evict_slow_store_test.go | 2 +- pkg/schedule/schedulers/evict_slow_trend.go | 7 +-- .../schedulers/evict_slow_trend_test.go | 2 +- pkg/schedule/schedulers/grant_hot_region.go | 7 +-- pkg/schedule/schedulers/grant_leader.go | 9 +--- pkg/schedule/schedulers/hot_region.go | 6 +-- pkg/schedule/schedulers/hot_region_test.go | 8 ++-- pkg/schedule/schedulers/init.go | 2 +- pkg/schedule/schedulers/label.go | 9 +--- pkg/schedule/schedulers/metrics.go | 2 +- pkg/schedule/schedulers/random_merge.go | 7 +-- pkg/schedule/schedulers/scatter_range.go | 7 +-- pkg/schedule/schedulers/scheduler.go | 2 +- .../schedulers/scheduler_controller.go | 5 +- pkg/schedule/schedulers/scheduler_test.go | 2 +- pkg/schedule/schedulers/shuffle_hot_region.go | 7 +-- pkg/schedule/schedulers/shuffle_leader.go | 7 +-- pkg/schedule/schedulers/shuffle_region.go | 13 ++--- pkg/schedule/schedulers/split_bucket.go | 4 +- .../schedulers/transfer_witness_leader.go | 4 +- .../transfer_witness_leader_test.go | 2 +- pkg/schedule/{type => types}/type.go | 0 plugin/scheduler_example/evict_leader.go | 2 +- server/api/diagnostic_test.go | 13 ++--- server/api/scheduler.go | 11 ++--- server/cluster/cluster_test.go | 48 +++++++++---------- server/cluster/scheduling_controller.go | 3 +- server/config/persist_options.go | 2 +- server/handler.go | 2 +- .../mcs/scheduling/config_test.go | 15 +++--- .../mcs/scheduling/server_test.go | 35 +++++++------- .../realcluster/scheduler_test.go | 26 +++++----- tests/server/api/scheduler_test.go | 2 +- tests/server/cluster/cluster_test.go | 11 +++-- 56 files changed, 164 insertions(+), 218 deletions(-) rename pkg/schedule/{type => types}/type.go (100%) diff --git a/pkg/mcs/scheduling/server/cluster.go b/pkg/mcs/scheduling/server/cluster.go index a5a3a709184..31120ae6082 100644 --- a/pkg/mcs/scheduling/server/cluster.go +++ b/pkg/mcs/scheduling/server/cluster.go @@ -27,7 +27,7 @@ import ( "github.com/tikv/pd/pkg/schedule/scatter" "github.com/tikv/pd/pkg/schedule/schedulers" "github.com/tikv/pd/pkg/schedule/splitter" - types "github.com/tikv/pd/pkg/schedule/type" + "github.com/tikv/pd/pkg/schedule/types" "github.com/tikv/pd/pkg/slice" "github.com/tikv/pd/pkg/statistics" "github.com/tikv/pd/pkg/statistics/buckets" diff --git a/pkg/mcs/scheduling/server/config/config.go b/pkg/mcs/scheduling/server/config/config.go index 4b855d09899..4d7cee91f4a 100644 --- a/pkg/mcs/scheduling/server/config/config.go +++ b/pkg/mcs/scheduling/server/config/config.go @@ -36,7 +36,7 @@ import ( "github.com/tikv/pd/pkg/core/storelimit" mcsconstant "github.com/tikv/pd/pkg/mcs/utils/constant" sc "github.com/tikv/pd/pkg/schedule/config" - types "github.com/tikv/pd/pkg/schedule/type" + "github.com/tikv/pd/pkg/schedule/types" "github.com/tikv/pd/pkg/slice" "github.com/tikv/pd/pkg/storage/endpoint" "github.com/tikv/pd/pkg/utils/configutil" diff --git a/pkg/mock/mockconfig/mockconfig.go b/pkg/mock/mockconfig/mockconfig.go index ccd1f98154d..6e595f982c1 100644 --- a/pkg/mock/mockconfig/mockconfig.go +++ b/pkg/mock/mockconfig/mockconfig.go @@ -16,7 +16,7 @@ package mockconfig import ( sc "github.com/tikv/pd/pkg/schedule/config" - types "github.com/tikv/pd/pkg/schedule/type" + "github.com/tikv/pd/pkg/schedule/types" "github.com/tikv/pd/server/config" ) diff --git a/pkg/schedule/checker/merge_checker.go b/pkg/schedule/checker/merge_checker.go index 65189d35c1d..d5a39da83ae 100644 --- a/pkg/schedule/checker/merge_checker.go +++ b/pkg/schedule/checker/merge_checker.go @@ -31,7 +31,7 @@ import ( "github.com/tikv/pd/pkg/schedule/labeler" "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/placement" - types "github.com/tikv/pd/pkg/schedule/type" + "github.com/tikv/pd/pkg/schedule/types" "github.com/tikv/pd/pkg/utils/logutil" ) diff --git a/pkg/schedule/checker/replica_checker.go b/pkg/schedule/checker/replica_checker.go index b0c42e88258..a21e19b3d66 100644 --- a/pkg/schedule/checker/replica_checker.go +++ b/pkg/schedule/checker/replica_checker.go @@ -26,7 +26,7 @@ import ( "github.com/tikv/pd/pkg/schedule/config" sche "github.com/tikv/pd/pkg/schedule/core" "github.com/tikv/pd/pkg/schedule/operator" - types "github.com/tikv/pd/pkg/schedule/type" + "github.com/tikv/pd/pkg/schedule/types" "go.uber.org/zap" ) diff --git a/pkg/schedule/checker/rule_checker.go b/pkg/schedule/checker/rule_checker.go index 23018ad7096..82807441bf8 100644 --- a/pkg/schedule/checker/rule_checker.go +++ b/pkg/schedule/checker/rule_checker.go @@ -31,7 +31,7 @@ import ( "github.com/tikv/pd/pkg/schedule/filter" "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/placement" - types "github.com/tikv/pd/pkg/schedule/type" + "github.com/tikv/pd/pkg/schedule/types" "github.com/tikv/pd/pkg/versioninfo" "go.uber.org/zap" ) diff --git a/pkg/schedule/config/config.go b/pkg/schedule/config/config.go index 124bff0a704..344569d6460 100644 --- a/pkg/schedule/config/config.go +++ b/pkg/schedule/config/config.go @@ -20,7 +20,7 @@ import ( "github.com/pingcap/errors" "github.com/pingcap/kvproto/pkg/metapb" "github.com/tikv/pd/pkg/core/storelimit" - types "github.com/tikv/pd/pkg/schedule/type" + "github.com/tikv/pd/pkg/schedule/types" "github.com/tikv/pd/pkg/utils/configutil" "github.com/tikv/pd/pkg/utils/syncutil" "github.com/tikv/pd/pkg/utils/typeutil" diff --git a/pkg/schedule/config/config_provider.go b/pkg/schedule/config/config_provider.go index d7bc38a7c03..95bcad5add0 100644 --- a/pkg/schedule/config/config_provider.go +++ b/pkg/schedule/config/config_provider.go @@ -22,7 +22,7 @@ import ( "github.com/pingcap/kvproto/pkg/metapb" "github.com/tikv/pd/pkg/core/constant" "github.com/tikv/pd/pkg/core/storelimit" - types "github.com/tikv/pd/pkg/schedule/type" + "github.com/tikv/pd/pkg/schedule/types" "github.com/tikv/pd/pkg/storage/endpoint" ) diff --git a/pkg/schedule/coordinator.go b/pkg/schedule/coordinator.go index 89c99ac90b8..2a31045129e 100644 --- a/pkg/schedule/coordinator.go +++ b/pkg/schedule/coordinator.go @@ -34,7 +34,7 @@ import ( "github.com/tikv/pd/pkg/schedule/scatter" "github.com/tikv/pd/pkg/schedule/schedulers" "github.com/tikv/pd/pkg/schedule/splitter" - types "github.com/tikv/pd/pkg/schedule/type" + "github.com/tikv/pd/pkg/schedule/types" "github.com/tikv/pd/pkg/statistics" "github.com/tikv/pd/pkg/statistics/utils" "github.com/tikv/pd/pkg/utils/logutil" @@ -182,7 +182,7 @@ func (c *Coordinator) driveSlowNodeScheduler() { case <-ticker.C: { // If enabled, exit. - if exists, _ := c.schedulers.IsSchedulerExisted(schedulers.EvictSlowTrendName); exists { + if exists, _ := c.schedulers.IsSchedulerExisted(types.EvictSlowTrendScheduler.String()); exists { return } // If the cluster was set up with `raft-kv2` engine, this cluster should diff --git a/pkg/schedule/filter/counter_test.go b/pkg/schedule/filter/counter_test.go index 5b8d5144412..7c7acc5e9a5 100644 --- a/pkg/schedule/filter/counter_test.go +++ b/pkg/schedule/filter/counter_test.go @@ -18,7 +18,7 @@ import ( "testing" "github.com/stretchr/testify/require" - types "github.com/tikv/pd/pkg/schedule/type" + "github.com/tikv/pd/pkg/schedule/types" ) func TestString(t *testing.T) { diff --git a/pkg/schedule/handler/handler.go b/pkg/schedule/handler/handler.go index 748a17b87ef..a8540b4b5f4 100644 --- a/pkg/schedule/handler/handler.go +++ b/pkg/schedule/handler/handler.go @@ -38,6 +38,7 @@ import ( "github.com/tikv/pd/pkg/schedule/placement" "github.com/tikv/pd/pkg/schedule/scatter" "github.com/tikv/pd/pkg/schedule/schedulers" + "github.com/tikv/pd/pkg/schedule/types" "github.com/tikv/pd/pkg/statistics" "github.com/tikv/pd/pkg/statistics/buckets" "github.com/tikv/pd/pkg/statistics/utils" @@ -884,7 +885,8 @@ func (h *Handler) GetSchedulerByStatus(status string, needTS bool) (any, error) // GetDiagnosticResult returns the diagnostic results of the specified scheduler. func (h *Handler) GetDiagnosticResult(name string) (*schedulers.DiagnosticResult, error) { - if _, ok := schedulers.DiagnosableSummaryFunc[name]; !ok { + tp := types.StringToSchedulerType[name] + if _, ok := schedulers.DiagnosableSummaryFunc[tp]; !ok { return nil, errs.ErrSchedulerUndiagnosable.FastGenByArgs(name) } co := h.GetCoordinator() diff --git a/pkg/schedule/operator/metrics.go b/pkg/schedule/operator/metrics.go index 74f9ddad0c7..47a165500e9 100644 --- a/pkg/schedule/operator/metrics.go +++ b/pkg/schedule/operator/metrics.go @@ -16,7 +16,7 @@ package operator import ( "github.com/prometheus/client_golang/prometheus" - types "github.com/tikv/pd/pkg/schedule/type" + "github.com/tikv/pd/pkg/schedule/types" ) var ( diff --git a/pkg/schedule/schedulers/balance_leader.go b/pkg/schedule/schedulers/balance_leader.go index ef254ee6950..60dbee79dc4 100644 --- a/pkg/schedule/schedulers/balance_leader.go +++ b/pkg/schedule/schedulers/balance_leader.go @@ -31,7 +31,7 @@ import ( "github.com/tikv/pd/pkg/schedule/filter" "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/plan" - types "github.com/tikv/pd/pkg/schedule/type" + "github.com/tikv/pd/pkg/schedule/types" "github.com/tikv/pd/pkg/utils/reflectutil" "github.com/tikv/pd/pkg/utils/syncutil" "github.com/tikv/pd/pkg/utils/typeutil" @@ -40,8 +40,6 @@ import ( ) const ( - // BalanceLeaderName is balance leader scheduler name. - BalanceLeaderName = "balance-leader-scheduler" // BalanceLeaderBatchSize is the default number of operators to transfer leaders by one scheduling. // Default value is 4 which is subjected by scheduler-max-waiting-operator and leader-schedule-limit // If you want to increase balance speed more, please increase above-mentioned param. diff --git a/pkg/schedule/schedulers/balance_region.go b/pkg/schedule/schedulers/balance_region.go index 174b6af1c83..1ebe65d732c 100644 --- a/pkg/schedule/schedulers/balance_region.go +++ b/pkg/schedule/schedulers/balance_region.go @@ -26,15 +26,10 @@ import ( "github.com/tikv/pd/pkg/schedule/filter" "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/plan" - types "github.com/tikv/pd/pkg/schedule/type" + "github.com/tikv/pd/pkg/schedule/types" "go.uber.org/zap" ) -const ( - // BalanceRegionName is balance region scheduler name. - BalanceRegionName = "balance-region-scheduler" -) - type balanceRegionSchedulerConfig struct { Ranges []core.KeyRange `json:"ranges"` // TODO: When we prepare to use Ranges, we will need to implement the ReloadConfig function for this scheduler. diff --git a/pkg/schedule/schedulers/balance_test.go b/pkg/schedule/schedulers/balance_test.go index 2f163742bbb..e9ae771d18b 100644 --- a/pkg/schedule/schedulers/balance_test.go +++ b/pkg/schedule/schedulers/balance_test.go @@ -31,7 +31,7 @@ import ( "github.com/tikv/pd/pkg/schedule/config" "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/plan" - types "github.com/tikv/pd/pkg/schedule/type" + "github.com/tikv/pd/pkg/schedule/types" "github.com/tikv/pd/pkg/storage" "github.com/tikv/pd/pkg/utils/operatorutil" "github.com/tikv/pd/pkg/versioninfo" diff --git a/pkg/schedule/schedulers/balance_witness.go b/pkg/schedule/schedulers/balance_witness.go index 450c43647cf..47f23d470cc 100644 --- a/pkg/schedule/schedulers/balance_witness.go +++ b/pkg/schedule/schedulers/balance_witness.go @@ -32,7 +32,7 @@ import ( "github.com/tikv/pd/pkg/schedule/filter" "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/plan" - types "github.com/tikv/pd/pkg/schedule/type" + "github.com/tikv/pd/pkg/schedule/types" "github.com/tikv/pd/pkg/utils/reflectutil" "github.com/tikv/pd/pkg/utils/syncutil" "github.com/unrolled/render" @@ -40,8 +40,6 @@ import ( ) const ( - // BalanceWitnessName is balance witness scheduler name. - BalanceWitnessName = "balance-witness-scheduler" // balanceWitnessBatchSize is the default number of operators to transfer witnesses by one scheduling. // Default value is 4 which is subjected by scheduler-max-waiting-operator and witness-schedule-limit // If you want to increase balance speed more, please increase above-mentioned param. diff --git a/pkg/schedule/schedulers/balance_witness_test.go b/pkg/schedule/schedulers/balance_witness_test.go index d8715f71784..03fcac77ccc 100644 --- a/pkg/schedule/schedulers/balance_witness_test.go +++ b/pkg/schedule/schedulers/balance_witness_test.go @@ -23,7 +23,7 @@ import ( "github.com/tikv/pd/pkg/schedule/config" "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/placement" - types "github.com/tikv/pd/pkg/schedule/type" + "github.com/tikv/pd/pkg/schedule/types" "github.com/tikv/pd/pkg/storage" ) diff --git a/pkg/schedule/schedulers/base_scheduler.go b/pkg/schedule/schedulers/base_scheduler.go index b3dae9856e6..6e160effea7 100644 --- a/pkg/schedule/schedulers/base_scheduler.go +++ b/pkg/schedule/schedulers/base_scheduler.go @@ -23,7 +23,7 @@ import ( "github.com/tikv/pd/pkg/errs" sche "github.com/tikv/pd/pkg/schedule/core" "github.com/tikv/pd/pkg/schedule/operator" - types "github.com/tikv/pd/pkg/schedule/type" + "github.com/tikv/pd/pkg/schedule/types" "github.com/tikv/pd/pkg/utils/typeutil" ) diff --git a/pkg/schedule/schedulers/diagnostic_recorder.go b/pkg/schedule/schedulers/diagnostic_recorder.go index df57dbebe71..cd99262ee48 100644 --- a/pkg/schedule/schedulers/diagnostic_recorder.go +++ b/pkg/schedule/schedulers/diagnostic_recorder.go @@ -23,6 +23,7 @@ import ( sc "github.com/tikv/pd/pkg/schedule/config" "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/plan" + "github.com/tikv/pd/pkg/schedule/types" ) const ( @@ -46,27 +47,27 @@ const ( // DiagnosableSummaryFunc includes all implementations of plan.Summary. // And it also includes all schedulers which pd support to diagnose. -var DiagnosableSummaryFunc = map[string]plan.Summary{ - BalanceRegionName: plan.BalancePlanSummary, - BalanceLeaderName: plan.BalancePlanSummary, +var DiagnosableSummaryFunc = map[types.CheckerSchedulerType]plan.Summary{ + types.BalanceRegionScheduler: plan.BalancePlanSummary, + types.BalanceLeaderScheduler: plan.BalancePlanSummary, } // DiagnosticRecorder is used to manage diagnostic for one scheduler. type DiagnosticRecorder struct { - schedulerName string + schedulerType types.CheckerSchedulerType config sc.SchedulerConfigProvider summaryFunc plan.Summary results *cache.FIFO } // NewDiagnosticRecorder creates a new DiagnosticRecorder. -func NewDiagnosticRecorder(name string, config sc.SchedulerConfigProvider) *DiagnosticRecorder { - summaryFunc, ok := DiagnosableSummaryFunc[name] +func NewDiagnosticRecorder(tp types.CheckerSchedulerType, config sc.SchedulerConfigProvider) *DiagnosticRecorder { + summaryFunc, ok := DiagnosableSummaryFunc[tp] if !ok { return nil } return &DiagnosticRecorder{ - schedulerName: name, + schedulerType: tp, config: config, summaryFunc: summaryFunc, results: cache.NewFIFO(maxDiagnosticResultNum), @@ -131,11 +132,11 @@ func (d *DiagnosticRecorder) GetLastResult() *DiagnosticResult { } } else if firstStatus == Pending { // This is used to handle pending status because of reach limit in `IsScheduleAllowed` - resStr = fmt.Sprintf("%s reach limit", d.schedulerName) + resStr = fmt.Sprintf("%v reach limit", d.schedulerType) } } return &DiagnosticResult{ - Name: d.schedulerName, + Name: d.schedulerType.String(), Status: firstStatus, Summary: resStr, Timestamp: uint64(time.Now().Unix()), @@ -147,7 +148,11 @@ func (d *DiagnosticRecorder) SetResultFromStatus(status string) { if d == nil { return } - result := &DiagnosticResult{Name: d.schedulerName, Timestamp: uint64(time.Now().Unix()), Status: status} + result := &DiagnosticResult{ + Name: d.schedulerType.String(), + Timestamp: uint64(time.Now().Unix()), + Status: status, + } d.results.Put(result.Timestamp, result) } @@ -161,11 +166,14 @@ func (d *DiagnosticRecorder) SetResultFromPlans(ops []*operator.Operator, plans } func (d *DiagnosticRecorder) analyze(ops []*operator.Operator, plans []plan.Plan, ts uint64) *DiagnosticResult { - res := &DiagnosticResult{Name: d.schedulerName, Timestamp: ts, Status: Normal} - name := d.schedulerName + res := &DiagnosticResult{ + Name: d.schedulerType.String(), + Timestamp: ts, + Status: Normal, + } // TODO: support more schedulers and checkers - switch name { - case BalanceRegionName, BalanceLeaderName: + switch d.schedulerType { + case types.BalanceRegionScheduler, types.BalanceLeaderScheduler: if len(ops) != 0 { res.Status = Scheduling return res diff --git a/pkg/schedule/schedulers/evict_leader.go b/pkg/schedule/schedulers/evict_leader.go index 85f861f0082..a7d656a3e42 100644 --- a/pkg/schedule/schedulers/evict_leader.go +++ b/pkg/schedule/schedulers/evict_leader.go @@ -28,7 +28,7 @@ import ( "github.com/tikv/pd/pkg/schedule/filter" "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/plan" - types "github.com/tikv/pd/pkg/schedule/type" + "github.com/tikv/pd/pkg/schedule/types" "github.com/tikv/pd/pkg/utils/apiutil" "github.com/tikv/pd/pkg/utils/syncutil" "github.com/unrolled/render" @@ -36,8 +36,6 @@ import ( ) const ( - // EvictLeaderName is evict leader scheduler name. - EvictLeaderName = "evict-leader-scheduler" // EvictLeaderBatchSize is the number of operators to transfer // leaders by one scheduling EvictLeaderBatchSize = 3 @@ -212,7 +210,7 @@ func (conf *evictLeaderSchedulerConfig) delete(id uint64) (any, error) { return resp, nil } conf.Unlock() - if err := conf.removeSchedulerCb(EvictLeaderName); err != nil { + if err := conf.removeSchedulerCb(types.EvictLeaderScheduler.String()); err != nil { if !errors.ErrorEqual(err, errs.ErrSchedulerNotFound.FastGenByArgs()) { conf.resetStore(id, keyRanges) } diff --git a/pkg/schedule/schedulers/evict_leader_test.go b/pkg/schedule/schedulers/evict_leader_test.go index 692dda63437..b2c84ec68b7 100644 --- a/pkg/schedule/schedulers/evict_leader_test.go +++ b/pkg/schedule/schedulers/evict_leader_test.go @@ -23,7 +23,7 @@ import ( "github.com/stretchr/testify/require" "github.com/tikv/pd/pkg/core" "github.com/tikv/pd/pkg/schedule/operator" - types "github.com/tikv/pd/pkg/schedule/type" + "github.com/tikv/pd/pkg/schedule/types" "github.com/tikv/pd/pkg/storage" "github.com/tikv/pd/pkg/utils/operatorutil" ) diff --git a/pkg/schedule/schedulers/evict_slow_store.go b/pkg/schedule/schedulers/evict_slow_store.go index 8f9c8841e04..d23fc2f8ff8 100644 --- a/pkg/schedule/schedulers/evict_slow_store.go +++ b/pkg/schedule/schedulers/evict_slow_store.go @@ -26,7 +26,7 @@ import ( sche "github.com/tikv/pd/pkg/schedule/core" "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/plan" - types "github.com/tikv/pd/pkg/schedule/type" + "github.com/tikv/pd/pkg/schedule/types" "github.com/tikv/pd/pkg/utils/apiutil" "github.com/tikv/pd/pkg/utils/syncutil" "github.com/unrolled/render" @@ -34,9 +34,6 @@ import ( ) const ( - // EvictSlowStoreName is evict leader scheduler name. - EvictSlowStoreName = "evict-slow-store-scheduler" - slowStoreEvictThreshold = 100 slowStoreRecoverThreshold = 1 ) diff --git a/pkg/schedule/schedulers/evict_slow_store_test.go b/pkg/schedule/schedulers/evict_slow_store_test.go index 406a08b9c99..79651fb5b5c 100644 --- a/pkg/schedule/schedulers/evict_slow_store_test.go +++ b/pkg/schedule/schedulers/evict_slow_store_test.go @@ -23,7 +23,7 @@ import ( "github.com/tikv/pd/pkg/core" "github.com/tikv/pd/pkg/mock/mockcluster" "github.com/tikv/pd/pkg/schedule/operator" - types "github.com/tikv/pd/pkg/schedule/type" + "github.com/tikv/pd/pkg/schedule/types" "github.com/tikv/pd/pkg/storage" "github.com/tikv/pd/pkg/utils/operatorutil" ) diff --git a/pkg/schedule/schedulers/evict_slow_trend.go b/pkg/schedule/schedulers/evict_slow_trend.go index b63b4b4a2e0..8fd76bdccd4 100644 --- a/pkg/schedule/schedulers/evict_slow_trend.go +++ b/pkg/schedule/schedulers/evict_slow_trend.go @@ -27,18 +27,13 @@ import ( sche "github.com/tikv/pd/pkg/schedule/core" "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/plan" - types "github.com/tikv/pd/pkg/schedule/type" + "github.com/tikv/pd/pkg/schedule/types" "github.com/tikv/pd/pkg/utils/apiutil" "github.com/tikv/pd/pkg/utils/syncutil" "github.com/unrolled/render" "go.uber.org/zap" ) -const ( - // EvictSlowTrendName is evict leader by slow trend scheduler name. - EvictSlowTrendName = "evict-slow-trend-scheduler" -) - const ( alterEpsilon = 1e-9 minReCheckDurationGap = 120 // default gap for re-check the slow node, unit: s diff --git a/pkg/schedule/schedulers/evict_slow_trend_test.go b/pkg/schedule/schedulers/evict_slow_trend_test.go index 02cb65021eb..4be6eeb58d9 100644 --- a/pkg/schedule/schedulers/evict_slow_trend_test.go +++ b/pkg/schedule/schedulers/evict_slow_trend_test.go @@ -25,7 +25,7 @@ import ( "github.com/tikv/pd/pkg/core" "github.com/tikv/pd/pkg/mock/mockcluster" "github.com/tikv/pd/pkg/schedule/operator" - types "github.com/tikv/pd/pkg/schedule/type" + "github.com/tikv/pd/pkg/schedule/types" "github.com/tikv/pd/pkg/storage" "github.com/tikv/pd/pkg/utils/operatorutil" ) diff --git a/pkg/schedule/schedulers/grant_hot_region.go b/pkg/schedule/schedulers/grant_hot_region.go index a441f41062a..18402c14437 100644 --- a/pkg/schedule/schedulers/grant_hot_region.go +++ b/pkg/schedule/schedulers/grant_hot_region.go @@ -30,7 +30,7 @@ import ( "github.com/tikv/pd/pkg/schedule/filter" "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/plan" - types "github.com/tikv/pd/pkg/schedule/type" + "github.com/tikv/pd/pkg/schedule/types" "github.com/tikv/pd/pkg/slice" "github.com/tikv/pd/pkg/statistics" "github.com/tikv/pd/pkg/statistics/utils" @@ -40,11 +40,6 @@ import ( "go.uber.org/zap" ) -const ( - // GrantHotRegionName is grant hot region scheduler name. - GrantHotRegionName = "grant-hot-region-scheduler" -) - type grantHotRegionSchedulerConfig struct { syncutil.RWMutex schedulerConfig diff --git a/pkg/schedule/schedulers/grant_leader.go b/pkg/schedule/schedulers/grant_leader.go index d70c0b19d67..5dbb6eef5f6 100644 --- a/pkg/schedule/schedulers/grant_leader.go +++ b/pkg/schedule/schedulers/grant_leader.go @@ -28,18 +28,13 @@ import ( "github.com/tikv/pd/pkg/schedule/filter" "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/plan" - types "github.com/tikv/pd/pkg/schedule/type" + "github.com/tikv/pd/pkg/schedule/types" "github.com/tikv/pd/pkg/utils/apiutil" "github.com/tikv/pd/pkg/utils/syncutil" "github.com/unrolled/render" "go.uber.org/zap" ) -const ( - // GrantLeaderName is grant leader scheduler name. - GrantLeaderName = "grant-leader-scheduler" -) - type grantLeaderSchedulerConfig struct { syncutil.RWMutex schedulerConfig @@ -312,7 +307,7 @@ func (handler *grantLeaderHandler) deleteConfig(w http.ResponseWriter, r *http.R return } if last { - if err := handler.config.removeSchedulerCb(GrantLeaderName); err != nil { + if err := handler.config.removeSchedulerCb(types.GrantLeaderScheduler.String()); err != nil { if errors.ErrorEqual(err, errs.ErrSchedulerNotFound.FastGenByArgs()) { handler.rd.JSON(w, http.StatusNotFound, err.Error()) } else { diff --git a/pkg/schedule/schedulers/hot_region.go b/pkg/schedule/schedulers/hot_region.go index ab595ec9058..6506698b75c 100644 --- a/pkg/schedule/schedulers/hot_region.go +++ b/pkg/schedule/schedulers/hot_region.go @@ -34,7 +34,7 @@ import ( "github.com/tikv/pd/pkg/schedule/filter" "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/plan" - types "github.com/tikv/pd/pkg/schedule/type" + "github.com/tikv/pd/pkg/schedule/types" "github.com/tikv/pd/pkg/slice" "github.com/tikv/pd/pkg/statistics" "github.com/tikv/pd/pkg/statistics/buckets" @@ -45,8 +45,6 @@ import ( ) const ( - // HotRegionName is balance hot region scheduler name. - HotRegionName = "balance-hot-region-scheduler" splitHotReadBuckets = "split-hot-read-region" splitHotWriteBuckets = "split-hot-write-region" splitProgressiveRank = 5 @@ -191,7 +189,6 @@ func (h *baseHotScheduler) randomType() resourceType { } type hotScheduler struct { - name string *baseHotScheduler syncutil.RWMutex // config of hot scheduler @@ -203,7 +200,6 @@ func newHotScheduler(opController *operator.Controller, conf *hotRegionScheduler base := newBaseHotScheduler(opController, conf.getHistorySampleDuration(), conf.getHistorySampleInterval()) ret := &hotScheduler{ - name: HotRegionName, baseHotScheduler: base, conf: conf, } diff --git a/pkg/schedule/schedulers/hot_region_test.go b/pkg/schedule/schedulers/hot_region_test.go index 7ee0883fb7d..a4b3225312d 100644 --- a/pkg/schedule/schedulers/hot_region_test.go +++ b/pkg/schedule/schedulers/hot_region_test.go @@ -29,7 +29,7 @@ import ( "github.com/tikv/pd/pkg/mock/mockcluster" "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/placement" - types "github.com/tikv/pd/pkg/schedule/type" + "github.com/tikv/pd/pkg/schedule/types" "github.com/tikv/pd/pkg/statistics" "github.com/tikv/pd/pkg/statistics/buckets" "github.com/tikv/pd/pkg/statistics/utils" @@ -61,14 +61,12 @@ func init() { func newHotReadScheduler(opController *operator.Controller, conf *hotRegionSchedulerConfig) *hotScheduler { ret := newHotScheduler(opController, conf) - ret.name = "" ret.types = []resourceType{readLeader, readPeer} return ret } func newHotWriteScheduler(opController *operator.Controller, conf *hotRegionSchedulerConfig) *hotScheduler { ret := newHotScheduler(opController, conf) - ret.name = "" ret.types = []resourceType{writeLeader, writePeer} return ret } @@ -2465,7 +2463,7 @@ func TestCompatibilityConfig(t *testing.T) { "dst-tolerance-ratio": 1.05, }) re.NoError(err) - err = storage.SaveSchedulerConfig(HotRegionName, data) + err = storage.SaveSchedulerConfig(types.BalanceHotRegionScheduler.String(), data) re.NoError(err) hb, err = CreateScheduler(types.BalanceHotRegionScheduler, oc, storage, ConfigJSONDecoder(data)) re.NoError(err) @@ -2481,7 +2479,7 @@ func TestCompatibilityConfig(t *testing.T) { cfg.WriteLeaderPriorities = []string{"query", "key"} data, err = EncodeConfig(cfg) re.NoError(err) - err = storage.SaveSchedulerConfig(HotRegionName, data) + err = storage.SaveSchedulerConfig(types.BalanceHotRegionScheduler.String(), data) re.NoError(err) hb, err = CreateScheduler(types.BalanceHotRegionScheduler, oc, storage, ConfigJSONDecoder(data)) re.NoError(err) diff --git a/pkg/schedule/schedulers/init.go b/pkg/schedule/schedulers/init.go index 16f78284cf8..e3101d6788b 100644 --- a/pkg/schedule/schedulers/init.go +++ b/pkg/schedule/schedulers/init.go @@ -22,7 +22,7 @@ import ( "github.com/tikv/pd/pkg/core" "github.com/tikv/pd/pkg/errs" "github.com/tikv/pd/pkg/schedule/operator" - types "github.com/tikv/pd/pkg/schedule/type" + "github.com/tikv/pd/pkg/schedule/types" "github.com/tikv/pd/pkg/storage/endpoint" ) diff --git a/pkg/schedule/schedulers/label.go b/pkg/schedule/schedulers/label.go index 8d4f42262ac..d1a06a5c4ff 100644 --- a/pkg/schedule/schedulers/label.go +++ b/pkg/schedule/schedulers/label.go @@ -24,15 +24,10 @@ import ( "github.com/tikv/pd/pkg/schedule/filter" "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/plan" - types "github.com/tikv/pd/pkg/schedule/type" + "github.com/tikv/pd/pkg/schedule/types" "go.uber.org/zap" ) -const ( - // LabelName is label scheduler name. - LabelName = "label-scheduler" -) - type labelSchedulerConfig struct { Ranges []core.KeyRange `json:"ranges"` // TODO: When we prepare to use Ranges, we will need to implement the ReloadConfig function for this scheduler. @@ -95,7 +90,7 @@ func (s *labelScheduler) Schedule(cluster sche.SchedulerCluster, _ bool) ([]*ope f := filter.NewExcludedFilter(s.GetName(), nil, excludeStores) target := filter.NewCandidates(cluster.GetFollowerStores(region)). - FilterTarget(cluster.GetSchedulerConfig(), nil, nil, &filter.StoreStateFilter{ActionScope: LabelName, TransferLeader: true, OperatorLevel: constant.Medium}, f). + FilterTarget(cluster.GetSchedulerConfig(), nil, nil, &filter.StoreStateFilter{ActionScope: s.GetName(), TransferLeader: true, OperatorLevel: constant.Medium}, f). RandomPick() if target == nil { log.Debug("label scheduler no target found for region", zap.Uint64("region-id", region.GetID())) diff --git a/pkg/schedule/schedulers/metrics.go b/pkg/schedule/schedulers/metrics.go index 90b0b0c6bb2..a518a167af7 100644 --- a/pkg/schedule/schedulers/metrics.go +++ b/pkg/schedule/schedulers/metrics.go @@ -16,7 +16,7 @@ package schedulers import ( "github.com/prometheus/client_golang/prometheus" - types "github.com/tikv/pd/pkg/schedule/type" + "github.com/tikv/pd/pkg/schedule/types" ) var ( diff --git a/pkg/schedule/schedulers/random_merge.go b/pkg/schedule/schedulers/random_merge.go index 676e5407e72..f6660472f57 100644 --- a/pkg/schedule/schedulers/random_merge.go +++ b/pkg/schedule/schedulers/random_merge.go @@ -26,12 +26,7 @@ import ( "github.com/tikv/pd/pkg/schedule/filter" "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/plan" - types "github.com/tikv/pd/pkg/schedule/type" -) - -const ( - // RandomMergeName is random merge scheduler name. - RandomMergeName = "random-merge-scheduler" + "github.com/tikv/pd/pkg/schedule/types" ) type randomMergeSchedulerConfig struct { diff --git a/pkg/schedule/schedulers/scatter_range.go b/pkg/schedule/schedulers/scatter_range.go index e86785fcc19..37f00d2df6e 100644 --- a/pkg/schedule/schedulers/scatter_range.go +++ b/pkg/schedule/schedulers/scatter_range.go @@ -25,17 +25,12 @@ import ( sche "github.com/tikv/pd/pkg/schedule/core" "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/plan" - types "github.com/tikv/pd/pkg/schedule/type" + "github.com/tikv/pd/pkg/schedule/types" "github.com/tikv/pd/pkg/utils/apiutil" "github.com/tikv/pd/pkg/utils/syncutil" "github.com/unrolled/render" ) -const ( - // ScatterRangeName is scatter range scheduler name - ScatterRangeName = "scatter-range" -) - type scatterRangeSchedulerConfig struct { syncutil.RWMutex schedulerConfig diff --git a/pkg/schedule/schedulers/scheduler.go b/pkg/schedule/schedulers/scheduler.go index 7fce5d9c46e..27be70680af 100644 --- a/pkg/schedule/schedulers/scheduler.go +++ b/pkg/schedule/schedulers/scheduler.go @@ -27,7 +27,7 @@ import ( sche "github.com/tikv/pd/pkg/schedule/core" "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/plan" - types "github.com/tikv/pd/pkg/schedule/type" + "github.com/tikv/pd/pkg/schedule/types" "github.com/tikv/pd/pkg/storage/endpoint" "go.uber.org/zap" ) diff --git a/pkg/schedule/schedulers/scheduler_controller.go b/pkg/schedule/schedulers/scheduler_controller.go index c29b75a371a..5e1082acee3 100644 --- a/pkg/schedule/schedulers/scheduler_controller.go +++ b/pkg/schedule/schedulers/scheduler_controller.go @@ -29,6 +29,7 @@ import ( "github.com/tikv/pd/pkg/schedule/labeler" "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/plan" + "github.com/tikv/pd/pkg/schedule/types" "github.com/tikv/pd/pkg/storage/endpoint" "github.com/tikv/pd/pkg/utils/logutil" "github.com/tikv/pd/pkg/utils/syncutil" @@ -400,7 +401,7 @@ func (c *Controller) GetPausedSchedulerDelayUntil(name string) (int64, error) { func (c *Controller) CheckTransferWitnessLeader(region *core.RegionInfo) { if core.NeedTransferWitnessLeader(region) { c.RLock() - s, ok := c.schedulers[TransferWitnessLeaderName] + s, ok := c.schedulers[types.TransferWitnessLeaderScheduler.String()] c.RUnlock() if ok { select { @@ -440,7 +441,7 @@ func NewScheduleController(ctx context.Context, cluster sche.SchedulerCluster, o nextInterval: s.GetMinInterval(), ctx: ctx, cancel: cancel, - diagnosticRecorder: NewDiagnosticRecorder(s.GetName(), cluster.GetSchedulerConfig()), + diagnosticRecorder: NewDiagnosticRecorder(s.GetType(), cluster.GetSchedulerConfig()), } } diff --git a/pkg/schedule/schedulers/scheduler_test.go b/pkg/schedule/schedulers/scheduler_test.go index 8dfe9f3616f..ba734230ea5 100644 --- a/pkg/schedule/schedulers/scheduler_test.go +++ b/pkg/schedule/schedulers/scheduler_test.go @@ -28,7 +28,7 @@ import ( "github.com/tikv/pd/pkg/schedule/hbstream" "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/placement" - types "github.com/tikv/pd/pkg/schedule/type" + "github.com/tikv/pd/pkg/schedule/types" "github.com/tikv/pd/pkg/statistics/utils" "github.com/tikv/pd/pkg/storage" "github.com/tikv/pd/pkg/utils/operatorutil" diff --git a/pkg/schedule/schedulers/shuffle_hot_region.go b/pkg/schedule/schedulers/shuffle_hot_region.go index f8544fff48d..b8818dc48da 100644 --- a/pkg/schedule/schedulers/shuffle_hot_region.go +++ b/pkg/schedule/schedulers/shuffle_hot_region.go @@ -26,7 +26,7 @@ import ( "github.com/tikv/pd/pkg/schedule/filter" "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/plan" - types "github.com/tikv/pd/pkg/schedule/type" + "github.com/tikv/pd/pkg/schedule/types" "github.com/tikv/pd/pkg/statistics" "github.com/tikv/pd/pkg/utils/apiutil" "github.com/tikv/pd/pkg/utils/syncutil" @@ -34,11 +34,6 @@ import ( "go.uber.org/zap" ) -const ( - // ShuffleHotRegionName is shuffle hot region scheduler name. - ShuffleHotRegionName = "shuffle-hot-region-scheduler" -) - type shuffleHotRegionSchedulerConfig struct { syncutil.RWMutex schedulerConfig diff --git a/pkg/schedule/schedulers/shuffle_leader.go b/pkg/schedule/schedulers/shuffle_leader.go index 4270613667b..842a26d9b12 100644 --- a/pkg/schedule/schedulers/shuffle_leader.go +++ b/pkg/schedule/schedulers/shuffle_leader.go @@ -23,12 +23,7 @@ import ( "github.com/tikv/pd/pkg/schedule/filter" "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/plan" - types "github.com/tikv/pd/pkg/schedule/type" -) - -const ( - // ShuffleLeaderName is shuffle leader scheduler name. - ShuffleLeaderName = "shuffle-leader-scheduler" + "github.com/tikv/pd/pkg/schedule/types" ) type shuffleLeaderSchedulerConfig struct { diff --git a/pkg/schedule/schedulers/shuffle_region.go b/pkg/schedule/schedulers/shuffle_region.go index 5d4c49e0fcc..a4c247d3363 100644 --- a/pkg/schedule/schedulers/shuffle_region.go +++ b/pkg/schedule/schedulers/shuffle_region.go @@ -24,12 +24,7 @@ import ( "github.com/tikv/pd/pkg/schedule/filter" "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/plan" - types "github.com/tikv/pd/pkg/schedule/type" -) - -const ( - // ShuffleRegionName is shuffle region scheduler name. - ShuffleRegionName = "shuffle-region-scheduler" + "github.com/tikv/pd/pkg/schedule/types" ) type shuffleRegionScheduler struct { @@ -41,11 +36,11 @@ type shuffleRegionScheduler struct { // newShuffleRegionScheduler creates an admin scheduler that shuffles regions // between stores. func newShuffleRegionScheduler(opController *operator.Controller, conf *shuffleRegionSchedulerConfig) Scheduler { + base := NewBaseScheduler(opController, types.ShuffleRegionScheduler) filters := []filter.Filter{ - &filter.StoreStateFilter{ActionScope: ShuffleRegionName, MoveRegion: true, OperatorLevel: constant.Low}, - filter.NewSpecialUseFilter(ShuffleRegionName), + &filter.StoreStateFilter{ActionScope: base.GetName(), MoveRegion: true, OperatorLevel: constant.Low}, + filter.NewSpecialUseFilter(base.GetName()), } - base := NewBaseScheduler(opController, types.ShuffleRegionScheduler) return &shuffleRegionScheduler{ BaseScheduler: base, conf: conf, diff --git a/pkg/schedule/schedulers/split_bucket.go b/pkg/schedule/schedulers/split_bucket.go index 0d8fa614aef..a0881ae1a34 100644 --- a/pkg/schedule/schedulers/split_bucket.go +++ b/pkg/schedule/schedulers/split_bucket.go @@ -28,7 +28,7 @@ import ( sche "github.com/tikv/pd/pkg/schedule/core" "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/plan" - types "github.com/tikv/pd/pkg/schedule/type" + "github.com/tikv/pd/pkg/schedule/types" "github.com/tikv/pd/pkg/statistics/buckets" "github.com/tikv/pd/pkg/utils/reflectutil" "github.com/tikv/pd/pkg/utils/syncutil" @@ -36,8 +36,6 @@ import ( ) const ( - // SplitBucketName is the split bucket name. - SplitBucketName = "split-bucket-scheduler" // defaultHotDegree is the default hot region threshold. defaultHotDegree = 3 defaultSplitLimit = 10 diff --git a/pkg/schedule/schedulers/transfer_witness_leader.go b/pkg/schedule/schedulers/transfer_witness_leader.go index 9e7bd9bccc1..2ef0fc6a4f2 100644 --- a/pkg/schedule/schedulers/transfer_witness_leader.go +++ b/pkg/schedule/schedulers/transfer_witness_leader.go @@ -24,12 +24,10 @@ import ( "github.com/tikv/pd/pkg/schedule/filter" "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/plan" - types "github.com/tikv/pd/pkg/schedule/type" + "github.com/tikv/pd/pkg/schedule/types" ) const ( - // TransferWitnessLeaderName is transfer witness leader scheduler name. - TransferWitnessLeaderName = "transfer-witness-leader-scheduler" // TransferWitnessLeaderBatchSize is the number of operators to to transfer // leaders by one scheduling transferWitnessLeaderBatchSize = 3 diff --git a/pkg/schedule/schedulers/transfer_witness_leader_test.go b/pkg/schedule/schedulers/transfer_witness_leader_test.go index 046b7aeb53f..b100e0a9535 100644 --- a/pkg/schedule/schedulers/transfer_witness_leader_test.go +++ b/pkg/schedule/schedulers/transfer_witness_leader_test.go @@ -22,7 +22,7 @@ import ( "github.com/stretchr/testify/require" "github.com/tikv/pd/pkg/core" "github.com/tikv/pd/pkg/schedule/operator" - types "github.com/tikv/pd/pkg/schedule/type" + "github.com/tikv/pd/pkg/schedule/types" "github.com/tikv/pd/pkg/storage" "github.com/tikv/pd/pkg/utils/operatorutil" ) diff --git a/pkg/schedule/type/type.go b/pkg/schedule/types/type.go similarity index 100% rename from pkg/schedule/type/type.go rename to pkg/schedule/types/type.go diff --git a/plugin/scheduler_example/evict_leader.go b/plugin/scheduler_example/evict_leader.go index c7842debdcb..c20cfd41814 100644 --- a/plugin/scheduler_example/evict_leader.go +++ b/plugin/scheduler_example/evict_leader.go @@ -30,7 +30,7 @@ import ( "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/plan" "github.com/tikv/pd/pkg/schedule/schedulers" - types "github.com/tikv/pd/pkg/schedule/type" + "github.com/tikv/pd/pkg/schedule/types" "github.com/tikv/pd/pkg/storage/endpoint" "github.com/tikv/pd/pkg/utils/apiutil" "github.com/tikv/pd/pkg/utils/syncutil" diff --git a/server/api/diagnostic_test.go b/server/api/diagnostic_test.go index 8c4089a8710..c85d9a45369 100644 --- a/server/api/diagnostic_test.go +++ b/server/api/diagnostic_test.go @@ -25,6 +25,7 @@ import ( "github.com/stretchr/testify/suite" "github.com/tikv/pd/pkg/core" "github.com/tikv/pd/pkg/schedule/schedulers" + "github.com/tikv/pd/pkg/schedule/types" tu "github.com/tikv/pd/pkg/utils/testutil" "github.com/tikv/pd/server" "github.com/tikv/pd/server/config" @@ -95,17 +96,17 @@ func (suite *diagnosticTestSuite) TestSchedulerDiagnosticAPI() { re.NoError(tu.ReadGetJSON(re, testDialClient, addr, cfg)) re.True(cfg.Schedule.EnableDiagnostic) - balanceRegionURL := suite.urlPrefix + "/" + schedulers.BalanceRegionName + balanceRegionURL := suite.urlPrefix + "/" + types.BalanceRegionScheduler.String() result := &schedulers.DiagnosticResult{} err = tu.ReadGetJSON(re, testDialClient, balanceRegionURL, result) re.NoError(err) re.Equal("disabled", result.Status) - evictLeaderURL := suite.urlPrefix + "/" + schedulers.EvictLeaderName + evictLeaderURL := suite.urlPrefix + "/" + types.EvictLeaderScheduler.String() re.NoError(tu.CheckGetJSON(testDialClient, evictLeaderURL, nil, tu.StatusNotOK(re))) input := make(map[string]any) - input["name"] = schedulers.BalanceRegionName + input["name"] = types.BalanceRegionScheduler.String() body, err := json.Marshal(input) re.NoError(err) err = tu.CheckPostJSON(testDialClient, suite.schedulerPrefix, body, tu.StatusOK(re)) @@ -116,14 +117,14 @@ func (suite *diagnosticTestSuite) TestSchedulerDiagnosticAPI() { input["delay"] = 30 pauseArgs, err := json.Marshal(input) re.NoError(err) - err = tu.CheckPostJSON(testDialClient, suite.schedulerPrefix+"/"+schedulers.BalanceRegionName, pauseArgs, tu.StatusOK(re)) + err = tu.CheckPostJSON(testDialClient, suite.schedulerPrefix+"/"+types.BalanceRegionScheduler.String(), pauseArgs, tu.StatusOK(re)) re.NoError(err) suite.checkStatus("paused", balanceRegionURL) input["delay"] = 0 pauseArgs, err = json.Marshal(input) re.NoError(err) - err = tu.CheckPostJSON(testDialClient, suite.schedulerPrefix+"/"+schedulers.BalanceRegionName, pauseArgs, tu.StatusOK(re)) + err = tu.CheckPostJSON(testDialClient, suite.schedulerPrefix+"/"+types.BalanceRegionScheduler.String(), pauseArgs, tu.StatusOK(re)) re.NoError(err) suite.checkStatus("pending", balanceRegionURL) @@ -132,7 +133,7 @@ func (suite *diagnosticTestSuite) TestSchedulerDiagnosticAPI() { fmt.Println("after put region") suite.checkStatus("normal", balanceRegionURL) - deleteURL := fmt.Sprintf("%s/%s", suite.schedulerPrefix, schedulers.BalanceRegionName) + deleteURL := fmt.Sprintf("%s/%s", suite.schedulerPrefix, types.BalanceRegionScheduler.String()) err = tu.CheckDelete(testDialClient, deleteURL, tu.StatusOK(re)) re.NoError(err) suite.checkStatus("disabled", balanceRegionURL) diff --git a/server/api/scheduler.go b/server/api/scheduler.go index 1d502013558..c96e4c123de 100644 --- a/server/api/scheduler.go +++ b/server/api/scheduler.go @@ -24,8 +24,7 @@ import ( "github.com/pingcap/errors" "github.com/pingcap/log" "github.com/tikv/pd/pkg/errs" - "github.com/tikv/pd/pkg/schedule/schedulers" - types "github.com/tikv/pd/pkg/schedule/type" + "github.com/tikv/pd/pkg/schedule/types" "github.com/tikv/pd/pkg/utils/apiutil" "github.com/tikv/pd/server" "github.com/unrolled/render" @@ -177,11 +176,11 @@ func (h *schedulerHandler) CreateScheduler(w http.ResponseWriter, r *http.Reques func (h *schedulerHandler) DeleteScheduler(w http.ResponseWriter, r *http.Request) { name := mux.Vars(r)["name"] switch { - case strings.HasPrefix(name, schedulers.EvictLeaderName) && name != schedulers.EvictLeaderName: - h.redirectSchedulerDelete(w, name, schedulers.EvictLeaderName) + case strings.HasPrefix(name, types.EvictLeaderScheduler.String()) && name != types.EvictLeaderScheduler.String(): + h.redirectSchedulerDelete(w, name, types.EvictLeaderScheduler.String()) return - case strings.HasPrefix(name, schedulers.GrantLeaderName) && name != schedulers.GrantLeaderName: - h.redirectSchedulerDelete(w, name, schedulers.GrantLeaderName) + case strings.HasPrefix(name, types.GrantLeaderScheduler.String()) && name != types.GrantLeaderScheduler.String(): + h.redirectSchedulerDelete(w, name, types.GrantLeaderScheduler.String()) return default: if err := h.RemoveScheduler(name); err != nil { diff --git a/server/cluster/cluster_test.go b/server/cluster/cluster_test.go index cffefbba444..9a9420988a1 100644 --- a/server/cluster/cluster_test.go +++ b/server/cluster/cluster_test.go @@ -53,7 +53,7 @@ import ( "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/placement" "github.com/tikv/pd/pkg/schedule/schedulers" - types "github.com/tikv/pd/pkg/schedule/type" + "github.com/tikv/pd/pkg/schedule/types" "github.com/tikv/pd/pkg/slice" "github.com/tikv/pd/pkg/statistics" "github.com/tikv/pd/pkg/statistics/utils" @@ -366,7 +366,7 @@ func TestSetOfflineStoreWithEvictLeader(t *testing.T) { err = cluster.RemoveStore(3, false) re.Error(err) re.Contains(err.Error(), string(errs.ErrNoStoreForRegionLeader.RFCCode())) - re.NoError(cluster.RemoveScheduler(schedulers.EvictLeaderName)) + re.NoError(cluster.RemoveScheduler(types.EvictLeaderScheduler.String())) re.NoError(cluster.RemoveStore(3, false)) } @@ -2453,10 +2453,10 @@ func TestDispatch(t *testing.T) { waitOperator(re, co, 1) controller := co.GetSchedulersController() operatorutil.CheckTransferPeer(re, co.GetOperatorController().GetOperator(1), operator.OpKind(0), 4, 1) - re.NoError(controller.RemoveScheduler(schedulers.BalanceRegionName)) + re.NoError(controller.RemoveScheduler(types.BalanceRegionScheduler.String())) waitOperator(re, co, 2) operatorutil.CheckTransferLeader(re, co.GetOperatorController().GetOperator(2), operator.OpKind(0), 4, 2) - re.NoError(controller.RemoveScheduler(schedulers.BalanceLeaderName)) + re.NoError(controller.RemoveScheduler(types.BalanceLeaderScheduler.String())) stream := mockhbstream.NewHeartbeatStream() @@ -3075,10 +3075,10 @@ func TestAddScheduler(t *testing.T) { defer cleanup() controller := co.GetSchedulersController() re.Len(controller.GetSchedulerNames(), len(sc.DefaultSchedulers)) - re.NoError(controller.RemoveScheduler(schedulers.BalanceLeaderName)) - re.NoError(controller.RemoveScheduler(schedulers.BalanceRegionName)) - re.NoError(controller.RemoveScheduler(schedulers.HotRegionName)) - re.NoError(controller.RemoveScheduler(schedulers.EvictSlowStoreName)) + re.NoError(controller.RemoveScheduler(types.BalanceLeaderScheduler.String())) + re.NoError(controller.RemoveScheduler(types.BalanceRegionScheduler.String())) + re.NoError(controller.RemoveScheduler(types.BalanceHotRegionScheduler.String())) + re.NoError(controller.RemoveScheduler(types.EvictSlowStoreScheduler.String())) re.Empty(controller.GetSchedulerNames()) stream := mockhbstream.NewHeartbeatStream() @@ -3170,10 +3170,10 @@ func TestPersistScheduler(t *testing.T) { re.Len(sches, defaultCount+2) // remove all default schedulers - re.NoError(controller.RemoveScheduler(schedulers.BalanceLeaderName)) - re.NoError(controller.RemoveScheduler(schedulers.BalanceRegionName)) - re.NoError(controller.RemoveScheduler(schedulers.HotRegionName)) - re.NoError(controller.RemoveScheduler(schedulers.EvictSlowStoreName)) + re.NoError(controller.RemoveScheduler(types.BalanceLeaderScheduler.String())) + re.NoError(controller.RemoveScheduler(types.BalanceRegionScheduler.String())) + re.NoError(controller.RemoveScheduler(types.BalanceHotRegionScheduler.String())) + re.NoError(controller.RemoveScheduler(types.EvictSlowStoreScheduler.String())) // only remains 2 items with independent config. re.Len(controller.GetSchedulerNames(), 2) re.NoError(co.GetCluster().GetSchedulerConfig().Persist(storage)) @@ -3233,7 +3233,7 @@ func TestPersistScheduler(t *testing.T) { // the scheduler option should contain 9 items // the `hot scheduler` are disabled re.Len(co.GetCluster().GetSchedulerConfig().(*config.PersistOptions).GetSchedulers(), defaultCount+3) - re.NoError(controller.RemoveScheduler(schedulers.GrantLeaderName)) + re.NoError(controller.RemoveScheduler(types.GrantLeaderScheduler.String())) // the scheduler that is not enable by default will be completely deleted re.Len(co.GetCluster().GetSchedulerConfig().(*config.PersistOptions).GetSchedulers(), defaultCount+2) re.Len(controller.GetSchedulerNames(), 4) @@ -3250,7 +3250,7 @@ func TestPersistScheduler(t *testing.T) { co.Run() controller = co.GetSchedulersController() re.Len(controller.GetSchedulerNames(), 4) - re.NoError(controller.RemoveScheduler(schedulers.EvictLeaderName)) + re.NoError(controller.RemoveScheduler(types.EvictLeaderScheduler.String())) re.Len(controller.GetSchedulerNames(), 3) } @@ -3283,11 +3283,11 @@ func TestRemoveScheduler(t *testing.T) { re.Len(sches, defaultCount+1) // remove all schedulers - re.NoError(controller.RemoveScheduler(schedulers.BalanceLeaderName)) - re.NoError(controller.RemoveScheduler(schedulers.BalanceRegionName)) - re.NoError(controller.RemoveScheduler(schedulers.HotRegionName)) - re.NoError(controller.RemoveScheduler(schedulers.GrantLeaderName)) - re.NoError(controller.RemoveScheduler(schedulers.EvictSlowStoreName)) + re.NoError(controller.RemoveScheduler(types.BalanceLeaderScheduler.String())) + re.NoError(controller.RemoveScheduler(types.BalanceRegionScheduler.String())) + re.NoError(controller.RemoveScheduler(types.BalanceHotRegionScheduler.String())) + re.NoError(controller.RemoveScheduler(types.GrantLeaderScheduler.String())) + re.NoError(controller.RemoveScheduler(types.EvictSlowStoreScheduler.String())) // all removed sches, _, err = storage.LoadAllSchedulerConfigs() re.NoError(err) @@ -3360,15 +3360,15 @@ func TestPauseScheduler(t *testing.T) { controller := co.GetSchedulersController() _, err := controller.IsSchedulerAllowed("test") re.Error(err) - controller.PauseOrResumeScheduler(schedulers.BalanceLeaderName, 60) - paused, _ := controller.IsSchedulerPaused(schedulers.BalanceLeaderName) + controller.PauseOrResumeScheduler(types.BalanceLeaderScheduler.String(), 60) + paused, _ := controller.IsSchedulerPaused(types.BalanceLeaderScheduler.String()) re.True(paused) - pausedAt, err := controller.GetPausedSchedulerDelayAt(schedulers.BalanceLeaderName) + pausedAt, err := controller.GetPausedSchedulerDelayAt(types.BalanceLeaderScheduler.String()) re.NoError(err) - resumeAt, err := controller.GetPausedSchedulerDelayUntil(schedulers.BalanceLeaderName) + resumeAt, err := controller.GetPausedSchedulerDelayUntil(types.BalanceLeaderScheduler.String()) re.NoError(err) re.Equal(int64(60), resumeAt-pausedAt) - allowed, _ := controller.IsSchedulerAllowed(schedulers.BalanceLeaderName) + allowed, _ := controller.IsSchedulerAllowed(types.BalanceLeaderScheduler.String()) re.False(allowed) } diff --git a/server/cluster/scheduling_controller.go b/server/cluster/scheduling_controller.go index b4c29ceed46..20a36a6817d 100644 --- a/server/cluster/scheduling_controller.go +++ b/server/cluster/scheduling_controller.go @@ -33,6 +33,7 @@ import ( "github.com/tikv/pd/pkg/schedule/scatter" "github.com/tikv/pd/pkg/schedule/schedulers" "github.com/tikv/pd/pkg/schedule/splitter" + "github.com/tikv/pd/pkg/schedule/types" "github.com/tikv/pd/pkg/statistics" "github.com/tikv/pd/pkg/statistics/buckets" "github.com/tikv/pd/pkg/statistics/utils" @@ -455,7 +456,7 @@ func (sc *schedulingController) getEvictLeaderStores() (evictStores []uint64) { if sc.coordinator == nil { return nil } - handler, ok := sc.coordinator.GetSchedulersController().GetSchedulerHandlers()[schedulers.EvictLeaderName] + handler, ok := sc.coordinator.GetSchedulersController().GetSchedulerHandlers()[types.EvictLeaderScheduler.String()] if !ok { return } diff --git a/server/config/persist_options.go b/server/config/persist_options.go index 19f0ef0d475..807e9699a25 100644 --- a/server/config/persist_options.go +++ b/server/config/persist_options.go @@ -33,7 +33,7 @@ import ( "github.com/tikv/pd/pkg/core/constant" "github.com/tikv/pd/pkg/core/storelimit" sc "github.com/tikv/pd/pkg/schedule/config" - types "github.com/tikv/pd/pkg/schedule/type" + "github.com/tikv/pd/pkg/schedule/types" "github.com/tikv/pd/pkg/slice" "github.com/tikv/pd/pkg/storage/endpoint" "github.com/tikv/pd/pkg/utils/etcdutil" diff --git a/server/handler.go b/server/handler.go index e3e4184f177..1464c768aed 100644 --- a/server/handler.go +++ b/server/handler.go @@ -34,7 +34,7 @@ import ( sche "github.com/tikv/pd/pkg/schedule/core" "github.com/tikv/pd/pkg/schedule/handler" "github.com/tikv/pd/pkg/schedule/schedulers" - types "github.com/tikv/pd/pkg/schedule/type" + "github.com/tikv/pd/pkg/schedule/types" "github.com/tikv/pd/pkg/statistics" "github.com/tikv/pd/pkg/statistics/utils" "github.com/tikv/pd/pkg/storage" diff --git a/tests/integrations/mcs/scheduling/config_test.go b/tests/integrations/mcs/scheduling/config_test.go index 168d3a8742c..fcc9a78b0f3 100644 --- a/tests/integrations/mcs/scheduling/config_test.go +++ b/tests/integrations/mcs/scheduling/config_test.go @@ -28,6 +28,7 @@ import ( "github.com/tikv/pd/pkg/mcs/scheduling/server/config" sc "github.com/tikv/pd/pkg/schedule/config" "github.com/tikv/pd/pkg/schedule/schedulers" + "github.com/tikv/pd/pkg/schedule/types" "github.com/tikv/pd/pkg/slice" "github.com/tikv/pd/pkg/storage/endpoint" "github.com/tikv/pd/pkg/storage/kv" @@ -165,14 +166,14 @@ func (suite *configTestSuite) TestSchedulerConfigWatch() { }) re.Equal(namesFromAPIServer, namesFromSchedulingServer) // Add a new scheduler. - api.MustAddScheduler(re, suite.pdLeaderServer.GetAddr(), schedulers.EvictLeaderName, map[string]any{ + api.MustAddScheduler(re, suite.pdLeaderServer.GetAddr(), types.EvictLeaderScheduler.String(), map[string]any{ "store_id": 1, }) // Check the new scheduler's config. testutil.Eventually(re, func() bool { namesFromSchedulingServer, _, err = storage.LoadAllSchedulerConfigs() re.NoError(err) - return slice.Contains(namesFromSchedulingServer, schedulers.EvictLeaderName) + return slice.Contains(namesFromSchedulingServer, types.EvictLeaderScheduler.String()) }) assertEvictLeaderStoreIDs(re, storage, []uint64{1}) // Update the scheduler by adding a store. @@ -187,20 +188,20 @@ func (suite *configTestSuite) TestSchedulerConfigWatch() { }, ) re.NoError(err) - api.MustAddScheduler(re, suite.pdLeaderServer.GetAddr(), schedulers.EvictLeaderName, map[string]any{ + api.MustAddScheduler(re, suite.pdLeaderServer.GetAddr(), types.EvictLeaderScheduler.String(), map[string]any{ "store_id": 2, }) assertEvictLeaderStoreIDs(re, storage, []uint64{1, 2}) // Update the scheduler by removing a store. - api.MustDeleteScheduler(re, suite.pdLeaderServer.GetAddr(), fmt.Sprintf("%s-%d", schedulers.EvictLeaderName, 1)) + api.MustDeleteScheduler(re, suite.pdLeaderServer.GetAddr(), fmt.Sprintf("%s-%d", types.EvictLeaderScheduler.String(), 1)) assertEvictLeaderStoreIDs(re, storage, []uint64{2}) // Delete the scheduler. - api.MustDeleteScheduler(re, suite.pdLeaderServer.GetAddr(), schedulers.EvictLeaderName) + api.MustDeleteScheduler(re, suite.pdLeaderServer.GetAddr(), types.EvictLeaderScheduler.String()) // Check the removed scheduler's config. testutil.Eventually(re, func() bool { namesFromSchedulingServer, _, err = storage.LoadAllSchedulerConfigs() re.NoError(err) - return !slice.Contains(namesFromSchedulingServer, schedulers.EvictLeaderName) + return !slice.Contains(namesFromSchedulingServer, types.EvictLeaderScheduler.String()) }) watcher.Close() } @@ -212,7 +213,7 @@ func assertEvictLeaderStoreIDs( StoreIDWithRanges map[uint64][]core.KeyRange `json:"store-id-ranges"` } testutil.Eventually(re, func() bool { - cfg, err := storage.LoadSchedulerConfig(schedulers.EvictLeaderName) + cfg, err := storage.LoadSchedulerConfig(types.EvictLeaderScheduler.String()) re.NoError(err) err = schedulers.DecodeConfig([]byte(cfg), &evictLeaderCfg) re.NoError(err) diff --git a/tests/integrations/mcs/scheduling/server_test.go b/tests/integrations/mcs/scheduling/server_test.go index 5c08892a972..57306640394 100644 --- a/tests/integrations/mcs/scheduling/server_test.go +++ b/tests/integrations/mcs/scheduling/server_test.go @@ -32,6 +32,7 @@ import ( "github.com/tikv/pd/pkg/mcs/utils/constant" "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/schedulers" + "github.com/tikv/pd/pkg/schedule/types" "github.com/tikv/pd/pkg/utils/testutil" "github.com/tikv/pd/server" "github.com/tikv/pd/tests" @@ -305,7 +306,7 @@ func (suite *serverTestSuite) TestSchedulerSync() { schedulersController := tc.GetPrimaryServer().GetCluster().GetCoordinator().GetSchedulersController() checkEvictLeaderSchedulerExist(re, schedulersController, false) // Add a new evict-leader-scheduler through the API server. - api.MustAddScheduler(re, suite.backendEndpoints, schedulers.EvictLeaderName, map[string]any{ + api.MustAddScheduler(re, suite.backendEndpoints, types.EvictLeaderScheduler.String(), map[string]any{ "store_id": 1, }) // Check if the evict-leader-scheduler is added. @@ -323,54 +324,54 @@ func (suite *serverTestSuite) TestSchedulerSync() { }, ) re.NoError(err) - api.MustAddScheduler(re, suite.backendEndpoints, schedulers.EvictLeaderName, map[string]any{ + api.MustAddScheduler(re, suite.backendEndpoints, types.EvictLeaderScheduler.String(), map[string]any{ "store_id": 2, }) checkEvictLeaderSchedulerExist(re, schedulersController, true) checkEvictLeaderStoreIDs(re, schedulersController, []uint64{1, 2}) // Delete a store_id from the evict-leader-scheduler through the API server. - api.MustDeleteScheduler(re, suite.backendEndpoints, fmt.Sprintf("%s-%d", schedulers.EvictLeaderName, 1)) + api.MustDeleteScheduler(re, suite.backendEndpoints, fmt.Sprintf("%s-%d", types.EvictLeaderScheduler.String(), 1)) checkEvictLeaderSchedulerExist(re, schedulersController, true) checkEvictLeaderStoreIDs(re, schedulersController, []uint64{2}) // Add a store_id to the evict-leader-scheduler through the API server by the scheduler handler. - api.MustCallSchedulerConfigAPI(re, http.MethodPost, suite.backendEndpoints, schedulers.EvictLeaderName, []string{"config"}, map[string]any{ - "name": schedulers.EvictLeaderName, + api.MustCallSchedulerConfigAPI(re, http.MethodPost, suite.backendEndpoints, types.EvictLeaderScheduler.String(), []string{"config"}, map[string]any{ + "name": types.EvictLeaderScheduler.String(), "store_id": 1, }) checkEvictLeaderSchedulerExist(re, schedulersController, true) checkEvictLeaderStoreIDs(re, schedulersController, []uint64{1, 2}) // Delete a store_id from the evict-leader-scheduler through the API server by the scheduler handler. - api.MustCallSchedulerConfigAPI(re, http.MethodDelete, suite.backendEndpoints, schedulers.EvictLeaderName, []string{"delete", "2"}, nil) + api.MustCallSchedulerConfigAPI(re, http.MethodDelete, suite.backendEndpoints, types.EvictLeaderScheduler.String(), []string{"delete", "2"}, nil) checkEvictLeaderSchedulerExist(re, schedulersController, true) checkEvictLeaderStoreIDs(re, schedulersController, []uint64{1}) // If the last store is deleted, the scheduler should be removed. - api.MustCallSchedulerConfigAPI(re, http.MethodDelete, suite.backendEndpoints, schedulers.EvictLeaderName, []string{"delete", "1"}, nil) + api.MustCallSchedulerConfigAPI(re, http.MethodDelete, suite.backendEndpoints, types.EvictLeaderScheduler.String(), []string{"delete", "1"}, nil) // Check if the scheduler is removed. checkEvictLeaderSchedulerExist(re, schedulersController, false) // Delete the evict-leader-scheduler through the API server by removing the last store_id. - api.MustAddScheduler(re, suite.backendEndpoints, schedulers.EvictLeaderName, map[string]any{ + api.MustAddScheduler(re, suite.backendEndpoints, types.EvictLeaderScheduler.String(), map[string]any{ "store_id": 1, }) checkEvictLeaderSchedulerExist(re, schedulersController, true) checkEvictLeaderStoreIDs(re, schedulersController, []uint64{1}) - api.MustDeleteScheduler(re, suite.backendEndpoints, fmt.Sprintf("%s-%d", schedulers.EvictLeaderName, 1)) + api.MustDeleteScheduler(re, suite.backendEndpoints, fmt.Sprintf("%s-%d", types.EvictLeaderScheduler.String(), 1)) checkEvictLeaderSchedulerExist(re, schedulersController, false) // Delete the evict-leader-scheduler through the API server. - api.MustAddScheduler(re, suite.backendEndpoints, schedulers.EvictLeaderName, map[string]any{ + api.MustAddScheduler(re, suite.backendEndpoints, types.EvictLeaderScheduler.String(), map[string]any{ "store_id": 1, }) checkEvictLeaderSchedulerExist(re, schedulersController, true) checkEvictLeaderStoreIDs(re, schedulersController, []uint64{1}) - api.MustDeleteScheduler(re, suite.backendEndpoints, schedulers.EvictLeaderName) + api.MustDeleteScheduler(re, suite.backendEndpoints, types.EvictLeaderScheduler.String()) checkEvictLeaderSchedulerExist(re, schedulersController, false) // The default scheduler could not be deleted, it could only be disabled. defaultSchedulerNames := []string{ - schedulers.BalanceLeaderName, - schedulers.BalanceRegionName, - schedulers.HotRegionName, + types.BalanceLeaderScheduler.String(), + types.BalanceRegionScheduler.String(), + types.BalanceHotRegionScheduler.String(), } checkDisabled := func(name string, shouldDisabled bool) { re.NotNil(schedulersController.GetScheduler(name), name) @@ -395,14 +396,14 @@ func (suite *serverTestSuite) TestSchedulerSync() { func checkEvictLeaderSchedulerExist(re *require.Assertions, sc *schedulers.Controller, exist bool) { testutil.Eventually(re, func() bool { if !exist { - return sc.GetScheduler(schedulers.EvictLeaderName) == nil + return sc.GetScheduler(types.EvictLeaderScheduler.String()) == nil } - return sc.GetScheduler(schedulers.EvictLeaderName) != nil + return sc.GetScheduler(types.EvictLeaderScheduler.String()) != nil }) } func checkEvictLeaderStoreIDs(re *require.Assertions, sc *schedulers.Controller, expected []uint64) { - handler, ok := sc.GetSchedulerHandlers()[schedulers.EvictLeaderName] + handler, ok := sc.GetSchedulerHandlers()[types.EvictLeaderScheduler.String()] re.True(ok) h, ok := handler.(interface { EvictStoreIDs() []uint64 diff --git a/tests/integrations/realcluster/scheduler_test.go b/tests/integrations/realcluster/scheduler_test.go index 3b75e6c8c88..98a18158114 100644 --- a/tests/integrations/realcluster/scheduler_test.go +++ b/tests/integrations/realcluster/scheduler_test.go @@ -25,7 +25,7 @@ import ( pd "github.com/tikv/pd/client/http" "github.com/tikv/pd/client/testutil" "github.com/tikv/pd/pkg/schedule/labeler" - "github.com/tikv/pd/pkg/schedule/schedulers" + "github.com/tikv/pd/pkg/schedule/types" ) // https://github.com/tikv/pd/issues/6988#issuecomment-1694924611 @@ -47,9 +47,9 @@ func TestTransferLeader(t *testing.T) { } // record scheduler - re.NoError(pdHTTPCli.CreateScheduler(ctx, schedulers.EvictLeaderName, 1)) + re.NoError(pdHTTPCli.CreateScheduler(ctx, types.EvictLeaderScheduler.String(), 1)) defer func() { - re.NoError(pdHTTPCli.DeleteScheduler(ctx, schedulers.EvictLeaderName)) + re.NoError(pdHTTPCli.DeleteScheduler(ctx, types.EvictLeaderScheduler.String())) }() res, err := pdHTTPCli.GetSchedulers(ctx) re.NoError(err) @@ -89,16 +89,16 @@ func TestRegionLabelDenyScheduler(t *testing.T) { re.NotEmpty(regions.Regions) region1 := regions.Regions[0] - err = pdHTTPCli.DeleteScheduler(ctx, schedulers.BalanceLeaderName) + err = pdHTTPCli.DeleteScheduler(ctx, types.BalanceLeaderScheduler.String()) if err == nil { defer func() { - pdHTTPCli.CreateScheduler(ctx, schedulers.BalanceLeaderName, 0) + pdHTTPCli.CreateScheduler(ctx, types.BalanceLeaderScheduler.String(), 0) }() } - re.NoError(pdHTTPCli.CreateScheduler(ctx, schedulers.GrantLeaderName, uint64(region1.Leader.StoreID))) + re.NoError(pdHTTPCli.CreateScheduler(ctx, types.GrantLeaderScheduler.String(), uint64(region1.Leader.StoreID))) defer func() { - pdHTTPCli.DeleteScheduler(ctx, schedulers.GrantLeaderName) + pdHTTPCli.DeleteScheduler(ctx, types.GrantLeaderScheduler.String()) }() // wait leader transfer @@ -135,10 +135,10 @@ func TestRegionLabelDenyScheduler(t *testing.T) { re.Equal(labelRule.RuleType, labelRules[1].RuleType) // enable evict leader scheduler, and check it works - re.NoError(pdHTTPCli.DeleteScheduler(ctx, schedulers.GrantLeaderName)) - re.NoError(pdHTTPCli.CreateScheduler(ctx, schedulers.EvictLeaderName, uint64(region1.Leader.StoreID))) + re.NoError(pdHTTPCli.DeleteScheduler(ctx, types.GrantLeaderScheduler.String())) + re.NoError(pdHTTPCli.CreateScheduler(ctx, types.EvictLeaderScheduler.String(), uint64(region1.Leader.StoreID))) defer func() { - pdHTTPCli.DeleteScheduler(ctx, schedulers.EvictLeaderName) + pdHTTPCli.DeleteScheduler(ctx, types.EvictLeaderScheduler.String()) }() testutil.Eventually(re, func() bool { regions, err := pdHTTPCli.GetRegions(ctx) @@ -151,10 +151,10 @@ func TestRegionLabelDenyScheduler(t *testing.T) { return true }, testutil.WithWaitFor(time.Minute)) - re.NoError(pdHTTPCli.DeleteScheduler(ctx, schedulers.EvictLeaderName)) - re.NoError(pdHTTPCli.CreateScheduler(ctx, schedulers.GrantLeaderName, uint64(region1.Leader.StoreID))) + re.NoError(pdHTTPCli.DeleteScheduler(ctx, types.EvictLeaderScheduler.String())) + re.NoError(pdHTTPCli.CreateScheduler(ctx, types.GrantLeaderScheduler.String(), uint64(region1.Leader.StoreID))) defer func() { - pdHTTPCli.DeleteScheduler(ctx, schedulers.GrantLeaderName) + pdHTTPCli.DeleteScheduler(ctx, types.GrantLeaderScheduler.String()) }() testutil.Eventually(re, func() bool { regions, err := pdHTTPCli.GetRegions(ctx) diff --git a/tests/server/api/scheduler_test.go b/tests/server/api/scheduler_test.go index f3b8509d2e5..330a69eca63 100644 --- a/tests/server/api/scheduler_test.go +++ b/tests/server/api/scheduler_test.go @@ -29,7 +29,7 @@ import ( "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" sc "github.com/tikv/pd/pkg/schedule/config" - types "github.com/tikv/pd/pkg/schedule/type" + "github.com/tikv/pd/pkg/schedule/types" "github.com/tikv/pd/pkg/slice" "github.com/tikv/pd/pkg/utils/apiutil" tu "github.com/tikv/pd/pkg/utils/testutil" diff --git a/tests/server/cluster/cluster_test.go b/tests/server/cluster/cluster_test.go index 27d3300b293..375edf027db 100644 --- a/tests/server/cluster/cluster_test.go +++ b/tests/server/cluster/cluster_test.go @@ -40,6 +40,7 @@ import ( sc "github.com/tikv/pd/pkg/schedule/config" "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/schedulers" + "github.com/tikv/pd/pkg/schedule/types" "github.com/tikv/pd/pkg/statistics" "github.com/tikv/pd/pkg/storage" "github.com/tikv/pd/pkg/syncer" @@ -1410,10 +1411,10 @@ func TestTransferLeaderForScheduler(t *testing.T) { re.True(leaderServer.GetRaftCluster().IsPrepared()) schedsNum := len(rc.GetCoordinator().GetSchedulersController().GetSchedulerNames()) // Add evict leader scheduler - api.MustAddScheduler(re, leaderServer.GetAddr(), schedulers.EvictLeaderName, map[string]any{ + api.MustAddScheduler(re, leaderServer.GetAddr(), types.EvictLeaderScheduler.String(), map[string]any{ "store_id": 1, }) - api.MustAddScheduler(re, leaderServer.GetAddr(), schedulers.EvictLeaderName, map[string]any{ + api.MustAddScheduler(re, leaderServer.GetAddr(), types.EvictLeaderScheduler.String(), map[string]any{ "store_id": 2, }) // Check scheduler updated. @@ -1468,14 +1469,14 @@ func TestTransferLeaderForScheduler(t *testing.T) { func checkEvictLeaderSchedulerExist(re *require.Assertions, sc *schedulers.Controller, exist bool) { testutil.Eventually(re, func() bool { if !exist { - return sc.GetScheduler(schedulers.EvictLeaderName) == nil + return sc.GetScheduler(types.EvictLeaderScheduler.String()) == nil } - return sc.GetScheduler(schedulers.EvictLeaderName) != nil + return sc.GetScheduler(types.EvictLeaderScheduler.String()) != nil }) } func checkEvictLeaderStoreIDs(re *require.Assertions, sc *schedulers.Controller, expected []uint64) { - handler, ok := sc.GetSchedulerHandlers()[schedulers.EvictLeaderName] + handler, ok := sc.GetSchedulerHandlers()[types.EvictLeaderScheduler.String()] re.True(ok) h, ok := handler.(interface { EvictStoreIDs() []uint64 From 6c30bbf6eda61747d96f8ae090fba5d1e298c4a6 Mon Sep 17 00:00:00 2001 From: Jack Lyu <63168620+JackL9u@users.noreply.github.com> Date: Wed, 21 Aug 2024 14:31:41 +0800 Subject: [PATCH 10/18] ctl: replace gc_safepoint call with PD HTTP SDK (#8504) ref tikv/pd#7300 replace gc_safepoint call with PD HTTP SDK Signed-off-by: Boyang Lyu Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- client/http/api.go | 6 + client/http/interface.go | 30 ++++ client/http/request_info.go | 2 + client/http/types.go | 16 ++ pkg/storage/endpoint/gc_safe_point.go | 1 + server/api/service_gc_safepoint.go | 1 + tests/integrations/client/http_client_test.go | 82 +++++++++++ .../pdctl/command/gc_safepoint_command.go | 34 +---- .../pd-ctl/tests/safepoint/safepoint_test.go | 138 ++++++++++++++++++ 9 files changed, 284 insertions(+), 26 deletions(-) create mode 100644 tools/pd-ctl/tests/safepoint/safepoint_test.go diff --git a/client/http/api.go b/client/http/api.go index 3376a48770d..d1bce99f4f9 100644 --- a/client/http/api.go +++ b/client/http/api.go @@ -79,6 +79,7 @@ const ( Status = "/pd/api/v1/status" Version = "/pd/api/v1/version" operators = "/pd/api/v1/operators" + safepoint = "/pd/api/v1/gc/safepoint" // Micro Service microServicePrefix = "/pd/api/v2/ms" // Keyspace @@ -215,3 +216,8 @@ func GetUpdateKeyspaceConfigURL(keyspaceName string) string { func GetKeyspaceMetaByNameURL(keyspaceName string) string { return fmt.Sprintf(GetKeyspaceMetaByName, keyspaceName) } + +// GetDeleteSafePointURI returns the URI for delete safepoint service +func GetDeleteSafePointURI(serviceID string) string { + return fmt.Sprintf("%s/%s", safepoint, serviceID) +} diff --git a/client/http/interface.go b/client/http/interface.go index cd9fc22702e..f5cd1a38211 100644 --- a/client/http/interface.go +++ b/client/http/interface.go @@ -100,6 +100,8 @@ type Client interface { /* Other interfaces */ GetMinResolvedTSByStoresIDs(context.Context, []uint64) (uint64, map[uint64]uint64, error) GetPDVersion(context.Context) (string, error) + GetGCSafePoint(context.Context) (ListServiceGCSafepoint, error) + DeleteGCSafePoint(context.Context, string) (string, error) /* Micro Service interfaces */ GetMicroServiceMembers(context.Context, string) ([]MicroServiceMember, error) GetMicroServicePrimary(context.Context, string) (string, error) @@ -1024,3 +1026,31 @@ func (c *client) GetKeyspaceMetaByName(ctx context.Context, keyspaceName string) } return &keyspaceMetaPB, nil } + +// GetGCSafePoint gets the GC safe point list. +func (c *client) GetGCSafePoint(ctx context.Context) (ListServiceGCSafepoint, error) { + var gcSafePoint ListServiceGCSafepoint + err := c.request(ctx, newRequestInfo(). + WithName(GetGCSafePointName). + WithURI(safepoint). + WithMethod(http.MethodGet). + WithResp(&gcSafePoint)) + if err != nil { + return gcSafePoint, err + } + return gcSafePoint, nil +} + +// DeleteGCSafePoint deletes a GC safe point with the given service ID. +func (c *client) DeleteGCSafePoint(ctx context.Context, serviceID string) (string, error) { + var msg string + err := c.request(ctx, newRequestInfo(). + WithName(DeleteGCSafePointName). + WithURI(GetDeleteSafePointURI(serviceID)). + WithMethod(http.MethodDelete). + WithResp(&msg)) + if err != nil { + return msg, err + } + return msg, nil +} diff --git a/client/http/request_info.go b/client/http/request_info.go index 783220bcc60..94f71c6186e 100644 --- a/client/http/request_info.go +++ b/client/http/request_info.go @@ -85,6 +85,8 @@ const ( deleteOperators = "DeleteOperators" UpdateKeyspaceGCManagementTypeName = "UpdateKeyspaceGCManagementType" GetKeyspaceMetaByNameName = "GetKeyspaceMetaByName" + GetGCSafePointName = "GetGCSafePoint" + DeleteGCSafePointName = "DeleteGCSafePoint" ) type requestInfo struct { diff --git a/client/http/types.go b/client/http/types.go index 55f9b65caad..4bc60978a0e 100644 --- a/client/http/types.go +++ b/client/http/types.go @@ -25,6 +25,22 @@ import ( pd "github.com/tikv/pd/client" ) +// ServiceSafePoint is the safepoint for a specific service +// NOTE: This type is in sync with pd/pkg/storage/endpoint/gc_safe_point.go +type ServiceSafePoint struct { + ServiceID string `json:"service_id"` + ExpiredAt int64 `json:"expired_at"` + SafePoint uint64 `json:"safe_point"` +} + +// ListServiceGCSafepoint is the response for list service GC safepoint. +// NOTE: This type is in sync with pd/server/api/service_gc_safepoint.go +type ListServiceGCSafepoint struct { + ServiceGCSafepoints []*ServiceSafePoint `json:"service_gc_safe_points"` + MinServiceGcSafepoint uint64 `json:"min_service_gc_safe_point,omitempty"` + GCSafePoint uint64 `json:"gc_safe_point"` +} + // ClusterState saves some cluster state information. // NOTE: This type sync with https://github.com/tikv/pd/blob/5eae459c01a797cbd0c416054c6f0cad16b8740a/server/cluster/cluster.go#L173 type ClusterState struct { diff --git a/pkg/storage/endpoint/gc_safe_point.go b/pkg/storage/endpoint/gc_safe_point.go index 8d59d827fa4..7b0b0bf86a7 100644 --- a/pkg/storage/endpoint/gc_safe_point.go +++ b/pkg/storage/endpoint/gc_safe_point.go @@ -28,6 +28,7 @@ import ( // ServiceSafePoint is the safepoint for a specific service // NOTE: This type is exported by HTTP API. Please pay more attention when modifying it. +// This type is in sync with `client/http/types.go`. type ServiceSafePoint struct { ServiceID string `json:"service_id"` ExpiredAt int64 `json:"expired_at"` diff --git a/server/api/service_gc_safepoint.go b/server/api/service_gc_safepoint.go index d6bb153eb6f..ca29f9c352f 100644 --- a/server/api/service_gc_safepoint.go +++ b/server/api/service_gc_safepoint.go @@ -38,6 +38,7 @@ func newServiceGCSafepointHandler(svr *server.Server, rd *render.Render) *servic // ListServiceGCSafepoint is the response for list service GC safepoint. // NOTE: This type is exported by HTTP API. Please pay more attention when modifying it. +// This type is in sync with `pd/client/http/types.go`. type ListServiceGCSafepoint struct { ServiceGCSafepoints []*endpoint.ServiceSafePoint `json:"service_gc_safe_points"` MinServiceGcSafepoint uint64 `json:"min_service_gc_safe_point,omitempty"` diff --git a/tests/integrations/client/http_client_test.go b/tests/integrations/client/http_client_test.go index fd8b65f01ba..fe0962012e6 100644 --- a/tests/integrations/client/http_client_test.go +++ b/tests/integrations/client/http_client_test.go @@ -37,9 +37,11 @@ import ( sc "github.com/tikv/pd/pkg/schedule/config" "github.com/tikv/pd/pkg/schedule/labeler" "github.com/tikv/pd/pkg/schedule/placement" + "github.com/tikv/pd/pkg/storage/endpoint" "github.com/tikv/pd/pkg/utils/testutil" "github.com/tikv/pd/pkg/utils/tsoutil" "github.com/tikv/pd/pkg/versioninfo" + "github.com/tikv/pd/server/api" "github.com/tikv/pd/tests" ) @@ -835,3 +837,83 @@ func (suite *httpClientTestSuite) TestRetryOnLeaderChange() { cancel() wg.Wait() } + +func (suite *httpClientTestSuite) TestGetGCSafePoint() { + re := suite.Require() + client := suite.client + ctx, cancel := context.WithCancel(suite.ctx) + defer cancel() + + // adding some safepoints to the server + list := &api.ListServiceGCSafepoint{ + ServiceGCSafepoints: []*endpoint.ServiceSafePoint{ + { + ServiceID: "AAA", + ExpiredAt: time.Now().Unix() + 10, + SafePoint: 1, + }, + { + ServiceID: "BBB", + ExpiredAt: time.Now().Unix() + 10, + SafePoint: 2, + }, + { + ServiceID: "CCC", + ExpiredAt: time.Now().Unix() + 10, + SafePoint: 3, + }, + }, + GCSafePoint: 1, + MinServiceGcSafepoint: 1, + } + + storage := suite.cluster.GetLeaderServer().GetServer().GetStorage() + for _, ssp := range list.ServiceGCSafepoints { + err := storage.SaveServiceGCSafePoint(ssp) + re.NoError(err) + } + storage.SaveGCSafePoint(1) + + // get the safepoints and start testing + l, err := client.GetGCSafePoint(ctx) + re.NoError(err) + + re.Equal(uint64(1), l.GCSafePoint) + re.Equal(uint64(1), l.MinServiceGcSafepoint) + re.Len(l.ServiceGCSafepoints, 3) + + // sort the gc safepoints based on order of ServiceID + sort.Slice(l.ServiceGCSafepoints, func(i, j int) bool { + return l.ServiceGCSafepoints[i].ServiceID < l.ServiceGCSafepoints[j].ServiceID + }) + + for i, val := range l.ServiceGCSafepoints { + re.Equal(list.ServiceGCSafepoints[i].ServiceID, val.ServiceID) + re.Equal(list.ServiceGCSafepoints[i].SafePoint, val.SafePoint) + } + + // delete the safepoints + for i := 0; i < 3; i++ { + msg, err := client.DeleteGCSafePoint(ctx, list.ServiceGCSafepoints[i].ServiceID) + re.NoError(err) + re.Equal("Delete service GC safepoint successfully.", msg) + } + + // check that the safepoitns are indeed deleted + l, err = client.GetGCSafePoint(ctx) + re.NoError(err) + + re.Equal(uint64(1), l.GCSafePoint) + re.Equal(uint64(0), l.MinServiceGcSafepoint) + re.Empty(l.ServiceGCSafepoints) + + // try delete gc_worker, should get an error + _, err = client.DeleteGCSafePoint(ctx, "gc_worker") + re.Error(err) + + // try delete some non-exist safepoints, should return normally + var msg string + msg, err = client.DeleteGCSafePoint(ctx, "non_exist") + re.NoError(err) + re.Equal("Delete service GC safepoint successfully.", msg) +} diff --git a/tools/pd-ctl/pdctl/command/gc_safepoint_command.go b/tools/pd-ctl/pdctl/command/gc_safepoint_command.go index f4a6b6fcfd0..9a07d92937f 100644 --- a/tools/pd-ctl/pdctl/command/gc_safepoint_command.go +++ b/tools/pd-ctl/pdctl/command/gc_safepoint_command.go @@ -15,24 +15,18 @@ package command import ( - "encoding/json" - "net/http" "sort" "github.com/spf13/cobra" - "github.com/tikv/pd/server/api" -) - -var ( - serviceGCSafepointPrefix = "pd/api/v1/gc/safepoint" ) // NewServiceGCSafepointCommand return a service gc safepoint subcommand of rootCmd func NewServiceGCSafepointCommand() *cobra.Command { l := &cobra.Command{ - Use: "service-gc-safepoint", - Short: "show all service gc safepoint", - Run: showSSPs, + Use: "service-gc-safepoint", + Short: "show all service gc safepoint", + PersistentPreRunE: requirePDClient, + Run: showSSPs, } l.AddCommand(NewDeleteServiceGCSafepointCommand()) return l @@ -50,25 +44,15 @@ func NewDeleteServiceGCSafepointCommand() *cobra.Command { } func showSSPs(cmd *cobra.Command, _ []string) { - r, err := doRequest(cmd, serviceGCSafepointPrefix, http.MethodGet, http.Header{}) + safepoint, err := PDCli.GetGCSafePoint(cmd.Context()) if err != nil { cmd.Printf("Failed to get service GC safepoint: %s\n", err) return } - var safepoint api.ListServiceGCSafepoint - if err := json.Unmarshal([]byte(r), &safepoint); err != nil { - cmd.Printf("Failed to unmarshal service GC safepoint: %s\n", err) - return - } sort.Slice(safepoint.ServiceGCSafepoints, func(i, j int) bool { return safepoint.ServiceGCSafepoints[i].SafePoint < safepoint.ServiceGCSafepoints[j].SafePoint }) - data, err := json.MarshalIndent(safepoint, "", " ") - if err != nil { - cmd.Printf("Failed to marshal service GC safepoint: %s\n", err) - return - } - cmd.Println(string(data)) + jsonPrint(cmd, safepoint) } func deleteSSP(cmd *cobra.Command, args []string) { @@ -76,12 +60,10 @@ func deleteSSP(cmd *cobra.Command, args []string) { cmd.Usage() return } - serviceID := args[0] - deleteURL := serviceGCSafepointPrefix + "/" + serviceID - r, err := doRequest(cmd, deleteURL, http.MethodDelete, http.Header{}) + r, err := PDCli.DeleteGCSafePoint(cmd.Context(), args[0]) if err != nil { cmd.Printf("Failed to delete service GC safepoint: %s\n", err) return } - cmd.Println(r) + jsonPrint(cmd, r) } diff --git a/tools/pd-ctl/tests/safepoint/safepoint_test.go b/tools/pd-ctl/tests/safepoint/safepoint_test.go new file mode 100644 index 00000000000..5551cce1fff --- /dev/null +++ b/tools/pd-ctl/tests/safepoint/safepoint_test.go @@ -0,0 +1,138 @@ +// Copyright 2024 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package safepoint_test + +import ( + "context" + "encoding/json" + "sort" + "testing" + "time" + + "github.com/stretchr/testify/require" + "github.com/tikv/pd/pkg/storage/endpoint" + "github.com/tikv/pd/server/api" + pdTests "github.com/tikv/pd/tests" + ctl "github.com/tikv/pd/tools/pd-ctl/pdctl" + "github.com/tikv/pd/tools/pd-ctl/tests" +) + +func TestSafepoint(t *testing.T) { + re := require.New(t) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + tc, err := pdTests.NewTestCluster(ctx, 3) + re.NoError(err) + defer tc.Destroy() + err = tc.RunInitialServers() + re.NoError(err) + tc.WaitLeader() + leaderServer := tc.GetLeaderServer() + re.NoError(leaderServer.BootstrapCluster()) + pdAddr := tc.GetConfig().GetClientURL() + cmd := ctl.GetRootCmd() + + // add some gc_safepoint to the server + list := &api.ListServiceGCSafepoint{ + ServiceGCSafepoints: []*endpoint.ServiceSafePoint{ + { + ServiceID: "AAA", + ExpiredAt: time.Now().Unix() + 10, + SafePoint: 1, + }, + { + ServiceID: "BBB", + ExpiredAt: time.Now().Unix() + 10, + SafePoint: 2, + }, + { + ServiceID: "CCC", + ExpiredAt: time.Now().Unix() + 10, + SafePoint: 3, + }, + }, + GCSafePoint: 1, + MinServiceGcSafepoint: 1, + } + + storage := leaderServer.GetServer().GetStorage() + for _, ssp := range list.ServiceGCSafepoints { + err := storage.SaveServiceGCSafePoint(ssp) + re.NoError(err) + } + storage.SaveGCSafePoint(1) + + // get the safepoints + args := []string{"-u", pdAddr, "service-gc-safepoint"} + output, err := tests.ExecuteCommand(cmd, args...) + re.NoError(err) + + // create an container to hold the received values + var l api.ListServiceGCSafepoint + re.NoError(json.Unmarshal(output, &l)) + + // test if the points are what we expected + re.Equal(uint64(1), l.GCSafePoint) + re.Equal(uint64(1), l.MinServiceGcSafepoint) + re.Len(l.ServiceGCSafepoints, 3) + + // sort the gc safepoints based on order of ServiceID + sort.Slice(l.ServiceGCSafepoints, func(i, j int) bool { + return l.ServiceGCSafepoints[i].ServiceID < l.ServiceGCSafepoints[j].ServiceID + }) + + for i, val := range l.ServiceGCSafepoints { + re.Equal(list.ServiceGCSafepoints[i].ServiceID, val.ServiceID) + re.Equal(list.ServiceGCSafepoints[i].SafePoint, val.SafePoint) + } + + // delete the safepoints + for i := 0; i < 3; i++ { + args = []string{"-u", pdAddr, "service-gc-safepoint", "delete", list.ServiceGCSafepoints[i].ServiceID} + output, err = tests.ExecuteCommand(cmd, args...) + re.NoError(err) + var msg string + re.NoError(json.Unmarshal(output, &msg)) + re.Equal("Delete service GC safepoint successfully.", msg) + } + + // do a second round of get safepoints to ensure that the safe points are indeed deleted + args = []string{"-u", pdAddr, "service-gc-safepoint"} + output, err = tests.ExecuteCommand(cmd, args...) + re.NoError(err) + + var ll api.ListServiceGCSafepoint + re.NoError(json.Unmarshal(output, &ll)) + + re.Equal(uint64(1), ll.GCSafePoint) + re.Equal(uint64(0), ll.MinServiceGcSafepoint) + re.Empty(ll.ServiceGCSafepoints) + + // try delete the "gc_worker", should get an error message + args = []string{"-u", pdAddr, "service-gc-safepoint", "delete", "gc_worker"} + output, err = tests.ExecuteCommand(cmd, args...) + re.NoError(err) + + // output should be an error message + re.Equal("Failed to delete service GC safepoint: request pd http api failed with status: '500 Internal Server Error', body: '\"cannot remove service safe point of gc_worker\"'\n", string(output)) + + // try delete a non-exist safepoint, should return normally + args = []string{"-u", pdAddr, "service-gc-safepoint", "delete", "non_exist"} + output, err = tests.ExecuteCommand(cmd, args...) + re.NoError(err) + var msg string + re.NoError(json.Unmarshal(output, &msg)) + re.Equal("Delete service GC safepoint successfully.", msg) +} From cbb5a5bfd549d5a80891ce88e40aba3d499f3422 Mon Sep 17 00:00:00 2001 From: Hu# Date: Wed, 21 Aug 2024 15:45:12 +0800 Subject: [PATCH 11/18] ms: change the default name of the microservice to lowercase (#8551) close tikv/pd#8550 ms: change the default name of the microservice to lowercase Signed-off-by: husharp --- pkg/mcs/resourcemanager/server/config.go | 2 +- pkg/mcs/scheduling/server/config/config.go | 2 +- pkg/mcs/tso/server/config.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/mcs/resourcemanager/server/config.go b/pkg/mcs/resourcemanager/server/config.go index 9a899c9dc07..47f07486c1d 100644 --- a/pkg/mcs/resourcemanager/server/config.go +++ b/pkg/mcs/resourcemanager/server/config.go @@ -35,7 +35,7 @@ import ( ) const ( - defaultName = "Resource Manager" + defaultName = "resource manager" defaultBackendEndpoints = "http://127.0.0.1:2379" defaultListenAddr = "http://127.0.0.1:3379" diff --git a/pkg/mcs/scheduling/server/config/config.go b/pkg/mcs/scheduling/server/config/config.go index 4d7cee91f4a..cca2651d99e 100644 --- a/pkg/mcs/scheduling/server/config/config.go +++ b/pkg/mcs/scheduling/server/config/config.go @@ -47,7 +47,7 @@ import ( ) const ( - defaultName = "Scheduling" + defaultName = "scheduling" defaultBackendEndpoints = "http://127.0.0.1:2379" defaultListenAddr = "http://127.0.0.1:3379" ) diff --git a/pkg/mcs/tso/server/config.go b/pkg/mcs/tso/server/config.go index 8a3fe1ca161..209a9deb949 100644 --- a/pkg/mcs/tso/server/config.go +++ b/pkg/mcs/tso/server/config.go @@ -37,7 +37,7 @@ import ( const ( defaultMaxResetTSGap = 24 * time.Hour - defaultName = "TSO" + defaultName = "tso" defaultBackendEndpoints = "http://127.0.0.1:2379" defaultListenAddr = "http://127.0.0.1:3379" From b132ea64d4f5fb5a53fa3fa18e5ed7c796e4d4ae Mon Sep 17 00:00:00 2001 From: lhy1024 Date: Thu, 22 Aug 2024 10:58:44 +0800 Subject: [PATCH 12/18] api: remove duplicate `/debug/pprof/trace` register (#8554) close tikv/pd#8553 Signed-off-by: lhy1024 --- server/util.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/server/util.go b/server/util.go index 83455e2a6fe..b80a07ab28a 100644 --- a/server/util.go +++ b/server/util.go @@ -17,7 +17,6 @@ package server import ( "context" "net/http" - "net/http/pprof" "path/filepath" "strings" @@ -133,10 +132,6 @@ func combineBuilderServerHTTPService(ctx context.Context, svr *Server, serviceBu apiService.UseHandler(router) userHandlers[pdAPIPrefix] = apiService - - // fix issue https://github.com/tikv/pd/issues/7253 - // FIXME: remove me after upgrade - userHandlers["/debug/pprof/trace"] = http.HandlerFunc(pprof.Trace) return userHandlers, nil } From c963e778efb19ca189765116699e97619fb05020 Mon Sep 17 00:00:00 2001 From: Hu# Date: Thu, 22 Aug 2024 11:09:12 +0800 Subject: [PATCH 13/18] tests/resource_control: enable trace log to help diagnose (#8545) ref tikv/pd#8512 enable trace log to help diagnose Signed-off-by: husharp Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- pkg/mcs/resourcemanager/server/config.go | 3 ++- .../resourcemanager/resource_manager_test.go | 17 +++++++++-------- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/pkg/mcs/resourcemanager/server/config.go b/pkg/mcs/resourcemanager/server/config.go index 47f07486c1d..360f3b169b9 100644 --- a/pkg/mcs/resourcemanager/server/config.go +++ b/pkg/mcs/resourcemanager/server/config.go @@ -127,8 +127,9 @@ func (rmc *ControllerConfig) Adjust(meta *configutil.ConfigMetaData) { if !meta.IsDefined("ltb-token-rpc-max-delay") { configutil.AdjustDuration(&rmc.LTBTokenRPCMaxDelay, defaultLTBTokenRPCMaxDelay) } - failpoint.Inject("enableDegradedMode", func() { + failpoint.Inject("enableDegradedModeAndTraceLog", func() { configutil.AdjustDuration(&rmc.DegradedModeWaitDuration, time.Second) + configutil.AdjustBool(&rmc.EnableControllerTraceLog, true) }) } diff --git a/tests/integrations/mcs/resourcemanager/resource_manager_test.go b/tests/integrations/mcs/resourcemanager/resource_manager_test.go index 10b1a0b4520..4b2e2040ab0 100644 --- a/tests/integrations/mcs/resourcemanager/resource_manager_test.go +++ b/tests/integrations/mcs/resourcemanager/resource_manager_test.go @@ -28,6 +28,7 @@ import ( "github.com/pingcap/failpoint" rmpb "github.com/pingcap/kvproto/pkg/resource_manager" + "github.com/pingcap/log" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" pd "github.com/tikv/pd/client" @@ -37,6 +38,7 @@ import ( "github.com/tikv/pd/pkg/utils/typeutil" "github.com/tikv/pd/tests" "go.uber.org/goleak" + "go.uber.org/zap" // Register Service _ "github.com/tikv/pd/pkg/mcs/registry" @@ -64,7 +66,7 @@ func (suite *resourceManagerClientTestSuite) SetupSuite() { var err error re := suite.Require() - re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/mcs/resourcemanager/server/enableDegradedMode", `return(true)`)) + re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/mcs/resourcemanager/server/enableDegradedModeAndTraceLog", `return(true)`)) re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/member/skipCampaignLeaderCheck", "return(true)")) suite.ctx, suite.clean = context.WithCancel(context.Background()) @@ -151,7 +153,7 @@ func (suite *resourceManagerClientTestSuite) TearDownSuite() { suite.client.Close() suite.cluster.Destroy() suite.clean() - re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/mcs/resourcemanager/server/enableDegradedMode")) + re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/mcs/resourcemanager/server/enableDegradedModeAndTraceLog")) re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/member/skipCampaignLeaderCheck")) } @@ -410,11 +412,9 @@ func (suite *resourceManagerClientTestSuite) TestResourceGroupController() { testCases := []struct { resourceGroupName string tcs []tokenConsumptionPerSecond - len int }{ { resourceGroupName: rg.Name, - len: 8, tcs: []tokenConsumptionPerSecond{ {rruTokensAtATime: 50, wruTokensAtATime: 20, times: 100, waitDuration: 0}, {rruTokensAtATime: 50, wruTokensAtATime: 100, times: 100, waitDuration: 0}, @@ -434,7 +434,7 @@ func (suite *resourceManagerClientTestSuite) TestResourceGroupController() { v := false <-tricker.C for _, cas := range testCases { - if i >= cas.len { + if i >= len(cas.tcs) { continue } v = true @@ -452,8 +452,9 @@ func (suite *resourceManagerClientTestSuite) TestResourceGroupController() { sum += time.Since(startTime) rgsController.OnResponse(cas.resourceGroupName, rreq, rres) rgsController.OnResponse(cas.resourceGroupName, wreq, wres) - time.Sleep(1000 * time.Microsecond) + time.Sleep(time.Millisecond) } + log.Info("finished test case", zap.Int("index", i), zap.Duration("sum", sum), zap.Duration("waitDuration", cas.tcs[i].waitDuration)) re.LessOrEqual(sum, buffDuration+cas.tcs[i].waitDuration) } i++ @@ -1479,12 +1480,12 @@ func (suite *resourceManagerClientTestSuite) TestResourceGroupControllerConfigCh readBaseCost := 1.5 defaultCfg := controller.DefaultConfig() expectCfg := server.ControllerConfig{ - // failpoint enableDegradedMode will setup and set it be 1s. + // failpoint enableDegradedModeAndTraceLog will set it be 1s and enable trace log. DegradedModeWaitDuration: typeutil.NewDuration(time.Second), + EnableControllerTraceLog: true, LTBMaxWaitDuration: typeutil.Duration(defaultCfg.LTBMaxWaitDuration), LTBTokenRPCMaxDelay: typeutil.Duration(defaultCfg.LTBTokenRPCMaxDelay), RequestUnit: server.RequestUnitConfig(defaultCfg.RequestUnit), - EnableControllerTraceLog: defaultCfg.EnableControllerTraceLog, } expectRUCfg := controller.GenerateRUConfig(defaultCfg) expectRUCfg.DegradedModeWaitDuration = time.Second From 32829f140db8a7b783668b8350e29d42c05c7aa4 Mon Sep 17 00:00:00 2001 From: Hu# Date: Thu, 22 Aug 2024 11:49:42 +0800 Subject: [PATCH 14/18] tests: make TestAllocIDAfterLeaderChange more stable (#8558) close tikv/pd#8557 need to wait leader Signed-off-by: husharp Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- tests/integrations/mcs/scheduling/server_test.go | 1 + tests/server/join/join_test.go | 2 ++ tests/server/watch/leader_watch_test.go | 1 + 3 files changed, 4 insertions(+) diff --git a/tests/integrations/mcs/scheduling/server_test.go b/tests/integrations/mcs/scheduling/server_test.go index 57306640394..4a19321ce57 100644 --- a/tests/integrations/mcs/scheduling/server_test.go +++ b/tests/integrations/mcs/scheduling/server_test.go @@ -106,6 +106,7 @@ func (suite *serverTestSuite) TestAllocIDAfterLeaderChange() { pd2, err := suite.cluster.Join(suite.ctx) re.NoError(err) err = pd2.Run() + re.NotEmpty(suite.cluster.WaitLeader()) re.NoError(err) tc, err := tests.NewTestSchedulingCluster(suite.ctx, 1, suite.backendEndpoints) re.NoError(err) diff --git a/tests/server/join/join_test.go b/tests/server/join/join_test.go index 1eaa59a4e2a..5d087caf5e4 100644 --- a/tests/server/join/join_test.go +++ b/tests/server/join/join_test.go @@ -58,6 +58,7 @@ func TestSimpleJoin(t *testing.T) { re.NoError(err) _, err = os.Stat(filepath.Join(pd2.GetConfig().DataDir, "join")) re.False(os.IsNotExist(err)) + re.NotEmpty(cluster.WaitLeader()) members, err = etcdutil.ListEtcdMembers(ctx, client) re.NoError(err) re.Len(members.Members, 2) @@ -73,6 +74,7 @@ func TestSimpleJoin(t *testing.T) { re.NoError(err) _, err = os.Stat(filepath.Join(pd3.GetConfig().DataDir, "join")) re.False(os.IsNotExist(err)) + re.NotEmpty(cluster.WaitLeader()) members, err = etcdutil.ListEtcdMembers(ctx, client) re.NoError(err) re.Len(members.Members, 3) diff --git a/tests/server/watch/leader_watch_test.go b/tests/server/watch/leader_watch_test.go index 863cf7a0c08..39c54b5c500 100644 --- a/tests/server/watch/leader_watch_test.go +++ b/tests/server/watch/leader_watch_test.go @@ -57,6 +57,7 @@ func TestWatcher(t *testing.T) { re.NoError(failpoint.Enable("github.com/tikv/pd/server/delayWatcher", `pause`)) err = pd3.Run() re.NoError(err) + re.NotEmpty(cluster.WaitLeader()) time.Sleep(200 * time.Millisecond) re.Equal(pd1.GetConfig().Name, pd3.GetLeader().GetName()) err = pd1.Stop() From 8e45f9e956436cd0f3924ccadebacbb7fa2def3a Mon Sep 17 00:00:00 2001 From: Hu# Date: Thu, 22 Aug 2024 12:01:43 +0800 Subject: [PATCH 15/18] tests: unify frequency check time to make test stable (#8556) close tikv/pd#8555 tests: unify frequency check time to make test stable Signed-off-by: husharp Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- pkg/member/member.go | 8 +------- tests/integrations/client/client_test.go | 8 ++------ tests/integrations/client/http_client_test.go | 4 ++++ tests/integrations/mcs/scheduling/server_test.go | 4 ++-- tests/server/cluster/cluster_test.go | 8 ++++---- tests/server/id/id_test.go | 4 ++-- tests/server/region_syncer/region_syncer_test.go | 4 ++-- 7 files changed, 17 insertions(+), 23 deletions(-) diff --git a/pkg/member/member.go b/pkg/member/member.go index 8bf50e6d310..2dc8be52031 100644 --- a/pkg/member/member.go +++ b/pkg/member/member.go @@ -187,13 +187,7 @@ func (m *EmbeddedEtcdMember) CampaignLeader(ctx context.Context, leaseTimeout in failpoint.Return(m.leadership.Campaign(leaseTimeout, m.MemberValue())) }) - checkTimes := campaignLeaderFrequencyTimes - failpoint.Inject("changeFrequencyTimes", func(val failpoint.Value) { - if v, ok := val.(int); ok { - checkTimes = v - } - }) - if m.leadership.GetCampaignTimesNum() > checkTimes { + if m.leadership.GetCampaignTimesNum() > campaignLeaderFrequencyTimes { if err := m.ResignEtcdLeader(ctx, m.Name(), ""); err != nil { return err } diff --git a/tests/integrations/client/client_test.go b/tests/integrations/client/client_test.go index 9517444d017..3c8bba1fafd 100644 --- a/tests/integrations/client/client_test.go +++ b/tests/integrations/client/client_test.go @@ -166,19 +166,15 @@ func TestClientLeaderChange(t *testing.T) { func TestLeaderTransferAndMoveCluster(t *testing.T) { re := require.New(t) - re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/member/changeFrequencyTimes", "return(10)")) + re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/member/skipCampaignLeaderCheck", "return(true)")) defer func() { - re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/member/changeFrequencyTimes")) + re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/member/skipCampaignLeaderCheck")) }() ctx, cancel := context.WithCancel(context.Background()) defer cancel() cluster, err := tests.NewTestCluster(ctx, 3) re.NoError(err) defer cluster.Destroy() - re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/member/skipCampaignLeaderCheck", "return(true)")) - defer func() { - re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/member/skipCampaignLeaderCheck")) - }() endpoints := runServer(re, cluster) cli := setupCli(ctx, re, endpoints) diff --git a/tests/integrations/client/http_client_test.go b/tests/integrations/client/http_client_test.go index fe0962012e6..68643d5f0c8 100644 --- a/tests/integrations/client/http_client_test.go +++ b/tests/integrations/client/http_client_test.go @@ -26,6 +26,7 @@ import ( "time" "github.com/pingcap/errors" + "github.com/pingcap/failpoint" "github.com/pingcap/kvproto/pkg/metapb" "github.com/prometheus/client_golang/prometheus" dto "github.com/prometheus/client_model/go" @@ -71,6 +72,7 @@ func TestHTTPClientTestSuiteWithServiceDiscovery(t *testing.T) { func (suite *httpClientTestSuite) SetupSuite() { re := suite.Require() + re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/member/skipCampaignLeaderCheck", "return(true)")) suite.ctx, suite.cancelFunc = context.WithCancel(context.Background()) cluster, err := tests.NewTestCluster(suite.ctx, 2) @@ -125,6 +127,8 @@ func (suite *httpClientTestSuite) SetupSuite() { } func (suite *httpClientTestSuite) TearDownSuite() { + re := suite.Require() + re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/member/skipCampaignLeaderCheck")) suite.cancelFunc() suite.client.Close() suite.cluster.Destroy() diff --git a/tests/integrations/mcs/scheduling/server_test.go b/tests/integrations/mcs/scheduling/server_test.go index 4a19321ce57..085b87afe86 100644 --- a/tests/integrations/mcs/scheduling/server_test.go +++ b/tests/integrations/mcs/scheduling/server_test.go @@ -654,9 +654,9 @@ func (suite *multipleServerTestSuite) TearDownSuite() { func (suite *multipleServerTestSuite) TestReElectLeader() { re := suite.Require() - re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/member/changeFrequencyTimes", "return(10)")) + re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/member/skipCampaignLeaderCheck", "return(true)")) defer func() { - re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/member/changeFrequencyTimes")) + re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/member/skipCampaignLeaderCheck")) }() tc, err := tests.NewTestSchedulingCluster(suite.ctx, 1, suite.backendEndpoints) re.NoError(err) diff --git a/tests/server/cluster/cluster_test.go b/tests/server/cluster/cluster_test.go index 375edf027db..a1a7f68646e 100644 --- a/tests/server/cluster/cluster_test.go +++ b/tests/server/cluster/cluster_test.go @@ -187,9 +187,9 @@ func TestDamagedRegion(t *testing.T) { func TestRegionStatistics(t *testing.T) { re := require.New(t) - re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/member/changeFrequencyTimes", "return(10)")) + re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/member/skipCampaignLeaderCheck", "return(true)")) defer func() { - re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/member/changeFrequencyTimes")) + re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/member/skipCampaignLeaderCheck")) }() ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -1650,9 +1650,9 @@ func TestMinResolvedTS(t *testing.T) { // See https://github.com/tikv/pd/issues/4941 func TestTransferLeaderBack(t *testing.T) { re := require.New(t) - re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/member/changeFrequencyTimes", "return(10)")) + re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/member/skipCampaignLeaderCheck", "return(true)")) defer func() { - re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/member/changeFrequencyTimes")) + re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/member/skipCampaignLeaderCheck")) }() ctx, cancel := context.WithCancel(context.Background()) defer cancel() diff --git a/tests/server/id/id_test.go b/tests/server/id/id_test.go index c7dee0d2924..8b0e7ec60b7 100644 --- a/tests/server/id/id_test.go +++ b/tests/server/id/id_test.go @@ -108,9 +108,9 @@ func TestCommand(t *testing.T) { func TestMonotonicID(t *testing.T) { re := require.New(t) - re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/member/changeFrequencyTimes", "return(10)")) + re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/member/skipCampaignLeaderCheck", "return(true)")) defer func() { - re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/member/changeFrequencyTimes")) + re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/member/skipCampaignLeaderCheck")) }() ctx, cancel := context.WithCancel(context.Background()) defer cancel() diff --git a/tests/server/region_syncer/region_syncer_test.go b/tests/server/region_syncer/region_syncer_test.go index 6a5c1ea361c..80f5186f904 100644 --- a/tests/server/region_syncer/region_syncer_test.go +++ b/tests/server/region_syncer/region_syncer_test.go @@ -255,9 +255,9 @@ func TestPrepareChecker(t *testing.T) { // ref: https://github.com/tikv/pd/issues/6988 func TestPrepareCheckerWithTransferLeader(t *testing.T) { re := require.New(t) - re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/member/changeFrequencyTimes", "return(10)")) + re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/member/skipCampaignLeaderCheck", "return(true)")) defer func() { - re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/member/changeFrequencyTimes")) + re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/member/skipCampaignLeaderCheck")) }() ctx, cancel := context.WithCancel(context.Background()) defer cancel() From 84138e4b876908274f68cedbb4c5e4c57472ee34 Mon Sep 17 00:00:00 2001 From: okJiang <819421878@qq.com> Date: Thu, 22 Aug 2024 15:27:13 +0800 Subject: [PATCH 16/18] heartbeat: reduce unnecessary `RunTask` (#8559) ref tikv/pd#7897 Signed-off-by: okJiang <819421878@qq.com> Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- pkg/cluster/cluster.go | 32 +++++++++++++--------------- pkg/mcs/scheduling/server/cluster.go | 21 ++++++++++-------- server/cluster/cluster.go | 26 +++++++++++----------- 3 files changed, 41 insertions(+), 38 deletions(-) diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index ddba8f89fb6..b1abe4677df 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -74,22 +74,20 @@ func HandleOverlaps(ctx context.Context, c Cluster, overlaps []*core.RegionInfo) } // Collect collects the cluster information. -func Collect(ctx context.Context, c Cluster, region *core.RegionInfo, hasRegionStats bool) { - if hasRegionStats { - // get region again from root tree. make sure the observed region is the latest. - bc := c.GetBasicCluster() - if bc == nil { - return - } - region = bc.GetRegion(region.GetID()) - if region == nil { - return - } - select { - case <-ctx.Done(): - return - default: - } - c.GetRegionStats().Observe(region, c.GetBasicCluster().GetRegionStores(region)) +func Collect(ctx context.Context, c Cluster, region *core.RegionInfo) { + // get region again from root tree. make sure the observed region is the latest. + bc := c.GetBasicCluster() + if bc == nil { + return + } + region = bc.GetRegion(region.GetID()) + if region == nil { + return + } + select { + case <-ctx.Done(): + return + default: } + c.GetRegionStats().Observe(region, c.GetBasicCluster().GetRegionStores(region)) } diff --git a/pkg/mcs/scheduling/server/cluster.go b/pkg/mcs/scheduling/server/cluster.go index 31120ae6082..0dcb26a1a1f 100644 --- a/pkg/mcs/scheduling/server/cluster.go +++ b/pkg/mcs/scheduling/server/cluster.go @@ -631,7 +631,7 @@ func (c *Cluster) processRegionHeartbeat(ctx *core.MetaProcessContext, region *c regionID, ratelimit.ObserveRegionStatsAsync, func(ctx context.Context) { - cluster.Collect(ctx, c, region, hasRegionStats) + cluster.Collect(ctx, c, region) }, ) } @@ -679,14 +679,17 @@ func (c *Cluster) processRegionHeartbeat(ctx *core.MetaProcessContext, region *c ) } tracer.OnSaveCacheFinished() - // handle region stats - ctx.TaskRunner.RunTask( - regionID, - ratelimit.CollectRegionStatsAsync, - func(ctx context.Context) { - cluster.Collect(ctx, c, region, hasRegionStats) - }, - ) + if hasRegionStats { + // handle region stats + ctx.TaskRunner.RunTask( + regionID, + ratelimit.CollectRegionStatsAsync, + func(ctx context.Context) { + cluster.Collect(ctx, c, region) + }, + ) + } + tracer.OnCollectRegionStatsFinished() return nil } diff --git a/server/cluster/cluster.go b/server/cluster/cluster.go index 9e12b158f42..9e85a0f1d84 100644 --- a/server/cluster/cluster.go +++ b/server/cluster/cluster.go @@ -1062,7 +1062,7 @@ func (c *RaftCluster) processRegionHeartbeat(ctx *core.MetaProcessContext, regio regionID, ratelimit.ObserveRegionStatsAsync, func(ctx context.Context) { - cluster.Collect(ctx, c, region, hasRegionStats) + cluster.Collect(ctx, c, region) }, ) } @@ -1119,17 +1119,19 @@ func (c *RaftCluster) processRegionHeartbeat(ctx *core.MetaProcessContext, regio } tracer.OnSaveCacheFinished() - // handle region stats - ctx.MiscRunner.RunTask( - regionID, - ratelimit.CollectRegionStatsAsync, - func(ctx context.Context) { - // TODO: Due to the accuracy requirements of the API "/regions/check/xxx", - // region stats needs to be collected in API mode. - // We need to think of a better way to reduce this part of the cost in the future. - cluster.Collect(ctx, c, region, hasRegionStats) - }, - ) + if hasRegionStats { + // handle region stats + ctx.MiscRunner.RunTask( + regionID, + ratelimit.CollectRegionStatsAsync, + func(ctx context.Context) { + // TODO: Due to the accuracy requirements of the API "/regions/check/xxx", + // region stats needs to be collected in API mode. + // We need to think of a better way to reduce this part of the cost in the future. + cluster.Collect(ctx, c, region) + }, + ) + } tracer.OnCollectRegionStatsFinished() if c.storage != nil { From 9e7a1b975d4ca63baffb23de80396cb324f64095 Mon Sep 17 00:00:00 2001 From: okJiang <819421878@qq.com> Date: Thu, 22 Aug 2024 15:52:42 +0800 Subject: [PATCH 17/18] *: make TestStores stable (#8522) close tikv/pd#7954 add a failpoint to make sure the offline store do not convert to tombstone Signed-off-by: okJiang <819421878@qq.com> --- server/cluster/cluster.go | 7 +++++-- tests/integrations/mcs/scheduling/api_test.go | 10 +++++----- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/server/cluster/cluster.go b/server/cluster/cluster.go index 9e85a0f1d84..fdeffd15e9c 100644 --- a/server/cluster/cluster.go +++ b/server/cluster/cluster.go @@ -1669,9 +1669,12 @@ func (c *RaftCluster) checkStores() { if c.IsPrepared() { c.updateProgress(id, store.GetAddress(), removingAction, float64(regionSize), float64(regionSize), false /* dec */) } - regionCount := c.GetStoreRegionCount(id) // If the store is empty, it can be buried. - if regionCount == 0 { + needBury := c.GetStoreRegionCount(id) == 0 + failpoint.Inject("doNotBuryStore", func(_ failpoint.Value) { + needBury = false + }) + if needBury { if err := c.BuryStore(id, false); err != nil { log.Error("bury store failed", zap.Stringer("store", offlineStore), diff --git a/tests/integrations/mcs/scheduling/api_test.go b/tests/integrations/mcs/scheduling/api_test.go index 68f276f9e22..d2027a616cb 100644 --- a/tests/integrations/mcs/scheduling/api_test.go +++ b/tests/integrations/mcs/scheduling/api_test.go @@ -629,13 +629,13 @@ func (suite *apiTestSuite) checkStores(cluster *tests.TestCluster) { Version: "2.0.0", }, } - // prevent the offline store from changing to tombstone - tests.MustPutRegion(re, cluster, 3, 6, []byte("a"), []byte("b")) + + re.NoError(failpoint.Enable("github.com/tikv/pd/server/cluster/doNotBuryStore", `return(true)`)) + defer func() { + re.NoError(failpoint.Disable("github.com/tikv/pd/server/cluster/doNotBuryStore")) + }() for _, store := range stores { tests.MustPutStore(re, cluster, store) - if store.GetId() == 6 { - cluster.GetLeaderServer().GetRaftCluster().GetBasicCluster().UpdateStoreStatus(6) - } } // Test /stores apiServerAddr := cluster.GetLeaderServer().GetAddr() From 61a85e52ecb851008953bb4efb4939681f2b1ade Mon Sep 17 00:00:00 2001 From: okJiang <819421878@qq.com> Date: Thu, 22 Aug 2024 16:52:42 +0800 Subject: [PATCH 18/18] *: enable revive.exported linter (#8563) close tikv/pd#8458 Signed-off-by: okJiang <819421878@qq.com> Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- .golangci.yml | 3 +- tools/pd-api-bench/cases/cases.go | 68 ++++++++-------- tools/pd-api-bench/cases/controller.go | 64 +++++++-------- tools/pd-ctl/pdctl/command/global.go | 1 + tools/pd-heartbeat-bench/metrics/util.go | 19 +++-- tools/pd-simulator/simulator/client.go | 78 +++++++++++-------- tools/pd-simulator/simulator/config/config.go | 2 + tools/pd-simulator/simulator/drive.go | 17 ++-- tools/pd-simulator/simulator/event.go | 4 +- tools/pd-simulator/simulator/raft.go | 2 +- tools/pd-tso-bench/main.go | 10 +-- tools/pd-ut/alloc/server.go | 1 + 12 files changed, 145 insertions(+), 124 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index e09011c45be..e5b5fe1ab56 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -226,5 +226,4 @@ issues: linters: - errcheck include: - # remove the comment after the path is ready - # - EXC0012 + - EXC0012 diff --git a/tools/pd-api-bench/cases/cases.go b/tools/pd-api-bench/cases/cases.go index d81e7e0fa9d..f863d3248c6 100644 --- a/tools/pd-api-bench/cases/cases.go +++ b/tools/pd-api-bench/cases/cases.go @@ -80,12 +80,12 @@ func (c *Config) Clone() *Config { // Case is the interface for all cases. type Case interface { - Name() string - SetQPS(int64) - GetQPS() int64 - SetBurst(int64) - GetBurst() int64 - GetConfig() *Config + getName() string + setQPS(int64) + getQPS() int64 + setBurst(int64) + getBurst() int64 + getConfig() *Config } type baseCase struct { @@ -93,35 +93,35 @@ type baseCase struct { cfg *Config } -func (c *baseCase) Name() string { +func (c *baseCase) getName() string { return c.name } -func (c *baseCase) SetQPS(qps int64) { +func (c *baseCase) setQPS(qps int64) { c.cfg.QPS = qps } -func (c *baseCase) GetQPS() int64 { +func (c *baseCase) getQPS() int64 { return c.cfg.QPS } -func (c *baseCase) SetBurst(burst int64) { +func (c *baseCase) setBurst(burst int64) { c.cfg.Burst = burst } -func (c *baseCase) GetBurst() int64 { +func (c *baseCase) getBurst() int64 { return c.cfg.Burst } -func (c *baseCase) GetConfig() *Config { +func (c *baseCase) getConfig() *Config { return c.cfg.Clone() } // EtcdCase is the interface for all etcd api cases. type EtcdCase interface { Case - Init(context.Context, *clientv3.Client) error - Unary(context.Context, *clientv3.Client) error + init(context.Context, *clientv3.Client) error + unary(context.Context, *clientv3.Client) error } // EtcdCreateFn is function type to create EtcdCase. @@ -138,7 +138,7 @@ var EtcdCaseFnMap = map[string]EtcdCreateFn{ // GRPCCase is the interface for all gRPC cases. type GRPCCase interface { Case - Unary(context.Context, pd.Client) error + unary(context.Context, pd.Client) error } // GRPCCreateFn is function type to create GRPCCase. @@ -159,7 +159,7 @@ var GRPCCaseFnMap = map[string]GRPCCreateFn{ // HTTPCase is the interface for all HTTP cases. type HTTPCase interface { Case - Do(context.Context, pdHttp.Client) error + do(context.Context, pdHttp.Client) error } // HTTPCreateFn is function type to create HTTPCase. @@ -186,7 +186,7 @@ func newMinResolvedTS() func() HTTPCase { } } -func (c *minResolvedTS) Do(ctx context.Context, cli pdHttp.Client) error { +func (c *minResolvedTS) do(ctx context.Context, cli pdHttp.Client) error { minResolvedTS, storesMinResolvedTS, err := cli.GetMinResolvedTSByStoresIDs(ctx, storesID) if Debug { log.Info("do HTTP case", zap.String("case", c.name), zap.Uint64("min-resolved-ts", minResolvedTS), zap.Any("store-min-resolved-ts", storesMinResolvedTS), zap.Error(err)) @@ -214,7 +214,7 @@ func newRegionStats() func() HTTPCase { } } -func (c *regionsStats) Do(ctx context.Context, cli pdHttp.Client) error { +func (c *regionsStats) do(ctx context.Context, cli pdHttp.Client) error { upperBound := totalRegion / c.regionSample if upperBound < 1 { upperBound = 1 @@ -248,7 +248,7 @@ func newUpdateGCSafePoint() func() GRPCCase { } } -func (*updateGCSafePoint) Unary(ctx context.Context, cli pd.Client) error { +func (*updateGCSafePoint) unary(ctx context.Context, cli pd.Client) error { s := time.Now().Unix() _, err := cli.UpdateGCSafePoint(ctx, uint64(s)) if err != nil { @@ -272,7 +272,7 @@ func newUpdateServiceGCSafePoint() func() GRPCCase { } } -func (*updateServiceGCSafePoint) Unary(ctx context.Context, cli pd.Client) error { +func (*updateServiceGCSafePoint) unary(ctx context.Context, cli pd.Client) error { s := time.Now().Unix() id := rand.Int63n(100) + 1 _, err := cli.UpdateServiceGCSafePoint(ctx, strconv.FormatInt(id, 10), id, uint64(s)) @@ -297,7 +297,7 @@ func newGetRegion() func() GRPCCase { } } -func (*getRegion) Unary(ctx context.Context, cli pd.Client) error { +func (*getRegion) unary(ctx context.Context, cli pd.Client) error { id := rand.Intn(totalRegion)*4 + 1 _, err := cli.GetRegion(ctx, generateKeyForSimulator(id)) if err != nil { @@ -321,7 +321,7 @@ func newGetRegionEnableFollower() func() GRPCCase { } } -func (*getRegionEnableFollower) Unary(ctx context.Context, cli pd.Client) error { +func (*getRegionEnableFollower) unary(ctx context.Context, cli pd.Client) error { id := rand.Intn(totalRegion)*4 + 1 _, err := cli.GetRegion(ctx, generateKeyForSimulator(id), pd.WithAllowFollowerHandle()) if err != nil { @@ -347,7 +347,7 @@ func newScanRegions() func() GRPCCase { } } -func (c *scanRegions) Unary(ctx context.Context, cli pd.Client) error { +func (c *scanRegions) unary(ctx context.Context, cli pd.Client) error { upperBound := totalRegion / c.regionSample random := rand.Intn(upperBound) startID := c.regionSample*random*4 + 1 @@ -375,7 +375,7 @@ func newTso() func() GRPCCase { } } -func (*tso) Unary(ctx context.Context, cli pd.Client) error { +func (*tso) unary(ctx context.Context, cli pd.Client) error { _, _, err := cli.GetTS(ctx) if err != nil { return err @@ -398,7 +398,7 @@ func newGetStore() func() GRPCCase { } } -func (*getStore) Unary(ctx context.Context, cli pd.Client) error { +func (*getStore) unary(ctx context.Context, cli pd.Client) error { storeIdx := rand.Intn(totalStore) _, err := cli.GetStore(ctx, storesID[storeIdx]) if err != nil { @@ -422,7 +422,7 @@ func newGetStores() func() GRPCCase { } } -func (*getStores) Unary(ctx context.Context, cli pd.Client) error { +func (*getStores) unary(ctx context.Context, cli pd.Client) error { _, err := cli.GetAllStores(ctx) if err != nil { return err @@ -451,7 +451,7 @@ func newGetKV() func() EtcdCase { } } -func (*getKV) Init(ctx context.Context, cli *clientv3.Client) error { +func (*getKV) init(ctx context.Context, cli *clientv3.Client) error { for i := 0; i < 100; i++ { _, err := cli.Put(ctx, fmt.Sprintf("/test/0001/%4d", i), fmt.Sprintf("%4d", i)) if err != nil { @@ -461,7 +461,7 @@ func (*getKV) Init(ctx context.Context, cli *clientv3.Client) error { return nil } -func (*getKV) Unary(ctx context.Context, cli *clientv3.Client) error { +func (*getKV) unary(ctx context.Context, cli *clientv3.Client) error { _, err := cli.Get(ctx, "/test/0001", clientv3.WithPrefix()) return err } @@ -481,9 +481,9 @@ func newPutKV() func() EtcdCase { } } -func (*putKV) Init(context.Context, *clientv3.Client) error { return nil } +func (*putKV) init(context.Context, *clientv3.Client) error { return nil } -func (*putKV) Unary(ctx context.Context, cli *clientv3.Client) error { +func (*putKV) unary(ctx context.Context, cli *clientv3.Client) error { _, err := cli.Put(ctx, "/test/0001/0000", "test") return err } @@ -503,9 +503,9 @@ func newDeleteKV() func() EtcdCase { } } -func (*deleteKV) Init(context.Context, *clientv3.Client) error { return nil } +func (*deleteKV) init(context.Context, *clientv3.Client) error { return nil } -func (*deleteKV) Unary(ctx context.Context, cli *clientv3.Client) error { +func (*deleteKV) unary(ctx context.Context, cli *clientv3.Client) error { _, err := cli.Delete(ctx, "/test/0001/0000") return err } @@ -525,9 +525,9 @@ func newTxnKV() func() EtcdCase { } } -func (*txnKV) Init(context.Context, *clientv3.Client) error { return nil } +func (*txnKV) init(context.Context, *clientv3.Client) error { return nil } -func (*txnKV) Unary(ctx context.Context, cli *clientv3.Client) error { +func (*txnKV) unary(ctx context.Context, cli *clientv3.Client) error { txn := cli.Txn(ctx) txn = txn.If(clientv3.Compare(clientv3.Value("/test/0001/0000"), "=", "test")) txn = txn.Then(clientv3.OpPut("/test/0001/0000", "test2")) diff --git a/tools/pd-api-bench/cases/controller.go b/tools/pd-api-bench/cases/controller.go index e19c79c0f3e..a77474db3a7 100644 --- a/tools/pd-api-bench/cases/controller.go +++ b/tools/pd-api-bench/cases/controller.go @@ -62,7 +62,7 @@ func (c *Coordinator) GetHTTPCase(name string) (*Config, error) { c.mu.RLock() defer c.mu.RUnlock() if controller, ok := c.http[name]; ok { - return controller.GetConfig(), nil + return controller.getConfig(), nil } return nil, errors.Errorf("case %v does not exist", name) } @@ -72,7 +72,7 @@ func (c *Coordinator) GetGRPCCase(name string) (*Config, error) { c.mu.RLock() defer c.mu.RUnlock() if controller, ok := c.grpc[name]; ok { - return controller.GetConfig(), nil + return controller.getConfig(), nil } return nil, errors.Errorf("case %v does not exist", name) } @@ -82,7 +82,7 @@ func (c *Coordinator) GetEtcdCase(name string) (*Config, error) { c.mu.RLock() defer c.mu.RUnlock() if controller, ok := c.etcd[name]; ok { - return controller.GetConfig(), nil + return controller.getConfig(), nil } return nil, errors.Errorf("case %v does not exist", name) } @@ -93,7 +93,7 @@ func (c *Coordinator) GetAllHTTPCases() map[string]*Config { defer c.mu.RUnlock() ret := make(map[string]*Config) for name, c := range c.http { - ret[name] = c.GetConfig() + ret[name] = c.getConfig() } return ret } @@ -104,7 +104,7 @@ func (c *Coordinator) GetAllGRPCCases() map[string]*Config { defer c.mu.RUnlock() ret := make(map[string]*Config) for name, c := range c.grpc { - ret[name] = c.GetConfig() + ret[name] = c.getConfig() } return ret } @@ -115,7 +115,7 @@ func (c *Coordinator) GetAllEtcdCases() map[string]*Config { defer c.mu.RUnlock() ret := make(map[string]*Config) for name, c := range c.etcd { - ret[name] = c.GetConfig() + ret[name] = c.getConfig() } return ret } @@ -131,9 +131,9 @@ func (c *Coordinator) SetHTTPCase(name string, cfg *Config) error { c.http[name] = controller } controller.stop() - controller.SetQPS(cfg.QPS) + controller.setQPS(cfg.QPS) if cfg.Burst > 0 { - controller.SetBurst(cfg.Burst) + controller.setBurst(cfg.Burst) } controller.run() } else { @@ -153,9 +153,9 @@ func (c *Coordinator) SetGRPCCase(name string, cfg *Config) error { c.grpc[name] = controller } controller.stop() - controller.SetQPS(cfg.QPS) + controller.setQPS(cfg.QPS) if cfg.Burst > 0 { - controller.SetBurst(cfg.Burst) + controller.setBurst(cfg.Burst) } controller.run() } else { @@ -175,9 +175,9 @@ func (c *Coordinator) SetEtcdCase(name string, cfg *Config) error { c.etcd[name] = controller } controller.stop() - controller.SetQPS(cfg.QPS) + controller.setQPS(cfg.QPS) if cfg.Burst > 0 { - controller.SetBurst(cfg.Burst) + controller.setBurst(cfg.Burst) } controller.run() } else { @@ -207,15 +207,15 @@ func newHTTPController(ctx context.Context, clis []pdHttp.Client, fn HTTPCreateF // run tries to run the HTTP api bench. func (c *httpController) run() { - if c.GetQPS() <= 0 || c.cancel != nil { + if c.getQPS() <= 0 || c.cancel != nil { return } c.ctx, c.cancel = context.WithCancel(c.pctx) - qps := c.GetQPS() - burst := c.GetBurst() + qps := c.getQPS() + burst := c.getBurst() cliNum := int64(len(c.clients)) tt := time.Duration(base*burst*cliNum/qps) * time.Microsecond - log.Info("begin to run http case", zap.String("case", c.Name()), zap.Int64("qps", qps), zap.Int64("burst", burst), zap.Duration("interval", tt)) + log.Info("begin to run http case", zap.String("case", c.getName()), zap.Int64("qps", qps), zap.Int64("burst", burst), zap.Duration("interval", tt)) for _, hCli := range c.clients { c.wg.Add(1) go func(hCli pdHttp.Client) { @@ -229,9 +229,9 @@ func (c *httpController) run() { for { select { case <-ticker.C: - err := c.Do(c.ctx, hCli) + err := c.do(c.ctx, hCli) if err != nil { - log.Error("meet error when doing HTTP request", zap.String("case", c.Name()), zap.Error(err)) + log.Error("meet error when doing HTTP request", zap.String("case", c.getName()), zap.Error(err)) } case <-c.ctx.Done(): log.Info("got signal to exit running HTTP case") @@ -276,15 +276,15 @@ func newGRPCController(ctx context.Context, clis []pd.Client, fn GRPCCreateFn) * // run tries to run the gRPC api bench. func (c *gRPCController) run() { - if c.GetQPS() <= 0 || c.cancel != nil { + if c.getQPS() <= 0 || c.cancel != nil { return } c.ctx, c.cancel = context.WithCancel(c.pctx) - qps := c.GetQPS() - burst := c.GetBurst() + qps := c.getQPS() + burst := c.getBurst() cliNum := int64(len(c.clients)) tt := time.Duration(base*burst*cliNum/qps) * time.Microsecond - log.Info("begin to run gRPC case", zap.String("case", c.Name()), zap.Int64("qps", qps), zap.Int64("burst", burst), zap.Duration("interval", tt)) + log.Info("begin to run gRPC case", zap.String("case", c.getName()), zap.Int64("qps", qps), zap.Int64("burst", burst), zap.Duration("interval", tt)) for _, cli := range c.clients { c.wg.Add(1) go func(cli pd.Client) { @@ -298,9 +298,9 @@ func (c *gRPCController) run() { for { select { case <-ticker.C: - err := c.Unary(c.ctx, cli) + err := c.unary(c.ctx, cli) if err != nil { - log.Error("meet error when doing gRPC request", zap.String("case", c.Name()), zap.Error(err)) + log.Error("meet error when doing gRPC request", zap.String("case", c.getName()), zap.Error(err)) } case <-c.ctx.Done(): log.Info("got signal to exit running gRPC case") @@ -345,18 +345,18 @@ func newEtcdController(ctx context.Context, clis []*clientv3.Client, fn EtcdCrea // run tries to run the gRPC api bench. func (c *etcdController) run() { - if c.GetQPS() <= 0 || c.cancel != nil { + if c.getQPS() <= 0 || c.cancel != nil { return } c.ctx, c.cancel = context.WithCancel(c.pctx) - qps := c.GetQPS() - burst := c.GetBurst() + qps := c.getQPS() + burst := c.getBurst() cliNum := int64(len(c.clients)) tt := time.Duration(base*burst*cliNum/qps) * time.Microsecond - log.Info("begin to run etcd case", zap.String("case", c.Name()), zap.Int64("qps", qps), zap.Int64("burst", burst), zap.Duration("interval", tt)) - err := c.Init(c.ctx, c.clients[0]) + log.Info("begin to run etcd case", zap.String("case", c.getName()), zap.Int64("qps", qps), zap.Int64("burst", burst), zap.Duration("interval", tt)) + err := c.init(c.ctx, c.clients[0]) if err != nil { - log.Error("init error", zap.String("case", c.Name()), zap.Error(err)) + log.Error("init error", zap.String("case", c.getName()), zap.Error(err)) return } for _, cli := range c.clients { @@ -372,9 +372,9 @@ func (c *etcdController) run() { for { select { case <-ticker.C: - err := c.Unary(c.ctx, cli) + err := c.unary(c.ctx, cli) if err != nil { - log.Error("meet error when doing etcd request", zap.String("case", c.Name()), zap.Error(err)) + log.Error("meet error when doing etcd request", zap.String("case", c.getName()), zap.Error(err)) } case <-c.ctx.Done(): log.Info("got signal to exit running etcd case") diff --git a/tools/pd-ctl/pdctl/command/global.go b/tools/pd-ctl/pdctl/command/global.go index c5c633de217..c7e0cfd691d 100644 --- a/tools/pd-ctl/pdctl/command/global.go +++ b/tools/pd-ctl/pdctl/command/global.go @@ -33,6 +33,7 @@ import ( ) const ( + // PDControlCallerID is used to set the caller ID for PD client PDControlCallerID = "pd-ctl" clusterPrefix = "pd/api/v1/cluster" ) diff --git a/tools/pd-heartbeat-bench/metrics/util.go b/tools/pd-heartbeat-bench/metrics/util.go index e409cf10815..f747e5b507f 100644 --- a/tools/pd-heartbeat-bench/metrics/util.go +++ b/tools/pd-heartbeat-bench/metrics/util.go @@ -32,12 +32,12 @@ import ( var ( prometheusCli api.Client - finalMetrics2Collect []Metric + finalMetrics2Collect []metric avgRegionStats report.Stats avgStoreTime float64 collectRound = 1.0 - metrics2Collect = []Metric{ + metrics2Collect = []metric{ {promSQL: cpuMetric, name: "max cpu usage(%)", max: true}, {promSQL: memoryMetric, name: "max memory usage(G)", max: true}, {promSQL: goRoutineMetric, name: "max go routines", max: true}, @@ -69,7 +69,7 @@ var ( } ) -type Metric struct { +type metric struct { promSQL string name string value float64 @@ -77,9 +77,10 @@ type Metric struct { max bool } +// InitMetric2Collect initializes the metrics to collect func InitMetric2Collect(endpoint string) (withMetric bool) { for _, name := range breakdownNames { - metrics2Collect = append(metrics2Collect, Metric{ + metrics2Collect = append(metrics2Collect, metric{ promSQL: hbBreakdownMetricByName(name), name: name, }) @@ -94,7 +95,7 @@ func InitMetric2Collect(endpoint string) (withMetric bool) { log.Error("parse prometheus url error", zap.Error(err)) return false } - prometheusCli, err = NewPrometheusClient(*cu) + prometheusCli, err = newPrometheusClient(*cu) if err != nil { log.Error("create prometheus client error", zap.Error(err)) return false @@ -108,7 +109,7 @@ func InitMetric2Collect(endpoint string) (withMetric bool) { return true } -func NewPrometheusClient(prometheusURL url.URL) (api.Client, error) { +func newPrometheusClient(prometheusURL url.URL) (api.Client, error) { client, err := api.NewClient(api.Config{ Address: prometheusURL.String(), }) @@ -122,6 +123,7 @@ func NewPrometheusClient(prometheusURL url.URL) (api.Client, error) { // WarmUpRound wait for the first round to warm up const WarmUpRound = 1 +// CollectMetrics collects the metrics func CollectMetrics(curRound int, wait time.Duration) { if curRound < WarmUpRound { return @@ -183,7 +185,7 @@ func getMetric(cli api.Client, query string, ts time.Time) ([]float64, error) { return value, nil } -func formatMetrics(ms []Metric) string { +func formatMetrics(ms []metric) string { res := "" for _, m := range ms { res += "[" + m.name + "]" + " " + fmt.Sprintf("%.10f", m.value) + " " @@ -191,6 +193,7 @@ func formatMetrics(ms []Metric) string { return res } +// CollectRegionAndStoreStats collects the region and store stats func CollectRegionAndStoreStats(regionStats *report.Stats, storeTime *float64) { if regionStats != nil && storeTime != nil { collect(*regionStats, *storeTime) @@ -211,6 +214,7 @@ func collect(regionStats report.Stats, storeTime float64) { avgStoreTime = average(avgStoreTime, storeTime) } +// OutputConclusion outputs the final conclusion func OutputConclusion() { logFields := RegionFields(avgRegionStats, zap.Float64("avg store time", avgStoreTime), @@ -219,6 +223,7 @@ func OutputConclusion() { log.Info("final metrics collected", logFields...) } +// RegionFields returns the fields for region stats func RegionFields(stats report.Stats, fields ...zap.Field) []zap.Field { return append([]zap.Field{ zap.String("total", fmt.Sprintf("%.4fs", stats.Total.Seconds())), diff --git a/tools/pd-simulator/simulator/client.go b/tools/pd-simulator/simulator/client.go index 8acf3ccd9ab..224d54b05d6 100644 --- a/tools/pd-simulator/simulator/client.go +++ b/tools/pd-simulator/simulator/client.go @@ -40,13 +40,13 @@ import ( // Client is a PD (Placement Driver) client. // It should not be used after calling Close(). type Client interface { - AllocID(context.Context) (uint64, error) + allocID(context.Context) (uint64, error) PutStore(context.Context, *metapb.Store) error StoreHeartbeat(context.Context, *pdpb.StoreStats) error RegionHeartbeat(context.Context, *core.RegionInfo) error - HeartbeatStreamLoop() - ChangeConn(*grpc.ClientConn) error + heartbeatStreamLoop() + changeConn(*grpc.ClientConn) error Close() } @@ -61,15 +61,17 @@ const ( var ( // errFailInitClusterID is returned when failed to load clusterID from all supplied PD addresses. errFailInitClusterID = errors.New("[pd] failed to get cluster id") - PDHTTPClient pdHttp.Client - SD pd.ServiceDiscovery - ClusterID atomic.Uint64 + // PDHTTPClient is a client for PD HTTP API. + PDHTTPClient pdHttp.Client + // SD is a service discovery for PD. + SD pd.ServiceDiscovery + clusterID atomic.Uint64 ) // requestHeader returns a header for fixed ClusterID. func requestHeader() *pdpb.RequestHeader { return &pdpb.RequestHeader{ - ClusterId: ClusterID.Load(), + ClusterId: clusterID.Load(), } } @@ -110,7 +112,7 @@ func createConn(url string) (*grpc.ClientConn, error) { return cc, nil } -func (c *client) ChangeConn(cc *grpc.ClientConn) error { +func (c *client) changeConn(cc *grpc.ClientConn) error { c.clientConn = cc simutil.Logger.Info("change pd client with endpoints", zap.String("tag", c.tag), zap.String("pd-address", cc.Target())) return nil @@ -143,7 +145,7 @@ func (c *client) createHeartbeatStream() (pdpb.PD_RegionHeartbeatClient, context return stream, ctx, cancel } -func (c *client) HeartbeatStreamLoop() { +func (c *client) heartbeatStreamLoop() { c.wg.Add(1) defer c.wg.Done() for { @@ -173,10 +175,10 @@ func (c *client) HeartbeatStreamLoop() { if client := SD.GetServiceClient(); client != nil { _, conn, err := getLeaderURL(ctx, client.GetClientConn()) if err != nil { - simutil.Logger.Error("[HeartbeatStreamLoop] failed to get leader URL", zap.Error(err)) + simutil.Logger.Error("[heartbeatStreamLoop] failed to get leader URL", zap.Error(err)) continue } - if err = c.ChangeConn(conn); err == nil { + if err = c.changeConn(conn); err == nil { break } } @@ -234,6 +236,7 @@ func (c *client) reportRegionHeartbeat(ctx context.Context, stream pdpb.PD_Regio } } +// Close closes the client. func (c *client) Close() { if c.cancel == nil { simutil.Logger.Info("pd client has been closed", zap.String("tag", c.tag)) @@ -248,7 +251,7 @@ func (c *client) Close() { } } -func (c *client) AllocID(ctx context.Context) (uint64, error) { +func (c *client) allocID(ctx context.Context) (uint64, error) { ctx, cancel := context.WithTimeout(ctx, pdTimeout) resp, err := c.pdClient().AllocID(ctx, &pdpb.AllocIDRequest{ Header: requestHeader(), @@ -263,6 +266,7 @@ func (c *client) AllocID(ctx context.Context) (uint64, error) { return resp.GetId(), nil } +// PutStore sends PutStore to PD. func (c *client) PutStore(ctx context.Context, store *metapb.Store) error { ctx, cancel := context.WithTimeout(ctx, pdTimeout) newStore := typeutil.DeepClone(store, core.StoreFactory) @@ -281,6 +285,7 @@ func (c *client) PutStore(ctx context.Context, store *metapb.Store) error { return nil } +// StoreHeartbeat sends a StoreHeartbeat to PD. func (c *client) StoreHeartbeat(ctx context.Context, newStats *pdpb.StoreStats) error { ctx, cancel := context.WithTimeout(ctx, pdTimeout) resp, err := c.pdClient().StoreHeartbeat(ctx, &pdpb.StoreHeartbeatRequest{ @@ -298,17 +303,18 @@ func (c *client) StoreHeartbeat(ctx context.Context, newStats *pdpb.StoreStats) return nil } +// RegionHeartbeat sends a RegionHeartbeat to PD. func (c *client) RegionHeartbeat(_ context.Context, region *core.RegionInfo) error { c.reportRegionHeartbeatCh <- region return nil } -type RetryClient struct { +type retryClient struct { client Client retryCount int } -func NewRetryClient(node *Node) *RetryClient { +func newRetryClient(node *Node) *retryClient { // Init PD client and putting it into node. tag := fmt.Sprintf("store %d", node.Store.Id) var ( @@ -331,8 +337,8 @@ func NewRetryClient(node *Node) *RetryClient { } node.client = client - // Init RetryClient - retryClient := &RetryClient{ + // Init retryClient + retryClient := &retryClient{ client: client, retryCount: retryTimes, } @@ -342,12 +348,12 @@ func NewRetryClient(node *Node) *RetryClient { }) // start heartbeat stream node.receiveRegionHeartbeatCh = receiveRegionHeartbeatCh - go client.HeartbeatStreamLoop() + go client.heartbeatStreamLoop() return retryClient } -func (rc *RetryClient) requestWithRetry(f func() (any, error)) (any, error) { +func (rc *retryClient) requestWithRetry(f func() (any, error)) (any, error) { // execute the function directly if res, err := f(); err == nil { return res, nil @@ -362,7 +368,7 @@ func (rc *RetryClient) requestWithRetry(f func() (any, error)) (any, error) { simutil.Logger.Error("[retry] failed to get leader URL", zap.Error(err)) return nil, err } - if err = rc.client.ChangeConn(conn); err != nil { + if err = rc.client.changeConn(conn); err != nil { simutil.Logger.Error("failed to change connection", zap.Error(err)) return nil, err } @@ -381,8 +387,8 @@ func getLeaderURL(ctx context.Context, conn *grpc.ClientConn) (string, *grpc.Cli if members.GetHeader().GetError() != nil { return "", nil, errors.New(members.GetHeader().GetError().String()) } - ClusterID.Store(members.GetHeader().GetClusterId()) - if ClusterID.Load() == 0 { + clusterID.Store(members.GetHeader().GetClusterId()) + if clusterID.Load() == 0 { return "", nil, errors.New("cluster id is 0") } if members.GetLeader() == nil { @@ -393,9 +399,9 @@ func getLeaderURL(ctx context.Context, conn *grpc.ClientConn) (string, *grpc.Cli return leaderURL, conn, err } -func (rc *RetryClient) AllocID(ctx context.Context) (uint64, error) { +func (rc *retryClient) allocID(ctx context.Context) (uint64, error) { res, err := rc.requestWithRetry(func() (any, error) { - id, err := rc.client.AllocID(ctx) + id, err := rc.client.allocID(ctx) return id, err }) if err != nil { @@ -404,7 +410,8 @@ func (rc *RetryClient) AllocID(ctx context.Context) (uint64, error) { return res.(uint64), nil } -func (rc *RetryClient) PutStore(ctx context.Context, store *metapb.Store) error { +// PutStore sends PutStore to PD. +func (rc *retryClient) PutStore(ctx context.Context, store *metapb.Store) error { _, err := rc.requestWithRetry(func() (any, error) { err := rc.client.PutStore(ctx, store) return nil, err @@ -412,7 +419,8 @@ func (rc *RetryClient) PutStore(ctx context.Context, store *metapb.Store) error return err } -func (rc *RetryClient) StoreHeartbeat(ctx context.Context, newStats *pdpb.StoreStats) error { +// StoreHeartbeat sends a StoreHeartbeat to PD. +func (rc *retryClient) StoreHeartbeat(ctx context.Context, newStats *pdpb.StoreStats) error { _, err := rc.requestWithRetry(func() (any, error) { err := rc.client.StoreHeartbeat(ctx, newStats) return nil, err @@ -420,7 +428,8 @@ func (rc *RetryClient) StoreHeartbeat(ctx context.Context, newStats *pdpb.StoreS return err } -func (rc *RetryClient) RegionHeartbeat(ctx context.Context, region *core.RegionInfo) error { +// RegionHeartbeat sends a RegionHeartbeat to PD. +func (rc *retryClient) RegionHeartbeat(ctx context.Context, region *core.RegionInfo) error { _, err := rc.requestWithRetry(func() (any, error) { err := rc.client.RegionHeartbeat(ctx, region) return nil, err @@ -428,15 +437,16 @@ func (rc *RetryClient) RegionHeartbeat(ctx context.Context, region *core.RegionI return err } -func (*RetryClient) ChangeConn(_ *grpc.ClientConn) error { +func (*retryClient) changeConn(_ *grpc.ClientConn) error { panic("unImplement") } -func (rc *RetryClient) HeartbeatStreamLoop() { - rc.client.HeartbeatStreamLoop() +func (rc *retryClient) heartbeatStreamLoop() { + rc.client.heartbeatStreamLoop() } -func (rc *RetryClient) Close() { +// Close closes the client. +func (rc *retryClient) Close() { rc.client.Close() } @@ -465,10 +475,10 @@ retry: break retry } } - if ClusterID.Load() == 0 { + if clusterID.Load() == 0 { return "", nil, errors.WithStack(errFailInitClusterID) } - simutil.Logger.Info("get cluster id successfully", zap.Uint64("cluster-id", ClusterID.Load())) + simutil.Logger.Info("get cluster id successfully", zap.Uint64("cluster-id", clusterID.Load())) // Check if the cluster is already bootstrapped. ctx, cancel := context.WithTimeout(ctx, pdTimeout) @@ -513,7 +523,6 @@ retry: } /* PDHTTPClient is a client for PD HTTP API, these are the functions that are used in the simulator */ - func PutPDConfig(config *sc.PDConfig) error { if len(config.PlacementRules) > 0 { ruleOps := make([]*pdHttp.RuleOp, 0) @@ -541,8 +550,9 @@ func PutPDConfig(config *sc.PDConfig) error { return nil } +// ChooseToHaltPDSchedule is used to choose whether to halt the PD schedule func ChooseToHaltPDSchedule(halt bool) { - HaltSchedule.Store(halt) + haltSchedule.Store(halt) PDHTTPClient.SetConfig(context.Background(), map[string]any{ "schedule.halt-scheduling": strconv.FormatBool(halt), }) diff --git a/tools/pd-simulator/simulator/config/config.go b/tools/pd-simulator/simulator/config/config.go index 4d182a2a03f..32c483ceba7 100644 --- a/tools/pd-simulator/simulator/config/config.go +++ b/tools/pd-simulator/simulator/config/config.go @@ -130,6 +130,8 @@ func (sc *SimConfig) Adjust(meta *toml.MetaData) error { return sc.ServerConfig.Adjust(meta, false) } + +// Speed returns the tick speed of the simulator. func (sc *SimConfig) Speed() uint64 { return uint64(time.Second / sc.SimTickInterval.Duration) } diff --git a/tools/pd-simulator/simulator/drive.go b/tools/pd-simulator/simulator/drive.go index e602ba2df65..947d0664755 100644 --- a/tools/pd-simulator/simulator/drive.go +++ b/tools/pd-simulator/simulator/drive.go @@ -135,7 +135,7 @@ func (d *Driver) allocID() error { return err } ctx, cancel := context.WithTimeout(context.Background(), requestTimeout) - rootPath := path.Join("/pd", strconv.FormatUint(ClusterID.Load(), 10)) + rootPath := path.Join("/pd", strconv.FormatUint(clusterID.Load(), 10)) allocIDPath := path.Join(rootPath, "alloc_id") _, err = etcdClient.Put(ctx, allocIDPath, string(typeutil.Uint64ToBytes(maxID+1000))) if err != nil { @@ -171,7 +171,7 @@ func (d *Driver) updateNodesClient() error { PDHTTPClient = pdHttp.NewClientWithServiceDiscovery("pd-simulator", SD) for _, node := range d.conn.Nodes { - node.client = NewRetryClient(node) + node.client = newRetryClient(node) } return nil } @@ -191,6 +191,7 @@ func (d *Driver) Tick() { }() } +// StepRegions steps regions. func (d *Driver) StepRegions(ctx context.Context) { for { select { @@ -209,6 +210,7 @@ func (d *Driver) StepRegions(ctx context.Context) { } } +// StoresHeartbeat sends heartbeat to all stores. func (d *Driver) StoresHeartbeat(ctx context.Context) { config := d.raftEngine.storeConfig storeInterval := uint64(config.RaftStore.StoreHeartBeatInterval.Duration / config.SimTickInterval.Duration) @@ -229,6 +231,7 @@ func (d *Driver) StoresHeartbeat(ctx context.Context) { } } +// RegionsHeartbeat sends heartbeat to all regions. func (d *Driver) RegionsHeartbeat(ctx context.Context) { // ensure only wait for the first time heartbeat done firstReport := true @@ -281,9 +284,9 @@ func (d *Driver) RegionsHeartbeat(ctx context.Context) { } } - // Only set HaltSchedule to false when the leader count is 80% of the total region count. + // Only set haltSchedule to false when the leader count is 80% of the total region count. // using firstReport to avoid the haltSchedule set to true manually. - if HaltSchedule.Load() && firstReport { + if haltSchedule.Load() && firstReport { storeInterval := uint64(config.RaftStore.StoreHeartBeatInterval.Duration / config.SimTickInterval.Duration) ticker := time.NewTicker(time.Duration(storeInterval)) for range ticker.C { @@ -294,7 +297,7 @@ func (d *Driver) RegionsHeartbeat(ctx context.Context) { leaderCount += store.Status.LeaderCount } // Add halt schedule check to avoid the situation that the leader count is always less than 80%. - if leaderCount > int64(float64(d.simConfig.TotalRegion)*0.8) || !HaltSchedule.Load() { + if leaderCount > int64(float64(d.simConfig.TotalRegion)*0.8) || !haltSchedule.Load() { ChooseToHaltPDSchedule(false) firstReport = false ticker.Stop() @@ -310,11 +313,11 @@ func (d *Driver) RegionsHeartbeat(ctx context.Context) { } } -var HaltSchedule atomic.Bool +var haltSchedule atomic.Bool // Check checks if the simulation is completed. func (d *Driver) Check() bool { - if !HaltSchedule.Load() { + if !haltSchedule.Load() { return false } var stats []info.StoreStats diff --git a/tools/pd-simulator/simulator/event.go b/tools/pd-simulator/simulator/event.go index 86da86ed20d..d22f35756ef 100644 --- a/tools/pd-simulator/simulator/event.go +++ b/tools/pd-simulator/simulator/event.go @@ -177,7 +177,7 @@ type AddNode struct{} func (*AddNode) Run(raft *RaftEngine, _ int64) bool { config := raft.storeConfig nodes := raft.conn.getNodes() - id, err := nodes[0].client.AllocID(context.TODO()) + id, err := nodes[0].client.allocID(context.TODO()) if err != nil { simutil.Logger.Error("alloc node id failed", zap.Error(err)) return false @@ -196,7 +196,7 @@ func (*AddNode) Run(raft *RaftEngine, _ int64) bool { raft.conn.Nodes[s.ID] = n n.raftEngine = raft - n.client = NewRetryClient(n) + n.client = newRetryClient(n) err = n.Start() if err != nil { diff --git a/tools/pd-simulator/simulator/raft.go b/tools/pd-simulator/simulator/raft.go index 9a219d32f9f..7f3bf78622f 100644 --- a/tools/pd-simulator/simulator/raft.go +++ b/tools/pd-simulator/simulator/raft.go @@ -294,6 +294,6 @@ func (r *RaftEngine) allocID(storeID uint64) (uint64, error) { if !ok { return 0, errors.Errorf("node %d not found", storeID) } - id, err := node.client.AllocID(context.Background()) + id, err := node.client.allocID(context.Background()) return id, errors.WithStack(err) } diff --git a/tools/pd-tso-bench/main.go b/tools/pd-tso-bench/main.go index 1e59dfd2a2a..96ff2a51d0a 100644 --- a/tools/pd-tso-bench/main.go +++ b/tools/pd-tso-bench/main.go @@ -176,7 +176,7 @@ func showStats(ctx context.Context, durCh chan time.Duration) { case <-ticker.C: // runtime.GC() if *verbose { - fmt.Println(s.Counter()) + fmt.Println(s.counter()) } total.merge(s) s = newStats() @@ -184,8 +184,8 @@ func showStats(ctx context.Context, durCh chan time.Duration) { s.update(d) case <-statCtx.Done(): fmt.Println("\nTotal:") - fmt.Println(total.Counter()) - fmt.Println(total.Percentage()) + fmt.Println(total.counter()) + fmt.Println(total.percentage()) // Calculate the percentiles by using the tDigest algorithm. fmt.Printf("P0.5: %.4fms, P0.8: %.4fms, P0.9: %.4fms, P0.99: %.4fms\n\n", latencyTDigest.Quantile(0.5), latencyTDigest.Quantile(0.8), latencyTDigest.Quantile(0.9), latencyTDigest.Quantile(0.99)) if *verbose { @@ -329,7 +329,7 @@ func (s *stats) merge(other *stats) { s.oneThousandCnt += other.oneThousandCnt } -func (s *stats) Counter() string { +func (s *stats) counter() string { return fmt.Sprintf( "count: %d, max: %.4fms, min: %.4fms, avg: %.4fms\n<1ms: %d, >1ms: %d, >2ms: %d, >5ms: %d, >10ms: %d, >30ms: %d, >50ms: %d, >100ms: %d, >200ms: %d, >400ms: %d, >800ms: %d, >1s: %d", s.count, float64(s.maxDur.Nanoseconds())/float64(time.Millisecond), float64(s.minDur.Nanoseconds())/float64(time.Millisecond), float64(s.totalDur.Nanoseconds())/float64(s.count)/float64(time.Millisecond), @@ -337,7 +337,7 @@ func (s *stats) Counter() string { s.eightHundredCnt, s.oneThousandCnt) } -func (s *stats) Percentage() string { +func (s *stats) percentage() string { return fmt.Sprintf( "count: %d, <1ms: %2.2f%%, >1ms: %2.2f%%, >2ms: %2.2f%%, >5ms: %2.2f%%, >10ms: %2.2f%%, >30ms: %2.2f%%, >50ms: %2.2f%%, >100ms: %2.2f%%, >200ms: %2.2f%%, >400ms: %2.2f%%, >800ms: %2.2f%%, >1s: %2.2f%%", s.count, s.calculate(s.submilliCnt), s.calculate(s.milliCnt), s.calculate(s.twoMilliCnt), s.calculate(s.fiveMilliCnt), s.calculate(s.tenMSCnt), s.calculate(s.thirtyCnt), s.calculate(s.fiftyCnt), diff --git a/tools/pd-ut/alloc/server.go b/tools/pd-ut/alloc/server.go index a5643001142..ffa3bce0aa5 100644 --- a/tools/pd-ut/alloc/server.go +++ b/tools/pd-ut/alloc/server.go @@ -30,6 +30,7 @@ import ( var statusAddress = flag.String("status-addr", "0.0.0.0:0", "status address") +// RunHTTPServer runs a HTTP server to provide alloc address. func RunHTTPServer() *http.Server { err := os.Setenv(tempurl.AllocURLFromUT, fmt.Sprintf("http://%s/alloc", *statusAddress)) if err != nil {