From e35e814662f83a610768ae9fed13eddc5d584a47 Mon Sep 17 00:00:00 2001 From: MyonKeminta Date: Wed, 25 Sep 2024 15:23:35 +0800 Subject: [PATCH 1/4] *: Add sysvar for setting TSO RPC concurrency Signed-off-by: MyonKeminta --- go.mod | 1 + go.sum | 4 ++-- pkg/domain/domain_sysvars.go | 18 ++++++++++++++++++ pkg/sessionctx/variable/sysvar.go | 5 +++++ pkg/sessionctx/variable/tidb_vars.go | 4 ++++ pkg/sessionctx/variable/variable.go | 12 ++++++++++++ 6 files changed, 42 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index e8d6774a5653e..8b480d2c82d87 100644 --- a/go.mod +++ b/go.mod @@ -319,6 +319,7 @@ require ( replace ( github.com/go-ldap/ldap/v3 => github.com/YangKeao/ldap/v3 v3.4.5-0.20230421065457-369a3bab1117 github.com/pingcap/tidb/pkg/parser => ./pkg/parser + github.com/tikv/pd/client => github.com/MyonKeminta/pd/client v0.0.0-20240925045805-0bfe9a69293d // TODO: `sourcegraph.com/sourcegraph/appdash` has been archived, and the original host has been removed. // Please remove these dependencies. diff --git a/go.sum b/go.sum index bda7d03316d30..fd25f7bdf2f38 100644 --- a/go.sum +++ b/go.sum @@ -80,6 +80,8 @@ github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0 github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= github.com/Masterminds/sprig/v3 v3.2.2 h1:17jRggJu518dr3QaafizSXOjKYp94wKfABxUmyxvxX8= github.com/Masterminds/sprig/v3 v3.2.2/go.mod h1:UoaO7Yp8KlPnJIYWTFkMaqPUYKTfGFPhxNuwnnxkKlk= +github.com/MyonKeminta/pd/client v0.0.0-20240925045805-0bfe9a69293d h1:8HHuVEu7PvaTv5wx+m8RqY/X5X/7gd9q79lFb2iU17w= +github.com/MyonKeminta/pd/client v0.0.0-20240925045805-0bfe9a69293d/go.mod h1:uBHhxAM/SPCMabt483gI/pN/+JXIMKYXohK96s+PwT8= github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA= github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= @@ -824,8 +826,6 @@ github.com/tidwall/btree v1.7.0 h1:L1fkJH/AuEh5zBnnBbmTwQ5Lt+bRJ5A8EWecslvo9iI= github.com/tidwall/btree v1.7.0/go.mod h1:twD9XRA5jj9VUQGELzDO4HPQTNJsoWWfYEL+EUQ2cKY= github.com/tikv/client-go/v2 v2.0.8-0.20240911041506-e7894a7b27ba h1:dwuuYqPYxU0xcv0bnDgT7I4btQ5c3joBG1HmNOhCTdo= github.com/tikv/client-go/v2 v2.0.8-0.20240911041506-e7894a7b27ba/go.mod h1:4HDOAx8OXAJPtqhCZ03IhChXgaFs4B3+vSrPWmiPxjg= -github.com/tikv/pd/client v0.0.0-20240914083230-71f6f96816e9 h1:J9LChGMzo95eBrjE03NHITDWgxfPgskH+QrCnlW61/Y= -github.com/tikv/pd/client v0.0.0-20240914083230-71f6f96816e9/go.mod h1:uBHhxAM/SPCMabt483gI/pN/+JXIMKYXohK96s+PwT8= github.com/timakin/bodyclose v0.0.0-20240125160201-f835fa56326a h1:A6uKudFIfAEpoPdaal3aSqGxBzLyU8TqyXImLwo6dIo= github.com/timakin/bodyclose v0.0.0-20240125160201-f835fa56326a/go.mod h1:mkjARE7Yr8qU23YcGMSALbIxTQ9r9QBVahQOBRfU460= github.com/tklauser/go-sysconf v0.3.9/go.mod h1:11DU/5sG7UexIrp/O6g35hrWzu0JxlwQ3LSFUzyeuhs= diff --git a/pkg/domain/domain_sysvars.go b/pkg/domain/domain_sysvars.go index 8314cb7e4516f..22132ee08784f 100644 --- a/pkg/domain/domain_sysvars.go +++ b/pkg/domain/domain_sysvars.go @@ -83,6 +83,24 @@ func (do *Domain) setPDClientDynamicOption(name, sVal string) error { return err } variable.EnablePDFollowerHandleRegion.Store(val) + case variable.TiDBTSOClientRPCMode: + var concurrency int + + switch sVal { + case variable.TSOClientRPCModeDefault: + concurrency = 1 + case variable.TSOClientRPCModeParallel: + concurrency = 2 + case variable.TSOClientRPCModeParallelFast: + concurrency = 4 + default: + return variable.ErrWrongValueForVar.GenWithStackByArgs(name, sVal) + } + + err := do.updatePDClient(pd.TSOClientRPCConcurrency, concurrency) + if err != nil { + return err + } } return nil } diff --git a/pkg/sessionctx/variable/sysvar.go b/pkg/sessionctx/variable/sysvar.go index cc1005a50fa25..e4a45ae51c911 100644 --- a/pkg/sessionctx/variable/sysvar.go +++ b/pkg/sessionctx/variable/sysvar.go @@ -3383,6 +3383,11 @@ var defaultSysVars = []*SysVar{ s.SharedLockPromotion = TiDBOptOn(val) return nil }}, + {Scope: ScopeGlobal, Name: TiDBTSOClientRPCMode, Value: DefTiDBTSOClientRPCMode, Type: TypeEnum, PossibleValues: []string{TSOClientRPCModeDefault, TSOClientRPCModeParallel, TSOClientRPCModeParallelFast}, + SetGlobal: func(_ context.Context, s *SessionVars, val string) error { + return (*SetPDClientDynamicOption.Load())(TiDBTSOClientRPCMode, val) + }, + }, } // GlobalSystemVariableInitialValue gets the default value for a system variable including ones that are dynamically set (e.g. based on the store) diff --git a/pkg/sessionctx/variable/tidb_vars.go b/pkg/sessionctx/variable/tidb_vars.go index 77145e44f9423..07d81834274c8 100644 --- a/pkg/sessionctx/variable/tidb_vars.go +++ b/pkg/sessionctx/variable/tidb_vars.go @@ -1212,6 +1212,9 @@ const ( // TiDBEnableLazyCursorFetch defines whether to enable the lazy cursor fetch. If it's `OFF`, all results of // of a cursor will be stored in the tidb node in `EXECUTE` command. TiDBEnableLazyCursorFetch = "tidb_enable_lazy_cursor_fetch" + // TiDBTSOClientRPCMode controls how the TSO client performs the TSO RPC requests. It internally controls the + // concurrency of the RPC. This variable provides an approach to tune the latency of getting timestamps from PD. + TiDBTSOClientRPCMode = "tidb_tso_client_rpc_mode" ) // TiDB intentional limits @@ -1560,6 +1563,7 @@ const ( DefTiDBEnableLazyCursorFetch = false DefOptEnableProjectionPushDown = true DefTiDBEnableSharedLockPromotion = false + DefTiDBTSOClientRPCMode = TSOClientRPCModeDefault ) // Process global variables. diff --git a/pkg/sessionctx/variable/variable.go b/pkg/sessionctx/variable/variable.go index 5eb94f379a0f4..aceb862f8f77b 100644 --- a/pkg/sessionctx/variable/variable.go +++ b/pkg/sessionctx/variable/variable.go @@ -83,6 +83,18 @@ const ( OOMActionCancel = "CANCEL" // OOMActionLog constants represents the valid action configurations for OOMAction "LOG". OOMActionLog = "LOG" + + // TSOClientRPCModeDefault is a choice of variable TiDBTSOClientRPCMode. In this mode, the TSO client sends batched + // TSO requests serially. + TSOClientRPCModeDefault = "DEFAULT" + // TSOClientRPCModeParallel is a choice of variable TiDBTSOClientRPCMode. In this mode, the TSO client tries to + // keep approximately 2 batched TSO requests running in parallel. This option tries to reduce the batch-waiting time + // by half, at the expense of about twice the amount of TSO RPC calls. + TSOClientRPCModeParallel = "PARALLEL" + // TSOClientRPCModeParallelFast is a choice of variable TiDBTSOClientRPCMode. In this mode, the TSO client tries to + // keep approximately 4 batched TSO requests running in parallel. This option tries to reduce the batch-waiting time + // by 3/4, at the expense of about 4 times the amount of TSO RPC calls. + TSOClientRPCModeParallelFast = "PARALLEL-FAST" ) // Global config name list. From e8f64c1f0085a46e98352ead97d9868b1aee4546 Mon Sep 17 00:00:00 2001 From: MyonKeminta Date: Wed, 25 Sep 2024 18:13:27 +0800 Subject: [PATCH 2/4] add new metrics about tso to grafana Signed-off-by: MyonKeminta --- pkg/metrics/grafana/tidb.json | 115 ++++++++++++++++++++++++++++++++++ 1 file changed, 115 insertions(+) diff --git a/pkg/metrics/grafana/tidb.json b/pkg/metrics/grafana/tidb.json index 9bda9d2fd5f1e..e6dbd6c2570b9 100644 --- a/pkg/metrics/grafana/tidb.json +++ b/pkg/metrics/grafana/tidb.json @@ -12233,6 +12233,13 @@ "intervalFactor": 2, "legendFormat": "90", "refId": "C" + }, + { + "expr": "sum(rate(pd_client_cmd_handle_cmds_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"wait\"}[1m])) / sum(rate(pd_client_cmd_handle_cmds_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"wait\"}[1m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "avg", + "refId": "D" } ], "thresholds": [], @@ -12348,6 +12355,13 @@ "intervalFactor": 2, "legendFormat": "99", "refId": "C" + }, + { + "expr": "sum(rate(pd_client_request_handle_requests_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"tso\"}[1m])) / sum(rate(pd_client_request_handle_requests_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"tso\"}[1m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "avg", + "refId": "D" } ], "thresholds": [], @@ -12392,6 +12406,107 @@ "alignLevel": null } }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "description": "The estimated latency of TSO RPC calls that's used to adjust batching time for parallel RPC requests", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 23 + }, + "hiddenSeries": false, + "id": 23763572011, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.17", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "pd_client_request_estimate_tso_latency{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}-{{stream}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Estimate TSO RTT Latency", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, { "aliasColors": {}, "bars": false, From f1d4d9937a29a38af6b7145b87b252ab75b62bc0 Mon Sep 17 00:00:00 2001 From: MyonKeminta Date: Thu, 26 Sep 2024 11:51:28 +0800 Subject: [PATCH 3/4] Update pd client to master Signed-off-by: MyonKeminta --- go.mod | 3 +-- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 8b480d2c82d87..4a1455553970f 100644 --- a/go.mod +++ b/go.mod @@ -109,7 +109,7 @@ require ( github.com/tiancaiamao/appdash v0.0.0-20181126055449-889f96f722a2 github.com/tidwall/btree v1.7.0 github.com/tikv/client-go/v2 v2.0.8-0.20240911041506-e7894a7b27ba - github.com/tikv/pd/client v0.0.0-20240914083230-71f6f96816e9 + github.com/tikv/pd/client v0.0.0-20240926021936-642f0e919b0d github.com/timakin/bodyclose v0.0.0-20240125160201-f835fa56326a github.com/twmb/murmur3 v1.1.6 github.com/uber/jaeger-client-go v2.22.1+incompatible @@ -319,7 +319,6 @@ require ( replace ( github.com/go-ldap/ldap/v3 => github.com/YangKeao/ldap/v3 v3.4.5-0.20230421065457-369a3bab1117 github.com/pingcap/tidb/pkg/parser => ./pkg/parser - github.com/tikv/pd/client => github.com/MyonKeminta/pd/client v0.0.0-20240925045805-0bfe9a69293d // TODO: `sourcegraph.com/sourcegraph/appdash` has been archived, and the original host has been removed. // Please remove these dependencies. diff --git a/go.sum b/go.sum index fd25f7bdf2f38..5009f79d58cf1 100644 --- a/go.sum +++ b/go.sum @@ -80,8 +80,6 @@ github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0 github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= github.com/Masterminds/sprig/v3 v3.2.2 h1:17jRggJu518dr3QaafizSXOjKYp94wKfABxUmyxvxX8= github.com/Masterminds/sprig/v3 v3.2.2/go.mod h1:UoaO7Yp8KlPnJIYWTFkMaqPUYKTfGFPhxNuwnnxkKlk= -github.com/MyonKeminta/pd/client v0.0.0-20240925045805-0bfe9a69293d h1:8HHuVEu7PvaTv5wx+m8RqY/X5X/7gd9q79lFb2iU17w= -github.com/MyonKeminta/pd/client v0.0.0-20240925045805-0bfe9a69293d/go.mod h1:uBHhxAM/SPCMabt483gI/pN/+JXIMKYXohK96s+PwT8= github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA= github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= @@ -826,6 +824,8 @@ github.com/tidwall/btree v1.7.0 h1:L1fkJH/AuEh5zBnnBbmTwQ5Lt+bRJ5A8EWecslvo9iI= github.com/tidwall/btree v1.7.0/go.mod h1:twD9XRA5jj9VUQGELzDO4HPQTNJsoWWfYEL+EUQ2cKY= github.com/tikv/client-go/v2 v2.0.8-0.20240911041506-e7894a7b27ba h1:dwuuYqPYxU0xcv0bnDgT7I4btQ5c3joBG1HmNOhCTdo= github.com/tikv/client-go/v2 v2.0.8-0.20240911041506-e7894a7b27ba/go.mod h1:4HDOAx8OXAJPtqhCZ03IhChXgaFs4B3+vSrPWmiPxjg= +github.com/tikv/pd/client v0.0.0-20240926021936-642f0e919b0d h1:iBIi3+grJWLNI5c7BLVezgOonYtL/EQqSUDsPVK2kUw= +github.com/tikv/pd/client v0.0.0-20240926021936-642f0e919b0d/go.mod h1:uBHhxAM/SPCMabt483gI/pN/+JXIMKYXohK96s+PwT8= github.com/timakin/bodyclose v0.0.0-20240125160201-f835fa56326a h1:A6uKudFIfAEpoPdaal3aSqGxBzLyU8TqyXImLwo6dIo= github.com/timakin/bodyclose v0.0.0-20240125160201-f835fa56326a/go.mod h1:mkjARE7Yr8qU23YcGMSALbIxTQ9r9QBVahQOBRfU460= github.com/tklauser/go-sysconf v0.3.9/go.mod h1:11DU/5sG7UexIrp/O6g35hrWzu0JxlwQ3LSFUzyeuhs= From 75ef18ecab68a7dc6a5eb8d3fa5c76d59e8330fb Mon Sep 17 00:00:00 2001 From: MyonKeminta Date: Thu, 26 Sep 2024 12:02:29 +0800 Subject: [PATCH 4/4] update bazel Signed-off-by: MyonKeminta --- DEPS.bzl | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/DEPS.bzl b/DEPS.bzl index ad65258a3edb5..f7096a6060107 100644 --- a/DEPS.bzl +++ b/DEPS.bzl @@ -7024,13 +7024,13 @@ def go_deps(): name = "com_github_tikv_pd_client", build_file_proto_mode = "disable_global", importpath = "github.com/tikv/pd/client", - sha256 = "fc2042d3b3c753de90ac2afdfea97b663f3043aa716f1ee56d2fdf98864e3cbd", - strip_prefix = "github.com/tikv/pd/client@v0.0.0-20240914083230-71f6f96816e9", + sha256 = "ddfcb88a8b79c238b08c15c88f8142211cb9f64435c5ad371f682d8c81c8cc42", + strip_prefix = "github.com/tikv/pd/client@v0.0.0-20240926021936-642f0e919b0d", urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/github.com/tikv/pd/client/com_github_tikv_pd_client-v0.0.0-20240914083230-71f6f96816e9.zip", - "http://ats.apps.svc/gomod/github.com/tikv/pd/client/com_github_tikv_pd_client-v0.0.0-20240914083230-71f6f96816e9.zip", - "https://cache.hawkingrei.com/gomod/github.com/tikv/pd/client/com_github_tikv_pd_client-v0.0.0-20240914083230-71f6f96816e9.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/github.com/tikv/pd/client/com_github_tikv_pd_client-v0.0.0-20240914083230-71f6f96816e9.zip", + "http://bazel-cache.pingcap.net:8080/gomod/github.com/tikv/pd/client/com_github_tikv_pd_client-v0.0.0-20240926021936-642f0e919b0d.zip", + "http://ats.apps.svc/gomod/github.com/tikv/pd/client/com_github_tikv_pd_client-v0.0.0-20240926021936-642f0e919b0d.zip", + "https://cache.hawkingrei.com/gomod/github.com/tikv/pd/client/com_github_tikv_pd_client-v0.0.0-20240926021936-642f0e919b0d.zip", + "https://storage.googleapis.com/pingcapmirror/gomod/github.com/tikv/pd/client/com_github_tikv_pd_client-v0.0.0-20240926021936-642f0e919b0d.zip", ], ) go_repository(