From 599a8a9dce25069e06d0724d360e5e3937577e76 Mon Sep 17 00:00:00 2001 From: Sean Chittenden Date: Mon, 23 Sep 2024 10:56:10 -0700 Subject: [PATCH] pkg/kv/kvclient: update per-vCPU concurrency limits Increase the current values (arbitrarily selected) by 6x (also an arbitrary increase). Annecdotal evidence from customers suggests this may unlock some workloads where vCPU utilization appears to "plateau" around 70%, and now allows for ~100% vCPU utilization. Until #131125 and #131126 are implemented, it is not possible to directly monitor these incremental improvements to the defaults. Epic: CRDB-42388 Epic: CRDB-42389 Release note (performance improvement): Increase the per-vCPU concurrency limits for KV operations. Specifically, increase `kv.dist_sender.concurrency_limit` to 384/vCPU (up from 64/vCPU) and `kv.streamer.concurrency_limit` to 96/vCPU (up from 8/vCPU). --- pkg/kv/kvclient/kvcoord/dist_sender.go | 8 +++++--- pkg/kv/kvclient/kvstreamer/streamer.go | 13 +++++++++---- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/pkg/kv/kvclient/kvcoord/dist_sender.go b/pkg/kv/kvclient/kvcoord/dist_sender.go index c241404f61af..f258ef549b81 100644 --- a/pkg/kv/kvclient/kvcoord/dist_sender.go +++ b/pkg/kv/kvclient/kvcoord/dist_sender.go @@ -343,8 +343,10 @@ var CanSendToFollower = func( } const ( - // The default limit for asynchronous senders. - defaultSenderConcurrency = 1024 + // The default scaling factor for the number of async ops per vCPU. + DefaultSenderStreamsPerVCPU = 384 + // The minimum number of recommended vCPUs. + MinViableProcs = 2 // RangeLookupPrefetchCount is the maximum number of range descriptors to prefetch // during range lookups. RangeLookupPrefetchCount = 8 @@ -368,7 +370,7 @@ var senderConcurrencyLimit = settings.RegisterIntSetting( settings.ApplicationLevel, "kv.dist_sender.concurrency_limit", "maximum number of asynchronous send requests", - max(defaultSenderConcurrency, int64(64*runtime.GOMAXPROCS(0))), + DefaultSenderStreamsPerVCPU*max(MinViableProcs, int64(runtime.GOMAXPROCS(0))), settings.NonNegativeInt, ) diff --git a/pkg/kv/kvclient/kvstreamer/streamer.go b/pkg/kv/kvclient/kvstreamer/streamer.go index 17bc87d60788..9b5df0c9a0e8 100644 --- a/pkg/kv/kvclient/kvstreamer/streamer.go +++ b/pkg/kv/kvclient/kvstreamer/streamer.go @@ -340,15 +340,20 @@ type streamerStatistics struct { enqueuedSingleRangeRequests int } +const ( + // The default scaling factor for the number of asynchronous requests per + // Streamer per vCPU. The value for this setting is chosen arbitrarily as + // 1/4th of the default value for the DefaultSenderStreamsPerVCPU. + defaultStreamerStreamsPerVCPU = kvcoord.DefaultSenderStreamsPerVCPU / 4 +) + // streamerConcurrencyLimit is an upper bound on the number of asynchronous -// requests that a single Streamer can have in flight. The default value for -// this setting is chosen arbitrarily as 1/8th of the default value for the -// senderConcurrencyLimit. +// requests that a single Streamer can have in flight. var streamerConcurrencyLimit = settings.RegisterIntSetting( settings.ApplicationLevel, "kv.streamer.concurrency_limit", "maximum number of asynchronous requests by a single streamer", - max(128, int64(8*runtime.GOMAXPROCS(0))), + defaultStreamerStreamsPerVCPU*max(kvcoord.MinViableProcs, int64(runtime.GOMAXPROCS(0))), settings.PositiveInt, )