Merge #131226

131226: pkg/kv/kvclient: update per-vCPU concurrency limits r=sean- a=sean-

Increase the current values (arbitrarily selected) by 6x (also an
arbitrary increase).  Annecdotal evidence from customers suggests
this may unlock some workloads where vCPU utilization appears to
"plateau" around 70%, and now allows for ~100% vCPU utilization.
Until #131125 and #131126 are implemented, it is not possible to
directly monitor these incremental improvements to the defaults.

Epic: CRDB-42388
Epic: CRDB-42389

Release note (performance improvement): Increase the per-vCPU
concurrency limits for KV operations.  Specifically, increase
`kv.dist_sender.concurrency_limit` to 384/vCPU (up from 64/vCPU)
and `kv.streamer.concurrency_limit` to 96/vCPU (up from 8/vCPU).

Co-authored-by: Sean Chittenden <[email protected]>

pkg/kv/kvclient/kvcoord/dist_sender.go

@@ -343,8 +343,10 @@ var CanSendToFollower = func(
 }
 
 const (
-	// The default limit for asynchronous senders.
-	defaultSenderConcurrency = 1024
+	// The default scaling factor for the number of async ops per vCPU.
+	DefaultSenderStreamsPerVCPU = 384
+	// The minimum number of recommended vCPUs.
+	MinViableProcs = 2
 	// RangeLookupPrefetchCount is the maximum number of range descriptors to prefetch
 	// during range lookups.
 	RangeLookupPrefetchCount = 8
@@ -368,7 +370,7 @@ var senderConcurrencyLimit = settings.RegisterIntSetting(
 	settings.ApplicationLevel,
 	"kv.dist_sender.concurrency_limit",
 	"maximum number of asynchronous send requests",
-	max(defaultSenderConcurrency, int64(64*runtime.GOMAXPROCS(0))),
+	DefaultSenderStreamsPerVCPU*max(MinViableProcs, int64(runtime.GOMAXPROCS(0))),
 	settings.NonNegativeInt,
 )
 

pkg/kv/kvclient/kvstreamer/streamer.go

@@ -340,15 +340,20 @@ type streamerStatistics struct {
 	enqueuedSingleRangeRequests int
 }
 
+const (
+	// The default scaling factor for the number of asynchronous requests per
+	// Streamer per vCPU. The value for this setting is chosen arbitrarily as
+	// 1/4th of the default value for the DefaultSenderStreamsPerVCPU.
+	defaultStreamerStreamsPerVCPU = kvcoord.DefaultSenderStreamsPerVCPU / 4
+)
+
 // streamerConcurrencyLimit is an upper bound on the number of asynchronous
-// requests that a single Streamer can have in flight. The default value for
-// this setting is chosen arbitrarily as 1/8th of the default value for the
-// senderConcurrencyLimit.
+// requests that a single Streamer can have in flight.
 var streamerConcurrencyLimit = settings.RegisterIntSetting(
 	settings.ApplicationLevel,
 	"kv.streamer.concurrency_limit",
 	"maximum number of asynchronous requests by a single streamer",
-	max(128, int64(8*runtime.GOMAXPROCS(0))),
+	defaultStreamerStreamsPerVCPU*max(kvcoord.MinViableProcs, int64(runtime.GOMAXPROCS(0))),
 	settings.PositiveInt,
 )