From 14135bda349b35d53cdbf826e8d198dd1f598373 Mon Sep 17 00:00:00 2001 From: ShuNing Date: Sun, 7 Apr 2024 19:15:19 +0800 Subject: [PATCH 1/2] This is an automated cherry-pick of #8028 ref tikv/pd#7509 Signed-off-by: ti-chi-bot --- client/errs/errno.go | 2 +- client/resource_group/controller/limiter.go | 37 +++++++++++++------ .../resource_group/controller/limiter_test.go | 1 + 3 files changed, 28 insertions(+), 12 deletions(-) diff --git a/client/errs/errno.go b/client/errs/errno.go index 0f93ebf1472..9f55a7b2f6a 100644 --- a/client/errs/errno.go +++ b/client/errs/errno.go @@ -89,7 +89,7 @@ var ( var ( ErrClientListResourceGroup = errors.Normalize("get all resource group failed, %v", errors.RFCCodeText("PD:client:ErrClientListResourceGroup")) ErrClientResourceGroupConfigUnavailable = errors.Normalize("resource group config is unavailable, %v", errors.RFCCodeText("PD:client:ErrClientResourceGroupConfigUnavailable")) - ErrClientResourceGroupThrottled = errors.Normalize("exceeded resource group quota limitation", errors.RFCCodeText("PD:client:ErrClientResourceGroupThrottled")) + ErrClientResourceGroupThrottled = errors.Normalize("exceeded resource group quota limitation, estimated wait time %s, ltb state is %.2f:%.2f", errors.RFCCodeText("PD:client:ErrClientResourceGroupThrottled")) ) // ErrClientGetResourceGroup is the error type for getting resource group. diff --git a/client/resource_group/controller/limiter.go b/client/resource_group/controller/limiter.go index b2e2a03de70..0159371b710 100644 --- a/client/resource_group/controller/limiter.go +++ b/client/resource_group/controller/limiter.go @@ -136,13 +136,14 @@ func NewLimiterWithCfg(name string, now time.Time, cfg tokenBucketReconfigureArg // A Reservation holds information about events that are permitted by a Limiter to happen after a delay. // A Reservation may be canceled, which may enable the Limiter to permit additional events. type Reservation struct { - ok bool - lim *Limiter - tokens float64 - timeToAct time.Time - needWaitDurtion time.Duration + ok bool + lim *Limiter + tokens float64 + timeToAct time.Time + needWaitDuration time.Duration // This is the Limit at reservation time, it can change later. - limit Limit + limit Limit + remainingTokens float64 } // OK returns whether the limiter can provide the requested number of tokens @@ -386,10 +387,11 @@ func (lim *Limiter) reserveN(now time.Time, n float64, maxFutureReserve time.Dur // Prepare reservation r := Reservation{ - ok: ok, - lim: lim, - limit: lim.limit, - needWaitDurtion: waitDuration, + ok: ok, + lim: lim, + limit: lim.limit, + needWaitDuration: waitDuration, + remainingTokens: tokens, } if ok { r.tokens = n @@ -401,6 +403,7 @@ func (lim *Limiter) reserveN(now time.Time, n float64, maxFutureReserve time.Dur lim.tokens = tokens lim.maybeNotify() } else { +<<<<<<< HEAD // print log if the limiter cannot reserve for a while. if time.Since(lim.last) > reserveWarnLogInterval { log.Warn("[resource group controller] cannot reserve enough tokens", @@ -414,6 +417,18 @@ func (lim *Limiter) reserveN(now time.Time, n float64, maxFutureReserve time.Dur zap.Int64("burst", lim.burst), zap.Int("remaining-notify-times", lim.remainingNotifyTimes)) } +======= + log.Warn("[resource group controller] cannot reserve enough tokens", + zap.Duration("need-wait-duration", waitDuration), + zap.Duration("max-wait-duration", maxFutureReserve), + zap.Float64("current-ltb-tokens", lim.tokens), + zap.Float64("current-ltb-rate", float64(lim.limit)), + zap.Float64("request-tokens", n), + zap.Float64("notify-threshold", lim.notifyThreshold), + zap.Bool("is-low-process", lim.isLowProcess), + zap.Int64("burst", lim.burst), + zap.Int("remaining-notify-times", lim.remainingNotifyTimes)) +>>>>>>> cbde63645 (client/controller: add logs and export more information in throttled error (#8028)) lim.last = last if lim.limit == 0 { lim.notify() @@ -493,7 +508,7 @@ func WaitReservations(ctx context.Context, now time.Time, reservations []*Reserv for _, res := range reservations { if !res.ok { cancel() - return res.needWaitDurtion, errs.ErrClientResourceGroupThrottled + return res.needWaitDuration, errs.ErrClientResourceGroupThrottled.FastGenByArgs(res.needWaitDuration, res.limit, res.remainingTokens) } delay := res.DelayFrom(now) if delay > longestDelayDuration { diff --git a/client/resource_group/controller/limiter_test.go b/client/resource_group/controller/limiter_test.go index d963f830551..e573ca4f84c 100644 --- a/client/resource_group/controller/limiter_test.go +++ b/client/resource_group/controller/limiter_test.go @@ -163,6 +163,7 @@ func TestCancel(t *testing.T) { d, err := WaitReservations(ctx, t2, []*Reservation{r1, r2}) re.Equal(d, 4*time.Second) re.Error(err) + re.Contains(err.Error(), "estimated wait time 4s, ltb state is 1.00:-4.00") checkTokens(re, lim1, t3, 13) checkTokens(re, lim2, t3, 3) cancel1() From 4cf05623a3913445cd74e85ab59dcd188c75ce5c Mon Sep 17 00:00:00 2001 From: nolouch Date: Wed, 5 Jun 2024 16:38:54 +0800 Subject: [PATCH 2/2] fix Signed-off-by: nolouch --- client/resource_group/controller/limiter.go | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/client/resource_group/controller/limiter.go b/client/resource_group/controller/limiter.go index 0159371b710..e8c50425bdc 100644 --- a/client/resource_group/controller/limiter.go +++ b/client/resource_group/controller/limiter.go @@ -403,7 +403,6 @@ func (lim *Limiter) reserveN(now time.Time, n float64, maxFutureReserve time.Dur lim.tokens = tokens lim.maybeNotify() } else { -<<<<<<< HEAD // print log if the limiter cannot reserve for a while. if time.Since(lim.last) > reserveWarnLogInterval { log.Warn("[resource group controller] cannot reserve enough tokens", @@ -417,18 +416,6 @@ func (lim *Limiter) reserveN(now time.Time, n float64, maxFutureReserve time.Dur zap.Int64("burst", lim.burst), zap.Int("remaining-notify-times", lim.remainingNotifyTimes)) } -======= - log.Warn("[resource group controller] cannot reserve enough tokens", - zap.Duration("need-wait-duration", waitDuration), - zap.Duration("max-wait-duration", maxFutureReserve), - zap.Float64("current-ltb-tokens", lim.tokens), - zap.Float64("current-ltb-rate", float64(lim.limit)), - zap.Float64("request-tokens", n), - zap.Float64("notify-threshold", lim.notifyThreshold), - zap.Bool("is-low-process", lim.isLowProcess), - zap.Int64("burst", lim.burst), - zap.Int("remaining-notify-times", lim.remainingNotifyTimes)) ->>>>>>> cbde63645 (client/controller: add logs and export more information in throttled error (#8028)) lim.last = last if lim.limit == 0 { lim.notify()