Fix load shedding

databricks · Dec 19, 2024 · 7b7a5da · 7b7a5da
1 parent 95cb360
commit 7b7a5da
Show file tree

Hide file tree

Showing 2 changed files with 9 additions and 4 deletions.
diff --git a/pkg/receive/limiter.go b/pkg/receive/limiter.go
@@ -77,20 +77,20 @@ func (l *Limiter) HeadSeriesLimiter() headSeriesLimiter {
 	return l.headSeriesLimiter
 }
 
-func (l *Limiter) ShouldRejectNewRequest() bool {
+func (l *Limiter) ShouldRejectNewRequest() (bool, string) {
 	// maxPendingRequests doesn't change once set when a limiter instance is created.
 	// So, it's safe to read it without a lock.
 	if l.maxPendingRequests > 0 && l.pendingRequests.Load() >= l.maxPendingRequests {
 		if l.maxPendingRequestLimitHit != nil {
 			l.maxPendingRequestLimitHit.Inc()
 		}
-		return true
+		return true, fmt.Sprintf("too many pending write requests: %d >= %d", l.pendingRequests.Load(), l.maxPendingRequests)
 	}
 	newValue := l.pendingRequests.Add(1)
 	if l.pendingRequestsGauge != nil {
 		l.pendingRequestsGauge.Set(float64(newValue))
 	}
-	return false
+	return false, ""
 }
 
 func (l *Limiter) DecrementPendingRequests() {

diff --git a/pkg/store/limiter.go b/pkg/store/limiter.go
@@ -4,6 +4,7 @@
 package store
 
 import (
+	"fmt"
 	"sync"
 
 	"github.com/alecthomas/units"
@@ -161,7 +162,11 @@ func NewLimitedStoreServer(store storepb.StoreServer, reg prometheus.Registerer,
 
 func (s *limitedStoreServer) Series(req *storepb.SeriesRequest, srv storepb.Store_SeriesServer) error {
 	if s.maxPendingRequests > 0 && s.pendingRequests.Load() >= s.maxPendingRequests {
-		return status.Error(codes.ResourceExhausted, "too many pending series requests")
+		s.maxPendingRequestLimitHit.Inc()
+		return status.Error(
+			codes.ResourceExhausted,
+			fmt.Sprintf("too many pending series requests: %d >= %d", s.pendingRequests.Load(), s.maxPendingRequests),
+		)
 	}
 	s.pendingRequestsGauge.Set(float64(s.pendingRequests.Add(1)))
 	defer s.pendingRequests.Add(-1)