From 2eca0bc31acde41b3ededae0fa885c76ff2e53cf Mon Sep 17 00:00:00 2001 From: Austen McClernon Date: Thu, 19 Sep 2024 12:44:58 -0400 Subject: [PATCH] admission: lock work queue before reading waiting length When replicated work is submitted for admission, it returns early and admission proceeds asynchronously to the caller. When `V(1)` is enabled, we also log the current queue length in this code path, which is protected by a mutex that wasn't acquired previously. Acquire the queue mutex when `V(1)` is enabled to prevent a race. Part of: #130187 Release note: None --- pkg/util/admission/work_queue.go | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/pkg/util/admission/work_queue.go b/pkg/util/admission/work_queue.go index 5459d7a45d33..df5fa7cbad20 100644 --- a/pkg/util/admission/work_queue.go +++ b/pkg/util/admission/work_queue.go @@ -759,9 +759,12 @@ func (q *WorkQueue) Admit(ctx context.Context, info WorkInfo) (enabled bool, err q.metrics.recordStartWait(info.Priority) if info.ReplicatedWorkInfo.Enabled { if log.V(1) { + q.mu.Lock() + queueLen := tenant.waitingWorkHeap.Len() + q.mu.Unlock() + log.Infof(ctx, "async-path: len(waiting-work)=%d: enqueued t%d pri=%s r%s origin=n%s log-position=%s ingested=%t", - tenant.waitingWorkHeap.Len(), - tenant.id, info.Priority, + queueLen, tenant.id, info.Priority, info.ReplicatedWorkInfo.RangeID, info.ReplicatedWorkInfo.Origin, info.ReplicatedWorkInfo.LogPosition, @@ -919,9 +922,12 @@ func (q *WorkQueue) granted(grantChainID grantChainID) int64 { // NB: We don't use grant chains for store tokens, so they don't apply // to replicated writes. if log.V(1) { + q.mu.Lock() + queueLen := tenant.waitingWorkHeap.Len() + q.mu.Unlock() + log.Infof(q.ambientCtx, "async-path: len(waiting-work)=%d dequeued t%d pri=%s r%s origin=n%s log-position=%s ingested=%t", - tenant.waitingWorkHeap.Len(), - tenant.id, item.priority, + queueLen, tenant.id, item.priority, item.replicated.RangeID, item.replicated.Origin, item.replicated.LogPosition,