Skip to content

Commit

Permalink
ROX-0000: Fix mutex taking more than 5s to unlock (stackrox#7134)
Browse files Browse the repository at this point in the history
  • Loading branch information
fredrb authored Jul 27, 2023
1 parent d9337ee commit 31a2eca
Show file tree
Hide file tree
Showing 3 changed files with 12 additions and 7 deletions.
2 changes: 2 additions & 0 deletions sensor/kubernetes/listener/listener.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,9 @@ func New(client client.Interface, configHandler config.Handler, nodeName string,
traceWriter: traceWriter,
outputQueue: queue,
storeProvider: storeProvider,
mayCreateHandlers: concurrency.NewSignal(),
}
k.mayCreateHandlers.Signal()
return k
}

Expand Down
13 changes: 9 additions & 4 deletions sensor/kubernetes/listener/listener_impl.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@ import (

"github.com/stackrox/rox/pkg/buildinfo"
"github.com/stackrox/rox/pkg/concurrency"
"github.com/stackrox/rox/pkg/sync"
"github.com/stackrox/rox/sensor/common/awscredentials"
"github.com/stackrox/rox/sensor/common/config"
"github.com/stackrox/rox/sensor/kubernetes/client"
Expand All @@ -33,13 +32,19 @@ type listenerImpl struct {
traceWriter io.Writer
outputQueue component.Resolver
storeProvider *resources.InMemoryStoreProvider
mayCreateHandlers concurrency.Signal
context context.Context
contextMtx sync.Mutex
}

func (k *listenerImpl) StartWithContext(ctx context.Context) error {
k.contextMtx.Lock()
defer k.contextMtx.Unlock()
// There is a caveat here that we need to make sure that the previous Start has already
// finished before swap the context, otherwise there is a risk of data racing by swapping
// the context while its being used to create the handlers.
// Since the handleAllEvents function takes too long to run, using mutex is a problem in
// dev environments, since the mutex will be locked to more than 5s, resulting in a panic.
// The current workaround is to use a signal instead of a mutex.
k.mayCreateHandlers.Wait()
k.mayCreateHandlers.Reset()
k.context = ctx
return k.Start()
}
Expand Down
4 changes: 1 addition & 3 deletions sensor/kubernetes/listener/resource_event_handler.go
Original file line number Diff line number Diff line change
Expand Up @@ -51,9 +51,7 @@ func managedFieldsTransformer(obj interface{}) (interface{}, error) {
}

func (k *listenerImpl) handleAllEvents() {
k.contextMtx.Lock()
defer k.contextMtx.Unlock()

defer k.mayCreateHandlers.Signal()
// TODO(ROX-14194): remove resyncingSif once all resources are adapted
var resyncingSif informers.SharedInformerFactory
if env.ResyncDisabled.BooleanSetting() {
Expand Down

0 comments on commit 31a2eca

Please sign in to comment.