diff --git a/pkg/controller/registry/resolver/cache/cache.go b/pkg/controller/registry/resolver/cache/cache.go index f9b9e03a94..7db59bbc52 100644 --- a/pkg/controller/registry/resolver/cache/cache.go +++ b/pkg/controller/registry/resolver/cache/cache.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "io" + "slices" "sort" "sync" "time" @@ -147,7 +148,7 @@ func (c *NamespacedOperatorCache) Error() error { func (c *Cache) Namespaced(namespaces ...string) MultiCatalogOperatorFinder { const ( - CachePopulateTimeout = time.Minute + cachePopulateTimeout = time.Minute ) sources := c.sp.Sources(namespaces...) @@ -209,19 +210,38 @@ func (c *Cache) Namespaced(namespaces ...string) MultiCatalogOperatorFinder { } misses = misses[found:] + // remove any with a "live" outstanding request + misses = slices.DeleteFunc(misses, func(key SourceKey) bool { + fl := c.logger.(logrus.FieldLogger) + fl.Errorf(">>> checking for outstanding request for %s", key) + + hdr := c.snapshots[key] + + // if we already have a request timestamp, we have an outstanding request, so prevent stacking + // and just send new requests if the previous one has expired + if hdr != nil && hdr.RequestSentinelActive() { + fl.Errorf("skipping new request for %s, already in progress", key) + return true + } + return false + }) + for _, miss := range misses { - ctx, cancel := context.WithTimeout(context.Background(), CachePopulateTimeout) + ctx, cancel := context.WithTimeout(context.Background(), cachePopulateTimeout) hdr := snapshotHeader{ - key: miss, - pop: cancel, - priority: c.sourcePriorityProvider.Priority(miss), + key: miss, + pop: cancel, + priority: c.sourcePriorityProvider.Priority(miss), + requestSentinel: time.Now().Add(cachePopulateTimeout), // set sentinel to prevent stacking requests } hdr.m.Lock() c.snapshots[miss] = &hdr result.snapshots[miss] = &hdr + // don't adjust the request sentinel in the goroutine for any outcome, so that we don't stampede sources + // instead, reevaluate the sentinel during the next snapshot go func(ctx context.Context, hdr *snapshotHeader, source Source) { defer hdr.m.Unlock() c.sem <- struct{}{} @@ -294,6 +314,8 @@ type snapshotHeader struct { pop context.CancelFunc err error priority int + + requestSentinel time.Time } func (hdr *snapshotHeader) Cancel() { @@ -314,6 +336,13 @@ func (hdr *snapshotHeader) Valid() bool { return true } +func (hdr *snapshotHeader) RequestSentinelActive() bool { + hdr.m.RLock() + defer hdr.m.RUnlock() + + return time.Now().Before(hdr.requestSentinel) +} + type sortableSnapshots struct { snapshots []*snapshotHeader preferredNamespace string diff --git a/test/e2e/util.go b/test/e2e/util.go index 1241113bec..0db1541315 100644 --- a/test/e2e/util.go +++ b/test/e2e/util.go @@ -51,7 +51,7 @@ import ( const ( pollInterval = 100 * time.Millisecond - pollDuration = 5 * time.Minute + pollDuration = 15 * time.Minute olmConfigMap = "olm-operators" // No-longer used, how long do we keep this around?