Add fail-close mechanism for prohibited caches and update tests accor…

…dingly

director/advertise_test.go

@@ -223,6 +223,9 @@ func TestAdvertiseOSDF(t *testing.T) {
 		require.NotNil(t, foundServer.NamespaceAds)
 		assert.True(t, foundServer.NamespaceAds[0].FromTopology)
 
+		// Override the check for prohibited caches
+		prohibitedCachesLastSetTimestamp.Store(time.Now().Unix())
+
 		// Test a few values. If they're correct, it indicates the whole process likely succeeded
 		nsAd, oAds, cAds := getAdsForPath("/my/server/path/to/file")
 		assert.Equal(t, "/my/server", nsAd.Path)

director/cache_ads.go

@@ -311,6 +311,22 @@ func isProhibited(nsAd *server_structs.NamespaceAdV2, ad *server_structs.Adverti
 	if ad.Type == server_structs.OriginType.String() {
 		return false
 	}
+	if prohibitedCachesLastSetTimestamp.Load() == 0 {
+		log.Warning("Prohibited caches data is not set, waiting for it to be set before continuing with cache matchmaking")
+		start := time.Now()
+		// Wait until last set timestamp is updated
+		for prohibitedCachesLastSetTimestamp.Load() == 0 {
+			if time.Since(start) >= 3*time.Second {
+				log.Error("Prohibited caches data was not set within the 3-second timeout")
+				break
+			}
+			time.Sleep(100 * time.Millisecond)
+		}
+	}
+	if time.Since(time.Unix(prohibitedCachesLastSetTimestamp.Load(), 0)) >= 15*time.Minute {
+		log.Error("Prohibited caches data is outdated, caches will not be used.")
+		return true
+	}
 
 	serverHost := ad.ServerAd.URL.Host
 	serverHostname := strings.Split(serverHost, ":")[0]

director/cache_ads_test.go

@@ -159,6 +159,9 @@ func TestGetAdsForPath(t *testing.T) {
 	recordAd(context.Background(), cacheAd1, &c1Slice)
 	recordAd(context.Background(), cacheAd2, &o1Slice)
 
+	// Override the check for prohibited caches
+	prohibitedCachesLastSetTimestamp.Store(time.Now().Unix())
+
 	// If /chtc is served both from topology and Pelican, the Topology server/namespace should be ignored
 	nsAd, oAds, cAds := getAdsForPath("/chtc")
 	assert.Equal(t, "/chtc", nsAd.Path)

director/prohibited_caches.go

@@ -108,9 +108,9 @@ func LaunchPeriodicProhibitedCachesFetch(ctx context.Context, egrp *errgroup.Gro
 	// Initial fetch of prohibited caches
 	data, err := fetchProhibitedCaches(ctx)
 	if err != nil {
-		ticker.Reset(10 * time.Second) // Higher frequency (10s)
+		ticker.Reset(1 * time.Second) // Higher frequency (10s)
 		log.Warningf("Error fetching prohibited caches on first attempt: %v", err)
-		log.Debug("Switching to higher frequency (10s) for prohibited caches fetch")
+		log.Debug("Switching to higher frequency (1s) for prohibited caches fetch")
 	} else {
 		prohibitedCaches.Store(&data)
 		prohibitedCachesLastSetTimestamp.Store(time.Now().Unix())
@@ -130,7 +130,7 @@ func LaunchPeriodicProhibitedCachesFetch(ctx context.Context, egrp *errgroup.Gro
 					lastSet := prohibitedCachesLastSetTimestamp.Load()
 					if time.Since(time.Unix(lastSet, 0)) >= 15*time.Minute {
 						log.Debug("Prohibited caches last updated over 15 minutes ago, switching to higher frequency")
-						ticker.Reset(5 * time.Second) // Higher frequency (10s)
+						ticker.Reset(1 * time.Second) // Higher frequency (1s)
 					}
 					continue
 				}