Folder metadata in stat cache

internal/cache/metadata/stat_cache.go

@@ -18,6 +18,7 @@ import (
 	"math"
 	"time"
 
+	"cloud.google.com/go/storage/control/apiv2/controlpb"
 	"github.com/googlecloudplatform/gcsfuse/v2/internal/cache/lru"
 	"github.com/googlecloudplatform/gcsfuse/v2/internal/storage/gcs"
 	"github.com/googlecloudplatform/gcsfuse/v2/internal/util"
@@ -45,10 +46,29 @@ type StatCache interface {
 	// Erase the entry for the given object name, if any.
 	Erase(name string)
 
-	// Return the current entry for the given name, or nil if there is a negative
+	// Return the current object entry for the given name, or nil if there is a negative
 	// entry. Return hit == false when there is neither a positive nor a negative
 	// entry, or the entry has expired according to the supplied current time.
 	LookUp(name string, now time.Time) (hit bool, m *gcs.MinObject)
+
+	// Insert an entry for the given folder resource.
+	//
+	// In order to help cope with caching of arbitrarily out of date (i.e.
+	// inconsistent) object listings, entry will not replace any positive entry
+	// with a newer meta generation number.
+	//
+	// The entry will expire after the supplied time.
+	InsertFolder(f *controlpb.Folder, expiration time.Time)
+
+	// Return the current folder entry for the given name, or nil if there is a negative
+	// entry. Return hit == false when there is neither a positive nor a negative
+	// entry, or the entry has expired according to the supplied current time.
+	LookUpFolder(folderName string, now time.Time) (bool, *controlpb.Folder)
+
+	// Set up a negative entry for the given folder name, indicating that the name
+	// doesn't exist. Overwrite any existing entry for the name, positive or
+	// negative.
+	AddNegativeEntryForFolder(folderName string, expiration time.Time)
 }
 
 // Create a new bucket-view to the passed shared-cache object.
@@ -80,6 +100,7 @@ type statCacheBucketView struct {
 // entry. Nil object means negative entry.
 type entry struct {
 	m          *gcs.MinObject
+	f          *controlpb.Folder
 	expiration time.Time
 	key        string
 }
@@ -92,14 +113,18 @@ type entry struct {
 // benchmark runs) to heap-size per positive stat-cache entry
 // to calculate a size closer to the actual memory utilization.
 func (e entry) Size() (size uint64) {
-	// First, calculate size on heap.
+	// First, calculate size on heap (including folder size also in case of hns buckets, in case of non-hns buckets 0 will be added as e.f will be Nil ).
 	// Additional 2*util.UnsafeSizeOf(&e.key) is to account for the copies of string
 	// struct stored in the cache map and in the cache linked-list.
 	size = uint64(util.UnsafeSizeOf(&e) + len(e.key) + 2*util.UnsafeSizeOf(&e.key) + util.NestedSizeOfGcsMinObject(e.m))
 	if e.m != nil {
 		size += 515
 	}
 
+	if e.f != nil {
+		size += uint64(util.UnsafeSizeOf(&e.f))
+	}
+
 	// Convert heap-size to RSS (resident set size).
 	size = uint64(math.Ceil(util.HeapSizeToRssConversionFactor * float64(size)))
 
@@ -176,30 +201,86 @@ func (sc *statCacheBucketView) AddNegativeEntry(objectName string, expiration ti
 	}
 }
 
+func (sc *statCacheBucketView) AddNegativeEntryForFolder(folderName string, expiration time.Time) {
+	name := sc.key(folderName)
+
+	// Insert a negative entry.
+	e := entry{
+		f:          nil,
+		expiration: expiration,
+		key:        name,
+	}
+
+	if _, err := sc.sharedCache.Insert(name, e); err != nil {
+		panic(err)
+	}
+}
+
 func (sc *statCacheBucketView) Erase(objectName string) {
 	name := sc.key(objectName)
 	sc.sharedCache.Erase(name)
 }
 
 func (sc *statCacheBucketView) LookUp(
 	objectName string,
-	now time.Time) (hit bool, m *gcs.MinObject) {
+	now time.Time) (bool, *gcs.MinObject) {
 	// Look up in the LRU cache.
-	value := sc.sharedCache.LookUp(sc.key(objectName))
+	hit, entry := sc.sharedCacheLookup(objectName, now)
+	if hit {
+		return hit, entry.m
+	}
+
+	return false, nil
+}
+
+func (sc *statCacheBucketView) LookUpFolder(
+	folderName string,
+	now time.Time) (bool, *controlpb.Folder) {
+	// Look up in the LRU cache.
+	hit, entry := sc.sharedCacheLookup(folderName, now)
+	if hit {
+		return hit, entry.f
+	}
+
+	return false, nil
+}
+
+func (sc *statCacheBucketView) sharedCacheLookup(key string, now time.Time) (bool, *entry) {
+	value := sc.sharedCache.LookUp(sc.key(key))
 	if value == nil {
-		return
+		return false, nil
 	}
 
 	e := value.(entry)
 
 	// Has this entry expired?
 	if e.expiration.Before(now) {
-		sc.Erase(objectName)
-		return
+		sc.Erase(key)
+		return false, nil
 	}
 
-	hit = true
-	m = e.m
+	return true, &e
+}
 
-	return
+func (sc *statCacheBucketView) InsertFolder(f *controlpb.Folder, expiration time.Time) {
+	name := sc.key(f.Name)
+
+	// Return if there is already a better entry?
+	existing := sc.sharedCache.LookUp(name)
+	if existing != nil && existing.(entry).f != nil {
+		existingFolder := existing.(entry).f
+		if f.Metageneration != existingFolder.Metageneration && f.Metageneration < existingFolder.Metageneration {
+			return
+		}
+	}
+
+	e := entry{
+		f:          f,
+		expiration: expiration,
+		key:        name,
+	}
+
+	if _, err := sc.sharedCache.Insert(name, e); err != nil {
+		panic(err)
+	}
 }

internal/cache/metadata/stat_cache_test.go

@@ -18,6 +18,7 @@ import (
 	"testing"
 	"time"
 
+	"cloud.google.com/go/storage/control/apiv2/controlpb"
 	"github.com/googlecloudplatform/gcsfuse/v2/internal/cache/lru"
 	"github.com/googlecloudplatform/gcsfuse/v2/internal/cache/metadata"
 	"github.com/googlecloudplatform/gcsfuse/v2/internal/mount"
@@ -101,6 +102,14 @@ var expiration = someTime.Add(time.Second)
 type StatCacheTest struct {
 	suite.Suite
 	cache testHelperCache
+	//t.cache is the wrapper class on top of metadata.StatCache.
+	//This approach tests wrapper methods instead of directly testing actual functionality, compromising the safety net.
+	//For instance: If the helper class changes internally and stops calling stat cache methods, tests won't fail,
+	//hence tests are not being safety net capturing behaviour change of actual functionality.
+	//added stat cache to test metadata.StatCache directly, removing unnecessary wrappers for accurate unit testing.
+	//Changing every test will increase the scope and actual hns work will be affected, so taking cautious call to just add new test in refactored way in first go,
+	//we can update rest of tests slowly later.
+	statCache metadata.StatCache
 }
 
 type MultiBucketStatCacheTest struct {
@@ -111,6 +120,7 @@ type MultiBucketStatCacheTest struct {
 func (t *StatCacheTest) SetupTest() {
 	cache := lru.NewCache(uint64((mount.AverageSizeOfPositiveStatCacheEntry + mount.AverageSizeOfNegativeStatCacheEntry) * capacity))
 	t.cache.wrapped = metadata.NewStatCacheBucketView(cache, "") // this demonstrates
+	t.statCache = metadata.NewStatCacheBucketView(cache, "")     // this demonstrates
 	// that if you are using a cache for a single bucket, then
 	// its prepending bucketName can be left empty("") without any problem.
 }
@@ -408,3 +418,131 @@ func (t *MultiBucketStatCacheTest) Test_ExpiresLeastRecentlyUsed() {
 	assert.Equal(t.T(), cardamom, spices.LookUpOrNil("cardamom", someTime))
 	assert.Equal(t.T(), saffron, spices.LookUpOrNil("saffron", someTime))
 }
+
+func (t *StatCacheTest) Test_InsertFolderCreateEntryWhenNoEntryIsPresent() {
+	const name = "key1"
+	newEntry := &controlpb.Folder{
+		Name:           name,
+		Metageneration: 1,
+	}
+
+	t.statCache.InsertFolder(newEntry, expiration)
+
+	hit, entry := t.statCache.LookUpFolder(name, someTime)
+	assert.True(t.T(), hit)
+	assert.Equal(t.T(), "key1", entry.Name)
+	assert.Equal(t.T(), int64(1), entry.Metageneration)
+}
+
+func (t *StatCacheTest) Test_InsertFolderOverrideEntryOldEntryIsAlreadyPresent() {
+	const name = "key1"
+	existingEntry := &controlpb.Folder{
+		Name:           name,
+		Metageneration: 1,
+	}
+	t.statCache.InsertFolder(existingEntry, expiration)
+	newEntry := &controlpb.Folder{
+		Name:           name,
+		Metageneration: 2,
+	}
+
+	t.statCache.InsertFolder(newEntry, expiration)
+
+	hit, entry := t.statCache.LookUpFolder(name, someTime)
+	assert.True(t.T(), hit)
+	assert.Equal(t.T(), "key1", entry.Name)
+	assert.Equal(t.T(), int64(2), entry.Metageneration)
+}
+
+func (t *StatCacheTest) Test_LookupReturnFalseIfExpirationIsPassed() {
+	const name = "key1"
+	entry := &controlpb.Folder{
+		Name:           name,
+		Metageneration: 1,
+	}
+	t.statCache.InsertFolder(entry, expiration)
+
+	hit, result := t.statCache.LookUpFolder(name, expiration.Add(time.Second))
+
+	assert.False(t.T(), hit)
+	assert.Nil(t.T(), result)
+}
+
+func (t *StatCacheTest) Test_LookupReturnFalseWhenIsNotPresent() {
+	const name = "key1"
+
+	hit, result := t.statCache.LookUpFolder(name, expiration.Add(time.Second))
+
+	assert.False(t.T(), hit)
+	assert.Nil(t.T(), result)
+}
+
+func (t *StatCacheTest) Test_InsertFolderShouldNotOverrideEntryIfMetagenerationIsOld() {
+	const name = "key1"
+	existingEntry := &controlpb.Folder{
+		Name:           name,
+		Metageneration: 2,
+	}
+	t.statCache.InsertFolder(existingEntry, expiration)
+	newEntry := &controlpb.Folder{
+		Name:           name,
+		Metageneration: 1,
+	}
+
+	t.statCache.InsertFolder(newEntry, expiration)
+
+	hit, entry := t.statCache.LookUpFolder(name, someTime)
+	assert.True(t.T(), hit)
+	assert.Equal(t.T(), "key1", entry.Name)
+	assert.Equal(t.T(), int64(2), entry.Metageneration)
+}
+
+func (t *StatCacheTest) Test_AddNegativeEntryForFolderShouldAddNegativeEntryForFolder() {
+	const name = "key1"
+	existingEntry := &controlpb.Folder{
+		Name:           name,
+		Metageneration: 2,
+	}
+	t.statCache.InsertFolder(existingEntry, expiration)
+
+	t.statCache.AddNegativeEntryForFolder(name, expiration)
+
+	hit, entry := t.statCache.LookUpFolder(name, someTime)
+	assert.True(t.T(), hit)
+	assert.Nil(t.T(), entry)
+}
+
+func (t *StatCacheTest) Test_ShouldEvictEntryOnFullCapacityIncludingFolderSize() {
+	localCache := lru.NewCache(uint64(3000))
+	t.statCache = metadata.NewStatCacheBucketView(localCache, "local_bucket")
+	objectEntry1 := &gcs.MinObject{Name: "1"}
+	objectEntry2 := &gcs.MinObject{Name: "2"}
+	folderEntry := &controlpb.Folder{
+		Name:           "3",
+		Metageneration: 1,
+	}
+	t.statCache.Insert(objectEntry1, expiration) // adds size of 1428
+	t.statCache.Insert(objectEntry2, expiration) // adds size of 1428
+
+	hit1, entry1 := t.statCache.LookUp("1", someTime)
+	hit2, entry2 := t.statCache.LookUp("2", someTime)
+
+	assert.True(t.T(), hit1)
+	assert.Equal(t.T(), "1", entry1.Name)
+	assert.True(t.T(), hit2)
+	assert.Equal(t.T(), "2", entry2.Name)
+
+	t.statCache.InsertFolder(folderEntry, expiration) //adds size of 220 and exceeds capacity
+
+	hit1, entry1 = t.statCache.LookUp("1", someTime)
+	hit2, entry2 = t.statCache.LookUp("2", someTime)
+	hit3, entry3 := t.statCache.LookUpFolder("3", someTime)
+
+	assert.False(t.T(), hit1)
+	assert.Nil(t.T(), entry1)
+	assert.True(t.T(), hit2)
+	assert.Equal(t.T(), "2", entry2.Name)
+	assert.True(t.T(), hit3)
+	assert.Equal(t.T(), "3", entry3.Name)
+
+}