Skip to content

Commit 6548cd1

Browse files
Use total bucket capacity for size tracker
1 parent 818f8bc commit 6548cd1

File tree

3 files changed

+223
-18
lines changed

3 files changed

+223
-18
lines changed

component/size_tracker/size_tracker.go

+12-12
Original file line numberDiff line numberDiff line change
@@ -48,15 +48,15 @@ import (
4848
// Common structure for Component
4949
type SizeTracker struct {
5050
internal.BaseComponent
51-
mountSize *MountSize
52-
bucketCapacityFallback bool
53-
displayCapacityMb uint64
51+
mountSize *MountSize
52+
displayCapacityMb uint64
53+
totalBucketCapacity uint64
5454
}
5555

5656
// Structure defining your config parameters
5757
type SizeTrackerOptions struct {
58-
JournalName string `config:"journal-name" yaml:"journal-name,omitempty"`
59-
BucketCapacityFallback bool `config:"bucket-capacity-fallback" yaml:"bucket-capacity-fallback,omitempty"`
58+
JournalName string `config:"journal-name" yaml:"journal-name,omitempty"`
59+
TotalBucketCapacity uint64 `config:"bucket-capacity-fallback" yaml:"bucket-capacity-fallback,omitempty"`
6060
}
6161

6262
const compName = "size_tracker"
@@ -108,14 +108,14 @@ func (st *SizeTracker) Configure(_ bool) error {
108108
return fmt.Errorf("SizeTracker: config error [invalid config attributes]")
109109
}
110110

111-
st.bucketCapacityFallback = conf.BucketCapacityFallback
111+
st.totalBucketCapacity = conf.TotalBucketCapacity
112112

113-
if st.bucketCapacityFallback {
113+
if st.totalBucketCapacity != 0 {
114114
// Borrow enable-symlinks flag from attribute cache
115115
if config.IsSet("libfuse.display-capacity-mb") {
116116
err := config.UnmarshalKey("libfuse.display-capacity-mb", &st.displayCapacityMb)
117117
if err != nil {
118-
st.bucketCapacityFallback = false
118+
st.totalBucketCapacity = 0
119119
log.Err("Configure : Failed to unmarshal libfuse.display-capacity-mb. Attempting to use" +
120120
" bucket capacity fallback without setting display capacity.")
121121
}
@@ -322,7 +322,7 @@ func (st *SizeTracker) FlushFile(options internal.FlushFileOptions) error {
322322
diff := newSize - origSize
323323

324324
var journalErr error
325-
// File already exists and CopyFromFile succeeded subtract difference in file size
325+
// File already exists and FlushFile succeeded subtract difference in file size
326326
if diff < 0 {
327327
st.mountSize.Subtract(uint64(-diff))
328328
} else {
@@ -341,15 +341,15 @@ func (st *SizeTracker) StatFs() (*common.Statfs_t, bool, error) {
341341

342342
blocks := st.mountSize.GetSize() / uint64(blockSize)
343343

344-
if st.bucketCapacityFallback {
344+
if st.totalBucketCapacity != 0 {
345345
stat, ret, err := st.NextComponent().StatFs()
346346

347347
if err == nil && ret {
348348
// Custom logic for use with Nx Plugin
349349
// If the user is over the capacity limit set by Nx, then we need to prevent them from
350350
// accidental overuse of their bucket. So we change our reporting to instead report
351351
// the used capacity of the bucket to enable the VMS to start eviction
352-
if evictionThreshold*float64(stat.Blocks) > float64(st.displayCapacityMb) {
352+
if float64(stat.Blocks*uint64(blockSize)) > evictionThreshold*float64(st.totalBucketCapacity) {
353353
log.Warn("SizeTracker::StatFs : changing from size_tracker size to S3 bucket size due to overuse of bucket")
354354
blocks = stat.Blocks
355355
}
@@ -400,7 +400,7 @@ func (st *SizeTracker) CommitData(opt internal.CommitDataOptions) error {
400400
diff := newSize - origSize
401401

402402
var journalErr error
403-
// File already exists and CopyFromFile succeeded subtract difference in file size
403+
// File already exists and CommitData succeeded subtract difference in file size
404404
if diff < 0 {
405405
st.mountSize.Subtract(uint64(-diff))
406406
} else {
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,200 @@
1+
/*
2+
Licensed under the MIT License <http://opensource.org/licenses/MIT>.
3+
4+
Copyright © 2023-2025 Seagate Technology LLC and/or its Affiliates
5+
6+
Permission is hereby granted, free of charge, to any person obtaining a copy
7+
of this software and associated documentation files (the "Software"), to deal
8+
in the Software without restriction, including without limitation the rights
9+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10+
copies of the Software, and to permit persons to whom the Software is
11+
furnished to do so, subject to the following conditions:
12+
13+
The above copyright notice and this permission notice shall be included in all
14+
copies or substantial portions of the Software.
15+
16+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22+
SOFTWARE
23+
*/
24+
25+
package size_tracker
26+
27+
import (
28+
"context"
29+
"crypto/rand"
30+
"fmt"
31+
"os"
32+
"strings"
33+
"testing"
34+
35+
"github.com/Seagate/cloudfuse/common"
36+
"github.com/Seagate/cloudfuse/common/config"
37+
"github.com/Seagate/cloudfuse/common/log"
38+
"github.com/Seagate/cloudfuse/internal"
39+
"github.com/Seagate/cloudfuse/internal/handlemap"
40+
41+
"github.com/golang/mock/gomock"
42+
"github.com/stretchr/testify/assert"
43+
"github.com/stretchr/testify/suite"
44+
)
45+
46+
type sizeTrackerMockTestSuite struct {
47+
suite.Suite
48+
assert *assert.Assertions
49+
sizeTracker *SizeTracker
50+
mockCtrl *gomock.Controller
51+
mock *internal.MockComponent
52+
}
53+
54+
func newTestSizeTrackerMock(next internal.Component, configuration string) *SizeTracker {
55+
_ = config.ReadConfigFromReader(strings.NewReader(configuration))
56+
sizeTracker := NewSizeTrackerComponent()
57+
sizeTracker.SetNextComponent(next)
58+
_ = sizeTracker.Configure(true)
59+
60+
return sizeTracker.(*SizeTracker)
61+
}
62+
63+
func (suite *sizeTrackerMockTestSuite) SetupTest() {
64+
err := log.SetDefaultLogger("silent", common.LogConfig{})
65+
if err != nil {
66+
panic(fmt.Sprintf("Unable to set silent logger as default: %v", err))
67+
}
68+
cfg := fmt.Sprintf("size_tracker:\n journal-name: %s", journal_test_name)
69+
suite.setupTestHelper(cfg)
70+
}
71+
72+
func (suite *sizeTrackerMockTestSuite) setupTestHelper(config string) {
73+
suite.assert = assert.New(suite.T())
74+
75+
suite.mockCtrl = gomock.NewController(suite.T())
76+
suite.mock = internal.NewMockComponent(suite.mockCtrl)
77+
suite.sizeTracker = newTestSizeTrackerMock(suite.mock, config)
78+
_ = suite.sizeTracker.Start(context.Background())
79+
}
80+
81+
func (suite *sizeTrackerMockTestSuite) cleanupTest() {
82+
err := suite.sizeTracker.Stop()
83+
if err != nil {
84+
panic(fmt.Sprintf("Unable to stop size tracker [%s]", err.Error()))
85+
}
86+
journal_file := common.JoinUnixFilepath(common.DefaultWorkDir, journal_test_name)
87+
os.Remove(journal_file)
88+
suite.mockCtrl.Finish()
89+
}
90+
91+
// Tests the default configuration of attribute cache
92+
func (suite *sizeTrackerMockTestSuite) TestDefault() {
93+
defer suite.cleanupTest()
94+
suite.assert.Equal("size_tracker", suite.sizeTracker.Name())
95+
suite.assert.EqualValues(uint64(0), suite.sizeTracker.mountSize.GetSize())
96+
}
97+
98+
func (suite *sizeTrackerMockTestSuite) TestStatFSFallBackEnabledUnderThreshold() {
99+
defer suite.cleanupTest()
100+
suite.assert.EqualValues(0, suite.sizeTracker.mountSize.GetSize())
101+
suite.sizeTracker.totalBucketCapacity = 10 * 1024 * 1024
102+
103+
// Create File
104+
file := generateFileName()
105+
suite.mock.EXPECT().GetAttr(internal.GetAttrOptions{Name: file}).Return(&internal.ObjAttr{Path: file}, nil)
106+
suite.mock.EXPECT().CreateFile(internal.CreateFileOptions{Name: file, Mode: 0644}).Return(&handlemap.Handle{}, nil)
107+
handle, err := suite.sizeTracker.CreateFile(internal.CreateFileOptions{Name: file, Mode: 0644})
108+
suite.assert.NoError(err)
109+
110+
// Write File
111+
data := make([]byte, 1024*1024)
112+
_, _ = rand.Read(data)
113+
suite.mock.EXPECT().GetAttr(internal.GetAttrOptions{Name: handle.Path}).Return(&internal.ObjAttr{Path: file}, nil)
114+
suite.mock.EXPECT().WriteFile(internal.WriteFileOptions{Handle: handle, Offset: 0, Data: data}).Return(len(data), nil)
115+
_, err = suite.sizeTracker.WriteFile(internal.WriteFileOptions{Handle: handle, Offset: 0, Data: data})
116+
suite.assert.NoError(err)
117+
118+
// Flush File
119+
suite.mock.EXPECT().GetAttr(internal.GetAttrOptions{Name: handle.Path}).Return(&internal.ObjAttr{Path: file, Size: int64(len(data))}, nil)
120+
suite.mock.EXPECT().GetAttr(internal.GetAttrOptions{Name: handle.Path}).Return(&internal.ObjAttr{Path: file, Size: int64(len(data))}, nil)
121+
suite.mock.EXPECT().FlushFile(internal.FlushFileOptions{Handle: handle}).Return(nil)
122+
err = suite.sizeTracker.FlushFile(internal.FlushFileOptions{Handle: handle})
123+
suite.assert.NoError(err)
124+
suite.assert.EqualValues(len(data), suite.sizeTracker.mountSize.GetSize())
125+
126+
// Call Statfs
127+
suite.mock.EXPECT().StatFs().Return(&common.Statfs_t{
128+
Blocks: 9 * 1024 * 1024 / 4096,
129+
Bavail: 0,
130+
Bfree: 0,
131+
Bsize: 4096,
132+
Ffree: 1e9,
133+
Files: 1e9,
134+
Frsize: 4096,
135+
Namemax: 255,
136+
}, true, nil)
137+
stat, ret, err := suite.sizeTracker.StatFs()
138+
suite.assert.True(ret)
139+
suite.assert.NoError(err)
140+
suite.assert.NotEqual(&common.Statfs_t{}, stat)
141+
suite.assert.Equal(uint64(1024*1024/4096), stat.Blocks)
142+
suite.assert.Equal(int64(4096), stat.Bsize)
143+
suite.assert.Equal(int64(4096), stat.Frsize)
144+
suite.assert.Equal(uint64(255), stat.Namemax)
145+
}
146+
147+
func (suite *sizeTrackerMockTestSuite) TestStatFSFallBackEnabledOverThreshold() {
148+
defer suite.cleanupTest()
149+
suite.assert.EqualValues(0, suite.sizeTracker.mountSize.GetSize())
150+
suite.sizeTracker.totalBucketCapacity = 10 * 1024 * 1024
151+
152+
// Create File
153+
file := generateFileName()
154+
suite.mock.EXPECT().GetAttr(internal.GetAttrOptions{Name: file}).Return(&internal.ObjAttr{Path: file}, nil)
155+
suite.mock.EXPECT().CreateFile(internal.CreateFileOptions{Name: file, Mode: 0644}).Return(&handlemap.Handle{}, nil)
156+
handle, err := suite.sizeTracker.CreateFile(internal.CreateFileOptions{Name: file, Mode: 0644})
157+
suite.assert.NoError(err)
158+
159+
// Write File
160+
data := make([]byte, 1024*1024)
161+
_, _ = rand.Read(data)
162+
suite.mock.EXPECT().GetAttr(internal.GetAttrOptions{Name: handle.Path}).Return(&internal.ObjAttr{Path: file}, nil)
163+
suite.mock.EXPECT().WriteFile(internal.WriteFileOptions{Handle: handle, Offset: 0, Data: data}).Return(len(data), nil)
164+
_, err = suite.sizeTracker.WriteFile(internal.WriteFileOptions{Handle: handle, Offset: 0, Data: data})
165+
suite.assert.NoError(err)
166+
167+
// Flush File
168+
suite.mock.EXPECT().GetAttr(internal.GetAttrOptions{Name: handle.Path}).Return(&internal.ObjAttr{Path: file, Size: int64(len(data))}, nil)
169+
suite.mock.EXPECT().GetAttr(internal.GetAttrOptions{Name: handle.Path}).Return(&internal.ObjAttr{Path: file, Size: int64(len(data))}, nil)
170+
suite.mock.EXPECT().FlushFile(internal.FlushFileOptions{Handle: handle}).Return(nil)
171+
err = suite.sizeTracker.FlushFile(internal.FlushFileOptions{Handle: handle})
172+
suite.assert.NoError(err)
173+
suite.assert.EqualValues(len(data), suite.sizeTracker.mountSize.GetSize())
174+
175+
// Call Statfs
176+
suite.mock.EXPECT().StatFs().Return(&common.Statfs_t{
177+
Blocks: 10 * 1024 * 1024 / 4096,
178+
Bavail: 0,
179+
Bfree: 0,
180+
Bsize: 4096,
181+
Ffree: 1e9,
182+
Files: 1e9,
183+
Frsize: 4096,
184+
Namemax: 255,
185+
}, true, nil)
186+
stat, ret, err := suite.sizeTracker.StatFs()
187+
suite.assert.True(ret)
188+
suite.assert.NoError(err)
189+
suite.assert.NotEqual(&common.Statfs_t{}, stat)
190+
suite.assert.Equal(uint64(10*1024*1024/4096), stat.Blocks)
191+
suite.assert.Equal(int64(4096), stat.Bsize)
192+
suite.assert.Equal(int64(4096), stat.Frsize)
193+
suite.assert.Equal(uint64(255), stat.Namemax)
194+
}
195+
196+
// In order for 'go test' to run this suite, we need to create
197+
// a normal test function and pass our suite to suite.Run
198+
func TestSizeTrackerMockTestSuite(t *testing.T) {
199+
suite.Run(t, new(sizeTrackerMockTestSuite))
200+
}

component/size_tracker/size_tracker_test.go

+11-6
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,6 @@ import (
4343
"github.com/Seagate/cloudfuse/internal"
4444
"github.com/Seagate/cloudfuse/internal/handlemap"
4545

46-
"github.com/golang/mock/gomock"
4746
"github.com/stretchr/testify/assert"
4847
"github.com/stretchr/testify/suite"
4948
)
@@ -53,8 +52,6 @@ type sizeTrackerTestSuite struct {
5352
assert *assert.Assertions
5453
sizeTracker *SizeTracker
5554
loopback internal.Component
56-
mockCtrl *gomock.Controller
57-
mock *internal.MockComponent
5855
loopback_storage_path string
5956
}
6057

@@ -108,8 +105,6 @@ func (suite *sizeTrackerTestSuite) SetupTest() {
108105
func (suite *sizeTrackerTestSuite) setupTestHelper(config string) {
109106
suite.assert = assert.New(suite.T())
110107

111-
suite.mockCtrl = gomock.NewController(suite.T())
112-
suite.mock = internal.NewMockComponent(suite.mockCtrl)
113108
suite.loopback = newLoopbackFS()
114109
suite.sizeTracker = newTestSizeTracker(suite.loopback, config)
115110
_ = suite.loopback.Start(context.Background())
@@ -125,7 +120,6 @@ func (suite *sizeTrackerTestSuite) cleanupTest() {
125120
journal_file := common.JoinUnixFilepath(common.DefaultWorkDir, journal_test_name)
126121
os.Remove(journal_file)
127122
os.RemoveAll(suite.loopback_storage_path)
128-
suite.mockCtrl.Finish()
129123
}
130124

131125
// Tests the default configuration of attribute cache
@@ -589,6 +583,17 @@ func (suite *sizeTrackerTestSuite) TestStatFS() {
589583
suite.assert.NoError(err)
590584
}
591585

586+
func (suite *sizeTrackerTestSuite) TestStatFSNoWrites() {
587+
defer suite.cleanupTest()
588+
stat, ret, err := suite.sizeTracker.StatFs()
589+
suite.assert.True(ret)
590+
suite.assert.NoError(err)
591+
suite.assert.Equal(uint64(0), stat.Blocks)
592+
suite.assert.Equal(int64(4096), stat.Bsize)
593+
suite.assert.Equal(uint64(0), stat.Bavail)
594+
suite.assert.Equal(uint64(0), stat.Bfree)
595+
}
596+
592597
// In order for 'go test' to run this suite, we need to create
593598
// a normal test function and pass our suite to suite.Run
594599
func TestSizeTrackerTestSuite(t *testing.T) {

0 commit comments

Comments
 (0)