From 5ad54212812dd1d8f36f52e81ff6e5d02f374123 Mon Sep 17 00:00:00 2001 From: ssd04 Date: Wed, 3 Jan 2024 21:35:54 +0200 Subject: [PATCH 01/13] remove NewDB func --- storageUnit/storageunit.go | 40 +----------------- storageUnit/storageunit_test.go | 48 ---------------------- testscommon/persisterFactoryHandlerMock.go | 5 +++ 3 files changed, 7 insertions(+), 86 deletions(-) diff --git a/storageUnit/storageunit.go b/storageUnit/storageunit.go index bb262d1b..c119d322 100644 --- a/storageUnit/storageunit.go +++ b/storageUnit/storageunit.go @@ -294,6 +294,7 @@ func NewStorageUnit(c types.Cacher, p types.Persister) (*Unit, error) { // PersisterFactoryHandler defines the behaviour of a component which is able to create persisters type PersisterFactoryHandler interface { Create(path string) (types.Persister, error) + CreateWithRetries(path string) (types.Persister, error) IsInterfaceNil() bool } @@ -315,7 +316,7 @@ func NewStorageUnitFromConf(cacheConf CacheConfig, dbConf DBConfig, persisterFac return nil, err } - db, err = NewDB(persisterFactory, dbConf.FilePath) + db, err = persisterFactory.CreateWithRetries(dbConf.FilePath) if err != nil { return nil, err } @@ -369,43 +370,6 @@ func NewCache(config CacheConfig) (types.Cacher, error) { return cacher, nil } -// ArgDB is a structure that is used to create a new storage.Persister implementation -type ArgDB struct { - DBType DBType - Path string - BatchDelaySeconds int - MaxBatchSize int - MaxOpenFiles int -} - -// NewDB creates a new database from database config -// TODO: refactor to integrate retries loop into persister factory; maybe implement persister -// factory separatelly in storage repo -func NewDB(persisterFactory PersisterFactoryHandler, path string) (types.Persister, error) { - if check.IfNil(persisterFactory) { - return nil, ErrNilPersisterFactory - } - - var db types.Persister - var err error - - for i := 0; i < MaxRetriesToCreateDB; i++ { - db, err = persisterFactory.Create(path) - - if err == nil { - return db, nil - } - - // TODO: extract this in a parameter and inject it - time.Sleep(SleepTimeBetweenCreateDBRetries) - } - if err != nil { - return nil, err - } - - return db, nil -} - // NewHasher will return a hasher implementation form the string HasherType func (h HasherType) NewHasher() (hashing.Hasher, error) { switch h { diff --git a/storageUnit/storageunit_test.go b/storageUnit/storageunit_test.go index 45c43fb2..4a4d651e 100644 --- a/storageUnit/storageunit_test.go +++ b/storageUnit/storageunit_test.go @@ -254,54 +254,6 @@ func TestCreateCacheFromConfOK(t *testing.T) { assert.NotNil(t, cacher, "valid cacher expected but got nil") } -func TestCreateDBFromConfWrongType(t *testing.T) { - persisterFactory := testscommon.NewPersisterFactoryHandlerMock( - "NotLvlDB", - 10, - 10, - 10, - ) - persister, err := storageUnit.NewDB(persisterFactory, "test") - - assert.NotNil(t, err, "error expected") - assert.Nil(t, persister, "persister expected to be nil, but got %s", persister) -} - -func TestCreateDBFromConfWrongFileNameLvlDB(t *testing.T) { - if testing.Short() { - t.Skip("this is not a short test") - } - - path := "" - persisterFactory := testscommon.NewPersisterFactoryHandlerMock( - storageUnit.LvlDB, - 10, - 10, - 10, - ) - - persister, err := storageUnit.NewDB(persisterFactory, path) - assert.NotNil(t, err, "error expected") - assert.Nil(t, persister, "persister expected to be nil, but got %s", persister) -} - -func TestCreateDBFromConfLvlDBOk(t *testing.T) { - path := t.TempDir() - persisterFactory := testscommon.NewPersisterFactoryHandlerMock( - storageUnit.LvlDB, - 10, - 10, - 10, - ) - - persister, err := storageUnit.NewDB(persisterFactory, path) - assert.Nil(t, err, "no error expected") - assert.NotNil(t, persister, "valid persister expected but got nil") - - err = persister.Destroy() - assert.Nil(t, err, "no error expected destroying the persister") -} - func TestNewStorageUnit_FromConfWrongCacheSizeVsBatchSize(t *testing.T) { storer, err := storageUnit.NewStorageUnitFromConf(storageUnit.CacheConfig{ diff --git a/testscommon/persisterFactoryHandlerMock.go b/testscommon/persisterFactoryHandlerMock.go index e3b60ab6..3d94058b 100644 --- a/testscommon/persisterFactoryHandlerMock.go +++ b/testscommon/persisterFactoryHandlerMock.go @@ -25,6 +25,11 @@ func NewPersisterFactoryHandlerMock(dbType storageUnit.DBType, batchDelaySeconds } } +// CreateWithRetries - +func (mock *persisterFactoryHandlerMock) CreateWithRetries(path string) (types.Persister, error) { + return mock.Create(path) +} + // Create - func (mock *persisterFactoryHandlerMock) Create(path string) (types.Persister, error) { switch mock.dbType { From 0f238685b895724a3edacb258620809101b979c5 Mon Sep 17 00:00:00 2001 From: ssd04 Date: Tue, 30 Jan 2024 22:25:14 +0200 Subject: [PATCH 02/13] remove unused code --- storageUnit/storageunit.go | 138 +++++++++++++------------------------ 1 file changed, 49 insertions(+), 89 deletions(-) diff --git a/storageUnit/storageunit.go b/storageUnit/storageunit.go index c119d322..a93a119e 100644 --- a/storageUnit/storageunit.go +++ b/storageUnit/storageunit.go @@ -3,17 +3,12 @@ package storageUnit import ( "encoding/base64" "encoding/json" - "errors" "fmt" "sync" "time" "github.com/multiversx/mx-chain-core-go/core/check" "github.com/multiversx/mx-chain-core-go/data" - "github.com/multiversx/mx-chain-core-go/hashing" - "github.com/multiversx/mx-chain-core-go/hashing/blake2b" - "github.com/multiversx/mx-chain-core-go/hashing/fnv" - "github.com/multiversx/mx-chain-core-go/hashing/keccak" logger "github.com/multiversx/mx-chain-logger-go" "github.com/multiversx/mx-chain-storage-go/common" "github.com/multiversx/mx-chain-storage-go/fifocache" @@ -30,9 +25,6 @@ type CacheType string // DBType represents the type of the supported databases type DBType string -// HasherType represents the type of the supported hash functions -type HasherType string - // Cache types that are currently supported const ( LRUCache CacheType = "LRU" @@ -57,15 +49,6 @@ const ( BinarySplit ShardIDProviderType = "BinarySplit" ) -const ( - // Keccak is the string representation of the keccak hashing function - Keccak HasherType = "Keccak" - // Blake2b is the string representation of the blake2b hashing function - Blake2b HasherType = "Blake2b" - // Fnv is the string representation of the fnv hashing function - Fnv HasherType = "Fnv" -) - const minimumSizeForLRUCache = 1024 // MaxRetriesToCreateDB represents the maximum number of times to try to create DB if it failed @@ -74,13 +57,11 @@ const MaxRetriesToCreateDB = 10 // SleepTimeBetweenCreateDBRetries represents the number of seconds to sleep between DB creates const SleepTimeBetweenCreateDBRetries = 5 * time.Second -// ErrNilPersisterFactory signals that a nil persister factory handler has been provided -var ErrNilPersisterFactory = errors.New("nil persister factory") - -// UnitConfig holds the configurable elements of the storage unit -type UnitConfig struct { - CacheConf CacheConfig - DBConf DBConfig +// PersisterFactoryHandler defines the behaviour of a component which is able to create persisters +type PersisterFactoryHandler interface { + Create(path string) (types.Persister, error) + CreateWithRetries(path string) (types.Persister, error) + IsInterfaceNil() bool } // CacheConfig holds the configurable elements of a cache @@ -121,6 +102,50 @@ type Unit struct { cacher types.Cacher } +// NewStorageUnit is the constructor for the storage unit, creating a new storage unit +// from the given cacher and persister. +func NewStorageUnit(c types.Cacher, p types.Persister) (*Unit, error) { + if check.IfNil(p) { + return nil, common.ErrNilPersister + } + if check.IfNil(c) { + return nil, common.ErrNilCacher + } + + sUnit := &Unit{ + persister: p, + cacher: c, + } + + return sUnit, nil +} + +// NewStorageUnitFromConf creates a new storage unit from a storage unit config +func NewStorageUnitFromConf(cacheConf CacheConfig, dbConf DBConfig, persisterFactory PersisterFactoryHandler) (*Unit, error) { + var cache types.Cacher + var db types.Persister + var err error + + // TODO: if there will be a differentiation between the creation or opening of a DB, the DB could be destroyed + // in case of a failure while creating (not opening). + + if dbConf.MaxBatchSize > int(cacheConf.Capacity) { + return nil, common.ErrCacheSizeIsLowerThanBatchSize + } + + cache, err = NewCache(cacheConf) + if err != nil { + return nil, err + } + + db, err = persisterFactory.CreateWithRetries(dbConf.FilePath) + if err != nil { + return nil, err + } + + return NewStorageUnit(cache, db) +} + // Put adds data to both cache and persistence medium func (u *Unit) Put(key, data []byte) error { u.lock.Lock() @@ -273,57 +298,6 @@ func (u *Unit) IsInterfaceNil() bool { return u == nil } -// NewStorageUnit is the constructor for the storage unit, creating a new storage unit -// from the given cacher and persister. -func NewStorageUnit(c types.Cacher, p types.Persister) (*Unit, error) { - if check.IfNil(p) { - return nil, common.ErrNilPersister - } - if check.IfNil(c) { - return nil, common.ErrNilCacher - } - - sUnit := &Unit{ - persister: p, - cacher: c, - } - - return sUnit, nil -} - -// PersisterFactoryHandler defines the behaviour of a component which is able to create persisters -type PersisterFactoryHandler interface { - Create(path string) (types.Persister, error) - CreateWithRetries(path string) (types.Persister, error) - IsInterfaceNil() bool -} - -// NewStorageUnitFromConf creates a new storage unit from a storage unit config -func NewStorageUnitFromConf(cacheConf CacheConfig, dbConf DBConfig, persisterFactory PersisterFactoryHandler) (*Unit, error) { - var cache types.Cacher - var db types.Persister - var err error - - // TODO: if there will be a differentiation between the creation or opening of a DB, the DB could be destroyed - // in case of a failure while creating (not opening). - - if dbConf.MaxBatchSize > int(cacheConf.Capacity) { - return nil, common.ErrCacheSizeIsLowerThanBatchSize - } - - cache, err = NewCache(cacheConf) - if err != nil { - return nil, err - } - - db, err = persisterFactory.CreateWithRetries(dbConf.FilePath) - if err != nil { - return nil, err - } - - return NewStorageUnit(cache, db) -} - // NewCache creates a new cache from a cache config func NewCache(config CacheConfig) (types.Cacher, error) { monitoring.MonitorNewCache(config.Name, config.SizeInBytes) @@ -369,17 +343,3 @@ func NewCache(config CacheConfig) (types.Cacher, error) { return cacher, nil } - -// NewHasher will return a hasher implementation form the string HasherType -func (h HasherType) NewHasher() (hashing.Hasher, error) { - switch h { - case Keccak: - return keccak.NewKeccak(), nil - case Blake2b: - return blake2b.NewBlake2b(), nil - case Fnv: - return fnv.NewFnv(), nil - default: - return nil, common.ErrNotSupportedHashType - } -} From f8fcf674de9ded1a147026d11f625d0580302755 Mon Sep 17 00:00:00 2001 From: ssd04 Date: Wed, 31 Jan 2024 15:41:57 +0200 Subject: [PATCH 03/13] split code from storageunit to separate files --- common/config.go | 39 +++++ common/constants.go | 37 +++++ factory/cache.go | 59 +++++++ factory/cache_test.go | 25 +++ factory/db.go | 1 + storageUnit/storageunit.go | 122 +------------- storageUnit/storageunit_bench_test.go | 83 ++++++++++ storageUnit/storageunit_test.go | 180 ++++++++------------- testscommon/persisterFactoryHandlerMock.go | 11 +- 9 files changed, 316 insertions(+), 241 deletions(-) create mode 100644 common/config.go create mode 100644 common/constants.go create mode 100644 factory/cache.go create mode 100644 factory/cache_test.go create mode 100644 factory/db.go create mode 100644 storageUnit/storageunit_bench_test.go diff --git a/common/config.go b/common/config.go new file mode 100644 index 00000000..7a07d575 --- /dev/null +++ b/common/config.go @@ -0,0 +1,39 @@ +package common + +import ( + "encoding/json" + + logger "github.com/multiversx/mx-chain-logger-go" +) + +var log = logger.GetOrCreate("common") + +// CacheConfig holds the configurable elements of a cache +type CacheConfig struct { + Name string + Type CacheType + SizeInBytes uint64 + SizeInBytesPerSender uint32 + Capacity uint32 + SizePerSender uint32 + Shards uint32 +} + +// String returns a readable representation of the object +func (config *CacheConfig) String() string { + bytes, err := json.Marshal(config) + if err != nil { + log.Error("CacheConfig.String()", "err", err) + } + + return string(bytes) +} + +// DBConfig holds the configurable elements of a database +type DBConfig struct { + FilePath string + Type DBType + BatchDelaySeconds int + MaxBatchSize int + MaxOpenFiles int +} diff --git a/common/constants.go b/common/constants.go new file mode 100644 index 00000000..c1ecb595 --- /dev/null +++ b/common/constants.go @@ -0,0 +1,37 @@ +package common + +import "time" + +// CacheType represents the type of the supported caches +type CacheType string + +// Cache types that are currently supported +const ( + LRUCache CacheType = "LRU" + SizeLRUCache CacheType = "SizeLRU" + FIFOShardedCache CacheType = "FIFOSharded" +) + +// DBType represents the type of the supported databases +type DBType string + +// DB types that are currently supported +const ( + LvlDB DBType = "LvlDB" + LvlDBSerial DBType = "LvlDBSerial" + MemoryDB DBType = "MemoryDB" +) + +// ShardIDProviderType represents the type for the supported shard id provider +type ShardIDProviderType string + +// Shard id provider types that are currently supported +const ( + BinarySplit ShardIDProviderType = "BinarySplit" +) + +// MaxRetriesToCreateDB represents the maximum number of times to try to create DB if it failed +const MaxRetriesToCreateDB = 10 + +// SleepTimeBetweenCreateDBRetries represents the number of seconds to sleep between DB creates +const SleepTimeBetweenCreateDBRetries = 5 * time.Second diff --git a/factory/cache.go b/factory/cache.go new file mode 100644 index 00000000..6843a3c5 --- /dev/null +++ b/factory/cache.go @@ -0,0 +1,59 @@ +package factory + +import ( + "fmt" + + "github.com/multiversx/mx-chain-storage-go/common" + "github.com/multiversx/mx-chain-storage-go/fifocache" + "github.com/multiversx/mx-chain-storage-go/lrucache" + "github.com/multiversx/mx-chain-storage-go/monitoring" + "github.com/multiversx/mx-chain-storage-go/types" +) + +const minimumSizeForLRUCache = 1024 + +// NewCache creates a new cache from a cache config +func NewCache(config common.CacheConfig) (types.Cacher, error) { + monitoring.MonitorNewCache(config.Name, config.SizeInBytes) + + cacheType := config.Type + capacity := config.Capacity + shards := config.Shards + sizeInBytes := config.SizeInBytes + + var cacher types.Cacher + var err error + + switch cacheType { + case common.LRUCache: + if sizeInBytes != 0 { + return nil, common.ErrLRUCacheWithProvidedSize + } + + cacher, err = lrucache.NewCache(int(capacity)) + case common.SizeLRUCache: + if sizeInBytes < minimumSizeForLRUCache { + return nil, fmt.Errorf("%w, provided %d, minimum %d", + common.ErrLRUCacheInvalidSize, + sizeInBytes, + minimumSizeForLRUCache, + ) + } + + cacher, err = lrucache.NewCacheWithSizeInBytes(int(capacity), int64(sizeInBytes)) + case common.FIFOShardedCache: + cacher, err = fifocache.NewShardedCache(int(capacity), int(shards)) + if err != nil { + return nil, err + } + // add other implementations if required + default: + return nil, common.ErrNotSupportedCacheType + } + + if err != nil { + return nil, err + } + + return cacher, nil +} diff --git a/factory/cache_test.go b/factory/cache_test.go new file mode 100644 index 00000000..9e768c0a --- /dev/null +++ b/factory/cache_test.go @@ -0,0 +1,25 @@ +package factory_test + +import ( + "testing" + + "github.com/multiversx/mx-chain-storage-go/common" + "github.com/multiversx/mx-chain-storage-go/factory" + "github.com/stretchr/testify/assert" +) + +func TestCreateCacheFromConfWrongType(t *testing.T) { + + cacher, err := factory.NewCache(common.CacheConfig{Type: "NotLRU", Capacity: 100, Shards: 1, SizeInBytes: 0}) + + assert.NotNil(t, err, "error expected") + assert.Nil(t, cacher, "cacher expected to be nil, but got %s", cacher) +} + +func TestCreateCacheFromConfOK(t *testing.T) { + + cacher, err := factory.NewCache(common.CacheConfig{Type: common.LRUCache, Capacity: 10, Shards: 1, SizeInBytes: 0}) + + assert.Nil(t, err, "no error expected but got %s", err) + assert.NotNil(t, cacher, "valid cacher expected but got nil") +} diff --git a/factory/db.go b/factory/db.go new file mode 100644 index 00000000..7312cd2e --- /dev/null +++ b/factory/db.go @@ -0,0 +1 @@ +package factory diff --git a/storageUnit/storageunit.go b/storageUnit/storageunit.go index a93a119e..a71501fd 100644 --- a/storageUnit/storageunit.go +++ b/storageUnit/storageunit.go @@ -2,61 +2,21 @@ package storageUnit import ( "encoding/base64" - "encoding/json" "fmt" "sync" - "time" "github.com/multiversx/mx-chain-core-go/core/check" "github.com/multiversx/mx-chain-core-go/data" logger "github.com/multiversx/mx-chain-logger-go" "github.com/multiversx/mx-chain-storage-go/common" - "github.com/multiversx/mx-chain-storage-go/fifocache" - "github.com/multiversx/mx-chain-storage-go/lrucache" - "github.com/multiversx/mx-chain-storage-go/monitoring" + "github.com/multiversx/mx-chain-storage-go/factory" "github.com/multiversx/mx-chain-storage-go/types" ) var _ types.Storer = (*Unit)(nil) -// CacheType represents the type of the supported caches -type CacheType string - -// DBType represents the type of the supported databases -type DBType string - -// Cache types that are currently supported -const ( - LRUCache CacheType = "LRU" - SizeLRUCache CacheType = "SizeLRU" - FIFOShardedCache CacheType = "FIFOSharded" -) - var log = logger.GetOrCreate("storage/storageUnit") -// DB types that are currently supported -const ( - LvlDB DBType = "LvlDB" - LvlDBSerial DBType = "LvlDBSerial" - MemoryDB DBType = "MemoryDB" -) - -// ShardIDProviderType represents the type for the supported shard id provider -type ShardIDProviderType string - -// Shard id provider types that are currently supported -const ( - BinarySplit ShardIDProviderType = "BinarySplit" -) - -const minimumSizeForLRUCache = 1024 - -// MaxRetriesToCreateDB represents the maximum number of times to try to create DB if it failed -const MaxRetriesToCreateDB = 10 - -// SleepTimeBetweenCreateDBRetries represents the number of seconds to sleep between DB creates -const SleepTimeBetweenCreateDBRetries = 5 * time.Second - // PersisterFactoryHandler defines the behaviour of a component which is able to create persisters type PersisterFactoryHandler interface { Create(path string) (types.Persister, error) @@ -64,36 +24,6 @@ type PersisterFactoryHandler interface { IsInterfaceNil() bool } -// CacheConfig holds the configurable elements of a cache -type CacheConfig struct { - Name string - Type CacheType - SizeInBytes uint64 - SizeInBytesPerSender uint32 - Capacity uint32 - SizePerSender uint32 - Shards uint32 -} - -// String returns a readable representation of the object -func (config *CacheConfig) String() string { - bytes, err := json.Marshal(config) - if err != nil { - log.Error("CacheConfig.String()", "err", err) - } - - return string(bytes) -} - -// DBConfig holds the configurable elements of a database -type DBConfig struct { - FilePath string - Type DBType - BatchDelaySeconds int - MaxBatchSize int - MaxOpenFiles int -} - // Unit represents a storer's data bank // holding the cache and persistence unit type Unit struct { @@ -121,7 +51,7 @@ func NewStorageUnit(c types.Cacher, p types.Persister) (*Unit, error) { } // NewStorageUnitFromConf creates a new storage unit from a storage unit config -func NewStorageUnitFromConf(cacheConf CacheConfig, dbConf DBConfig, persisterFactory PersisterFactoryHandler) (*Unit, error) { +func NewStorageUnitFromConf(cacheConf common.CacheConfig, dbConf common.DBConfig, persisterFactory PersisterFactoryHandler) (*Unit, error) { var cache types.Cacher var db types.Persister var err error @@ -133,7 +63,7 @@ func NewStorageUnitFromConf(cacheConf CacheConfig, dbConf DBConfig, persisterFac return nil, common.ErrCacheSizeIsLowerThanBatchSize } - cache, err = NewCache(cacheConf) + cache, err = factory.NewCache(cacheConf) if err != nil { return nil, err } @@ -297,49 +227,3 @@ func (u *Unit) DestroyUnit() error { func (u *Unit) IsInterfaceNil() bool { return u == nil } - -// NewCache creates a new cache from a cache config -func NewCache(config CacheConfig) (types.Cacher, error) { - monitoring.MonitorNewCache(config.Name, config.SizeInBytes) - - cacheType := config.Type - capacity := config.Capacity - shards := config.Shards - sizeInBytes := config.SizeInBytes - - var cacher types.Cacher - var err error - - switch cacheType { - case LRUCache: - if sizeInBytes != 0 { - return nil, common.ErrLRUCacheWithProvidedSize - } - - cacher, err = lrucache.NewCache(int(capacity)) - case SizeLRUCache: - if sizeInBytes < minimumSizeForLRUCache { - return nil, fmt.Errorf("%w, provided %d, minimum %d", - common.ErrLRUCacheInvalidSize, - sizeInBytes, - minimumSizeForLRUCache, - ) - } - - cacher, err = lrucache.NewCacheWithSizeInBytes(int(capacity), int64(sizeInBytes)) - case FIFOShardedCache: - cacher, err = fifocache.NewShardedCache(int(capacity), int(shards)) - if err != nil { - return nil, err - } - // add other implementations if required - default: - return nil, common.ErrNotSupportedCacheType - } - - if err != nil { - return nil, err - } - - return cacher, nil -} diff --git a/storageUnit/storageunit_bench_test.go b/storageUnit/storageunit_bench_test.go new file mode 100644 index 00000000..afd5a20b --- /dev/null +++ b/storageUnit/storageunit_bench_test.go @@ -0,0 +1,83 @@ +package storageUnit_test + +import ( + "fmt" + "math/rand" + "strconv" + "testing" +) + +const ( + valuesInDb = 100000 +) + +func logError(err error) { + if err != nil { + fmt.Println(err.Error()) + } +} + +func BenchmarkStorageUnit_Put(b *testing.B) { + b.StopTimer() + s := initStorageUnit(b, 1) + defer func() { + err := s.DestroyUnit() + logError(err) + }() + b.StartTimer() + + for i := 0; i < b.N; i++ { + b.StopTimer() + nr := rand.Intn(valuesInDb) + b.StartTimer() + + err := s.Put([]byte(strconv.Itoa(nr)), []byte(strconv.Itoa(nr))) + logError(err) + } +} + +func BenchmarkStorageUnit_GetWithDataBeingPresent(b *testing.B) { + b.StopTimer() + s := initStorageUnit(b, 1) + defer func() { + err := s.DestroyUnit() + logError(err) + }() + for i := 0; i < valuesInDb; i++ { + err := s.Put([]byte(strconv.Itoa(i)), []byte(strconv.Itoa(i))) + logError(err) + } + b.StartTimer() + + for i := 0; i < b.N; i++ { + b.StopTimer() + nr := rand.Intn(valuesInDb) + b.StartTimer() + + _, err := s.Get([]byte(strconv.Itoa(nr))) + logError(err) + } +} + +func BenchmarkStorageUnit_GetWithDataNotBeingPresent(b *testing.B) { + b.StopTimer() + s := initStorageUnit(b, 1) + defer func() { + err := s.DestroyUnit() + logError(err) + }() + for i := 0; i < valuesInDb; i++ { + err := s.Put([]byte(strconv.Itoa(i)), []byte(strconv.Itoa(i))) + logError(err) + } + b.StartTimer() + + for i := 0; i < b.N; i++ { + b.StopTimer() + nr := rand.Intn(valuesInDb) + valuesInDb + b.StartTimer() + + _, err := s.Get([]byte(strconv.Itoa(nr))) + logError(err) + } +} diff --git a/storageUnit/storageunit_test.go b/storageUnit/storageunit_test.go index 4a4d651e..6014a52b 100644 --- a/storageUnit/storageunit_test.go +++ b/storageUnit/storageunit_test.go @@ -1,9 +1,6 @@ package storageUnit_test import ( - "fmt" - "math/rand" - "strconv" "testing" "github.com/multiversx/mx-chain-storage-go/common" @@ -14,12 +11,6 @@ import ( "github.com/stretchr/testify/assert" ) -func logError(err error) { - if err != nil { - fmt.Println(err.Error()) - } -} - func initStorageUnit(tb testing.TB, cSize int) *storageUnit.Unit { mdb := memorydb.New() cache, err2 := lrucache.NewCache(cSize) @@ -32,6 +23,8 @@ func initStorageUnit(tb testing.TB, cSize int) *storageUnit.Unit { } func TestStorageUnitNilPersister(t *testing.T) { + t.Parallel() + cache, err1 := lrucache.NewCache(10) assert.Nil(t, err1, "no error expected but got %s", err1) @@ -42,6 +35,8 @@ func TestStorageUnitNilPersister(t *testing.T) { } func TestStorageUnitNilCacher(t *testing.T) { + t.Parallel() + mdb := memorydb.New() _, err1 := storageUnit.NewStorageUnit(nil, mdb) @@ -49,6 +44,8 @@ func TestStorageUnitNilCacher(t *testing.T) { } func TestStorageUnit(t *testing.T) { + t.Parallel() + cache, err1 := lrucache.NewCache(10) mdb := memorydb.New() @@ -59,6 +56,8 @@ func TestStorageUnit(t *testing.T) { } func TestPutNotPresent(t *testing.T) { + t.Parallel() + key, val := []byte("key0"), []byte("value0") s := initStorageUnit(t, 10) err := s.Put(key, val) @@ -71,6 +70,8 @@ func TestPutNotPresent(t *testing.T) { } func TestPutNotPresentCache(t *testing.T) { + t.Parallel() + key, val := []byte("key1"), []byte("value1") s := initStorageUnit(t, 10) err := s.Put(key, val) @@ -85,6 +86,8 @@ func TestPutNotPresentCache(t *testing.T) { } func TestPutPresentShouldOverwriteValue(t *testing.T) { + t.Parallel() + key, val := []byte("key2"), []byte("value2") s := initStorageUnit(t, 10) err := s.Put(key, val) @@ -101,6 +104,8 @@ func TestPutPresentShouldOverwriteValue(t *testing.T) { } func TestGetNotPresent(t *testing.T) { + t.Parallel() + key := []byte("key3") s := initStorageUnit(t, 10) v, err := s.Get(key) @@ -109,6 +114,8 @@ func TestGetNotPresent(t *testing.T) { } func TestGetNotPresentCache(t *testing.T) { + t.Parallel() + key, val := []byte("key4"), []byte("value4") s := initStorageUnit(t, 10) err := s.Put(key, val) @@ -124,6 +131,8 @@ func TestGetNotPresentCache(t *testing.T) { } func TestGetPresent(t *testing.T) { + t.Parallel() + key, val := []byte("key5"), []byte("value4") s := initStorageUnit(t, 10) err := s.Put(key, val) @@ -137,6 +146,8 @@ func TestGetPresent(t *testing.T) { } func TestHasNotPresent(t *testing.T) { + t.Parallel() + key := []byte("key6") s := initStorageUnit(t, 10) err := s.Has(key) @@ -146,6 +157,8 @@ func TestHasNotPresent(t *testing.T) { } func TestHasNotPresentCache(t *testing.T) { + t.Parallel() + key, val := []byte("key7"), []byte("value7") s := initStorageUnit(t, 10) err := s.Put(key, val) @@ -160,6 +173,8 @@ func TestHasNotPresentCache(t *testing.T) { } func TestHasPresent(t *testing.T) { + t.Parallel() + key, val := []byte("key8"), []byte("value8") s := initStorageUnit(t, 10) err := s.Put(key, val) @@ -172,6 +187,8 @@ func TestHasPresent(t *testing.T) { } func TestDeleteNotPresent(t *testing.T) { + t.Parallel() + key := []byte("key12") s := initStorageUnit(t, 10) err := s.Remove(key) @@ -180,6 +197,8 @@ func TestDeleteNotPresent(t *testing.T) { } func TestDeleteNotPresentCache(t *testing.T) { + t.Parallel() + key, val := []byte("key13"), []byte("value13") s := initStorageUnit(t, 10) err := s.Put(key, val) @@ -201,6 +220,8 @@ func TestDeleteNotPresentCache(t *testing.T) { } func TestDeletePresent(t *testing.T) { + t.Parallel() + key, val := []byte("key14"), []byte("value14") s := initStorageUnit(t, 10) err := s.Put(key, val) @@ -221,6 +242,8 @@ func TestDeletePresent(t *testing.T) { } func TestClearCacheNotAffectPersist(t *testing.T) { + t.Parallel() + key, val := []byte("key15"), []byte("value15") s := initStorageUnit(t, 10) err := s.Put(key, val) @@ -233,40 +256,27 @@ func TestClearCacheNotAffectPersist(t *testing.T) { } func TestDestroyUnitNoError(t *testing.T) { + t.Parallel() + s := initStorageUnit(t, 10) err := s.DestroyUnit() assert.Nil(t, err, "no error expected, but got %s", err) } -func TestCreateCacheFromConfWrongType(t *testing.T) { - - cacher, err := storageUnit.NewCache(storageUnit.CacheConfig{Type: "NotLRU", Capacity: 100, Shards: 1, SizeInBytes: 0}) - - assert.NotNil(t, err, "error expected") - assert.Nil(t, cacher, "cacher expected to be nil, but got %s", cacher) -} - -func TestCreateCacheFromConfOK(t *testing.T) { - - cacher, err := storageUnit.NewCache(storageUnit.CacheConfig{Type: storageUnit.LRUCache, Capacity: 10, Shards: 1, SizeInBytes: 0}) - - assert.Nil(t, err, "no error expected but got %s", err) - assert.NotNil(t, cacher, "valid cacher expected but got nil") -} - func TestNewStorageUnit_FromConfWrongCacheSizeVsBatchSize(t *testing.T) { + t.Parallel() - storer, err := storageUnit.NewStorageUnitFromConf(storageUnit.CacheConfig{ + storer, err := storageUnit.NewStorageUnitFromConf(common.CacheConfig{ Capacity: 10, - Type: storageUnit.LRUCache, - }, storageUnit.DBConfig{ + Type: common.LRUCache, + }, common.DBConfig{ FilePath: "Blocks", - Type: storageUnit.LvlDB, + Type: common.LvlDB, MaxBatchSize: 11, BatchDelaySeconds: 1, MaxOpenFiles: 10, }, - testscommon.NewPersisterFactoryHandlerMock(storageUnit.LvlDB, 11, 1, 10), + testscommon.NewPersisterFactoryHandlerMock(common.LvlDB, 11, 1, 10), ) assert.NotNil(t, err, "error expected") @@ -274,18 +284,19 @@ func TestNewStorageUnit_FromConfWrongCacheSizeVsBatchSize(t *testing.T) { } func TestNewStorageUnit_FromConfWrongCacheConfig(t *testing.T) { + t.Parallel() - storer, err := storageUnit.NewStorageUnitFromConf(storageUnit.CacheConfig{ + storer, err := storageUnit.NewStorageUnitFromConf(common.CacheConfig{ Capacity: 10, Type: "NotLRU", - }, storageUnit.DBConfig{ + }, common.DBConfig{ FilePath: "Blocks", - Type: storageUnit.LvlDB, + Type: common.LvlDB, BatchDelaySeconds: 1, MaxBatchSize: 1, MaxOpenFiles: 10, }, - testscommon.NewPersisterFactoryHandlerMock(storageUnit.LvlDB, 1, 1, 10), + testscommon.NewPersisterFactoryHandlerMock(common.LvlDB, 1, 1, 10), ) assert.NotNil(t, err, "error expected") @@ -293,10 +304,12 @@ func TestNewStorageUnit_FromConfWrongCacheConfig(t *testing.T) { } func TestNewStorageUnit_FromConfWrongDBConfig(t *testing.T) { - storer, err := storageUnit.NewStorageUnitFromConf(storageUnit.CacheConfig{ + t.Parallel() + + storer, err := storageUnit.NewStorageUnitFromConf(common.CacheConfig{ Capacity: 10, - Type: storageUnit.LRUCache, - }, storageUnit.DBConfig{ + Type: common.LRUCache, + }, common.DBConfig{ FilePath: "Blocks", Type: "NotLvlDB", }, @@ -308,17 +321,19 @@ func TestNewStorageUnit_FromConfWrongDBConfig(t *testing.T) { } func TestNewStorageUnit_FromConfLvlDBOk(t *testing.T) { - storer, err := storageUnit.NewStorageUnitFromConf(storageUnit.CacheConfig{ + t.Parallel() + + storer, err := storageUnit.NewStorageUnitFromConf(common.CacheConfig{ Capacity: 10, - Type: storageUnit.LRUCache, - }, storageUnit.DBConfig{ + Type: common.LRUCache, + }, common.DBConfig{ FilePath: "Blocks", - Type: storageUnit.LvlDB, + Type: common.LvlDB, MaxBatchSize: 1, BatchDelaySeconds: 1, MaxOpenFiles: 10, }, - testscommon.NewPersisterFactoryHandlerMock(storageUnit.LvlDB, 1, 1, 10), + testscommon.NewPersisterFactoryHandlerMock(common.LvlDB, 1, 1, 10), ) assert.Nil(t, err, "no error expected but got %s", err) @@ -328,17 +343,19 @@ func TestNewStorageUnit_FromConfLvlDBOk(t *testing.T) { } func TestNewStorageUnit_ShouldWorkLvlDB(t *testing.T) { - storer, err := storageUnit.NewStorageUnitFromConf(storageUnit.CacheConfig{ + t.Parallel() + + storer, err := storageUnit.NewStorageUnitFromConf(common.CacheConfig{ Capacity: 10, - Type: storageUnit.LRUCache, - }, storageUnit.DBConfig{ + Type: common.LRUCache, + }, common.DBConfig{ FilePath: "Blocks", - Type: storageUnit.LvlDB, + Type: common.LvlDB, BatchDelaySeconds: 1, MaxBatchSize: 1, MaxOpenFiles: 10, }, - testscommon.NewPersisterFactoryHandlerMock(storageUnit.LvlDB, 1, 1, 10), + testscommon.NewPersisterFactoryHandlerMock(common.LvlDB, 1, 1, 10), ) assert.Nil(t, err, "no error expected but got %s", err) @@ -346,72 +363,3 @@ func TestNewStorageUnit_ShouldWorkLvlDB(t *testing.T) { err = storer.DestroyUnit() assert.Nil(t, err, "no error expected destroying the persister") } - -const ( - valuesInDb = 100000 -) - -func BenchmarkStorageUnit_Put(b *testing.B) { - b.StopTimer() - s := initStorageUnit(b, 1) - defer func() { - err := s.DestroyUnit() - logError(err) - }() - b.StartTimer() - - for i := 0; i < b.N; i++ { - b.StopTimer() - nr := rand.Intn(valuesInDb) - b.StartTimer() - - err := s.Put([]byte(strconv.Itoa(nr)), []byte(strconv.Itoa(nr))) - logError(err) - } -} - -func BenchmarkStorageUnit_GetWithDataBeingPresent(b *testing.B) { - b.StopTimer() - s := initStorageUnit(b, 1) - defer func() { - err := s.DestroyUnit() - logError(err) - }() - for i := 0; i < valuesInDb; i++ { - err := s.Put([]byte(strconv.Itoa(i)), []byte(strconv.Itoa(i))) - logError(err) - } - b.StartTimer() - - for i := 0; i < b.N; i++ { - b.StopTimer() - nr := rand.Intn(valuesInDb) - b.StartTimer() - - _, err := s.Get([]byte(strconv.Itoa(nr))) - logError(err) - } -} - -func BenchmarkStorageUnit_GetWithDataNotBeingPresent(b *testing.B) { - b.StopTimer() - s := initStorageUnit(b, 1) - defer func() { - err := s.DestroyUnit() - logError(err) - }() - for i := 0; i < valuesInDb; i++ { - err := s.Put([]byte(strconv.Itoa(i)), []byte(strconv.Itoa(i))) - logError(err) - } - b.StartTimer() - - for i := 0; i < b.N; i++ { - b.StopTimer() - nr := rand.Intn(valuesInDb) + valuesInDb - b.StartTimer() - - _, err := s.Get([]byte(strconv.Itoa(nr))) - logError(err) - } -} diff --git a/testscommon/persisterFactoryHandlerMock.go b/testscommon/persisterFactoryHandlerMock.go index 3d94058b..6192100b 100644 --- a/testscommon/persisterFactoryHandlerMock.go +++ b/testscommon/persisterFactoryHandlerMock.go @@ -4,19 +4,18 @@ import ( "github.com/multiversx/mx-chain-storage-go/common" "github.com/multiversx/mx-chain-storage-go/leveldb" "github.com/multiversx/mx-chain-storage-go/memorydb" - "github.com/multiversx/mx-chain-storage-go/storageUnit" "github.com/multiversx/mx-chain-storage-go/types" ) type persisterFactoryHandlerMock struct { - dbType storageUnit.DBType + dbType common.DBType batchDelaySeconds int maxBatchSize int maxOpenFiles int } // NewPersisterFactoryHandlerMock - -func NewPersisterFactoryHandlerMock(dbType storageUnit.DBType, batchDelaySeconds int, maxBatchSize int, maxOpenFiles int) *persisterFactoryHandlerMock { +func NewPersisterFactoryHandlerMock(dbType common.DBType, batchDelaySeconds int, maxBatchSize int, maxOpenFiles int) *persisterFactoryHandlerMock { return &persisterFactoryHandlerMock{ dbType: dbType, batchDelaySeconds: batchDelaySeconds, @@ -33,11 +32,11 @@ func (mock *persisterFactoryHandlerMock) CreateWithRetries(path string) (types.P // Create - func (mock *persisterFactoryHandlerMock) Create(path string) (types.Persister, error) { switch mock.dbType { - case storageUnit.LvlDB: + case common.LvlDB: return leveldb.NewDB(path, mock.batchDelaySeconds, mock.maxBatchSize, mock.maxOpenFiles) - case storageUnit.LvlDBSerial: + case common.LvlDBSerial: return leveldb.NewSerialDB(path, mock.batchDelaySeconds, mock.maxBatchSize, mock.maxOpenFiles) - case storageUnit.MemoryDB: + case common.MemoryDB: return memorydb.New(), nil default: return nil, common.ErrNotSupportedDBType From 80d33b9914e74d53be61c0e7c39a454390bf62ed Mon Sep 17 00:00:00 2001 From: ssd04 Date: Wed, 31 Jan 2024 16:18:26 +0200 Subject: [PATCH 04/13] moved new db from config to factory --- factory/db.go | 30 +++++++++ factory/db_test.go | 63 +++++++++++++++++++ factory/storageUnit.go | 32 ++++++++++ factory/storageUnit_test.go | 105 ++++++++++++++++++++++++++++++++ storageUnit/nilStorer.go | 3 + storageUnit/storageunit.go | 27 -------- storageUnit/storageunit_test.go | 102 ------------------------------- 7 files changed, 233 insertions(+), 129 deletions(-) create mode 100644 factory/db_test.go create mode 100644 factory/storageUnit.go create mode 100644 factory/storageUnit_test.go diff --git a/factory/db.go b/factory/db.go index 7312cd2e..3dc5c20d 100644 --- a/factory/db.go +++ b/factory/db.go @@ -1 +1,31 @@ package factory + +import ( + "github.com/multiversx/mx-chain-storage-go/common" + "github.com/multiversx/mx-chain-storage-go/leveldb" + "github.com/multiversx/mx-chain-storage-go/memorydb" + "github.com/multiversx/mx-chain-storage-go/types" +) + +// ArgDB is a structure that is used to create a new storage.Persister implementation +type ArgDB struct { + DBType common.DBType + Path string + BatchDelaySeconds int + MaxBatchSize int + MaxOpenFiles int +} + +// NewDB creates a new database from database config +func NewDB(argDB ArgDB) (types.Persister, error) { + switch argDB.DBType { + case common.LvlDB: + return leveldb.NewDB(argDB.Path, argDB.BatchDelaySeconds, argDB.MaxBatchSize, argDB.MaxOpenFiles) + case common.LvlDBSerial: + return leveldb.NewSerialDB(argDB.Path, argDB.BatchDelaySeconds, argDB.MaxBatchSize, argDB.MaxOpenFiles) + case common.MemoryDB: + return memorydb.New(), nil + default: + return nil, common.ErrNotSupportedDBType + } +} diff --git a/factory/db_test.go b/factory/db_test.go new file mode 100644 index 00000000..73a65719 --- /dev/null +++ b/factory/db_test.go @@ -0,0 +1,63 @@ +package factory_test + +import ( + "testing" + + "github.com/multiversx/mx-chain-storage-go/common" + "github.com/multiversx/mx-chain-storage-go/factory" + "github.com/stretchr/testify/assert" +) + +func TestCreateDBFromConfWrongType(t *testing.T) { + t.Parallel() + + argsDB := factory.ArgDB{ + DBType: "NotLvlDB", + Path: "test", + BatchDelaySeconds: 10, + MaxBatchSize: 10, + MaxOpenFiles: 10, + } + persister, err := factory.NewDB(argsDB) + + assert.NotNil(t, err, "error expected") + assert.Nil(t, persister, "persister expected to be nil, but got %s", persister) +} + +func TestCreateDBFromConfWrongFileNameLvlDB(t *testing.T) { + if testing.Short() { + t.Skip("this is not a short test") + } + + path := "" + argsDB := factory.ArgDB{ + DBType: common.LvlDB, + Path: path, + BatchDelaySeconds: 10, + MaxBatchSize: 10, + MaxOpenFiles: 10, + } + persister, err := factory.NewDB(argsDB) + assert.NotNil(t, err, "error expected") + assert.Nil(t, persister, "persister expected to be nil, but got %s", persister) +} + +func TestCreateDBFromConfLvlDBOk(t *testing.T) { + t.Parallel() + + path := t.TempDir() + + argsDB := factory.ArgDB{ + DBType: common.LvlDB, + Path: path, + BatchDelaySeconds: 10, + MaxBatchSize: 10, + MaxOpenFiles: 10, + } + persister, err := factory.NewDB(argsDB) + assert.Nil(t, err, "no error expected") + assert.NotNil(t, persister, "valid persister expected but got nil") + + err = persister.Destroy() + assert.Nil(t, err, "no error expected destroying the persister") +} diff --git a/factory/storageUnit.go b/factory/storageUnit.go new file mode 100644 index 00000000..e9305414 --- /dev/null +++ b/factory/storageUnit.go @@ -0,0 +1,32 @@ +package factory + +import ( + "github.com/multiversx/mx-chain-storage-go/common" + "github.com/multiversx/mx-chain-storage-go/storageUnit" +) + +// NewStorageUnitFromConf creates a new storage unit from a storage unit config +func NewStorageUnitFromConf(cacheConf common.CacheConfig, dbConf common.DBConfig) (*storageUnit.Unit, error) { + if dbConf.MaxBatchSize > int(cacheConf.Capacity) { + return nil, common.ErrCacheSizeIsLowerThanBatchSize + } + + cache, err := NewCache(cacheConf) + if err != nil { + return nil, err + } + + argDB := ArgDB{ + DBType: dbConf.Type, + Path: dbConf.FilePath, + BatchDelaySeconds: dbConf.BatchDelaySeconds, + MaxBatchSize: dbConf.MaxBatchSize, + MaxOpenFiles: dbConf.MaxOpenFiles, + } + db, err := NewDB(argDB) + if err != nil { + return nil, err + } + + return storageUnit.NewStorageUnit(cache, db) +} diff --git a/factory/storageUnit_test.go b/factory/storageUnit_test.go new file mode 100644 index 00000000..80bf30ce --- /dev/null +++ b/factory/storageUnit_test.go @@ -0,0 +1,105 @@ +package factory_test + +import ( + "testing" + + "github.com/multiversx/mx-chain-storage-go/common" + "github.com/multiversx/mx-chain-storage-go/factory" + "github.com/stretchr/testify/assert" +) + +func TestNewStorageUnitFromConf_WrongCacheSizeVsBatchSize(t *testing.T) { + t.Parallel() + + storer, err := factory.NewStorageUnitFromConf(common.CacheConfig{ + Capacity: 10, + Type: common.LRUCache, + }, common.DBConfig{ + FilePath: "Blocks", + Type: common.LvlDB, + MaxBatchSize: 11, + BatchDelaySeconds: 1, + MaxOpenFiles: 10, + }, + ) + + assert.NotNil(t, err, "error expected") + assert.Nil(t, storer, "storer expected to be nil but got %s", storer) +} + +func TestNewStorageUnitFromConf_WrongCacheConfig(t *testing.T) { + t.Parallel() + + storer, err := factory.NewStorageUnitFromConf(common.CacheConfig{ + Capacity: 10, + Type: "NotLRU", + }, common.DBConfig{ + FilePath: "Blocks", + Type: common.LvlDB, + BatchDelaySeconds: 1, + MaxBatchSize: 1, + MaxOpenFiles: 10, + }, + ) + + assert.NotNil(t, err, "error expected") + assert.Nil(t, storer, "storer expected to be nil but got %s", storer) +} + +func TestNewStorageUnitFromConf_WrongDBConfig(t *testing.T) { + t.Parallel() + + storer, err := factory.NewStorageUnitFromConf(common.CacheConfig{ + Capacity: 10, + Type: common.LRUCache, + }, common.DBConfig{ + FilePath: "Blocks", + Type: "NotLvlDB", + }, + ) + + assert.NotNil(t, err, "error expected") + assert.Nil(t, storer, "storer expected to be nil but got %s", storer) +} + +func TestNewStorageUnitFromConf_LvlDBOk(t *testing.T) { + t.Parallel() + + storer, err := factory.NewStorageUnitFromConf(common.CacheConfig{ + Capacity: 10, + Type: common.LRUCache, + }, common.DBConfig{ + FilePath: "Blocks", + Type: common.LvlDB, + MaxBatchSize: 1, + BatchDelaySeconds: 1, + MaxOpenFiles: 10, + }, + ) + + assert.Nil(t, err, "no error expected but got %s", err) + assert.NotNil(t, storer, "valid storer expected but got nil") + err = storer.DestroyUnit() + assert.Nil(t, err, "no error expected destroying the persister") +} + +func TestNewStorageUnitFromConf_ShouldWorkLvlDB(t *testing.T) { + t.Parallel() + + storer, err := factory.NewStorageUnitFromConf(common.CacheConfig{ + Capacity: 10, + Type: common.LRUCache, + }, common.DBConfig{ + FilePath: "Blocks", + Type: common.LvlDB, + BatchDelaySeconds: 1, + MaxBatchSize: 1, + MaxOpenFiles: 10, + }, + ) + + assert.Nil(t, err, "no error expected but got %s", err) + assert.NotNil(t, storer, "valid storer expected but got nil") + err = storer.DestroyUnit() + assert.Nil(t, err, "no error expected destroying the persister") +} diff --git a/storageUnit/nilStorer.go b/storageUnit/nilStorer.go index 2330b452..63fdcc3c 100644 --- a/storageUnit/nilStorer.go +++ b/storageUnit/nilStorer.go @@ -3,8 +3,11 @@ package storageUnit import ( "github.com/multiversx/mx-chain-core-go/data" "github.com/multiversx/mx-chain-storage-go/common" + "github.com/multiversx/mx-chain-storage-go/types" ) +var _ types.Storer = (*NilStorer)(nil) + // NilStorer resembles a disabled implementation of the Storer interface type NilStorer struct { } diff --git a/storageUnit/storageunit.go b/storageUnit/storageunit.go index a71501fd..cf01ad4b 100644 --- a/storageUnit/storageunit.go +++ b/storageUnit/storageunit.go @@ -9,7 +9,6 @@ import ( "github.com/multiversx/mx-chain-core-go/data" logger "github.com/multiversx/mx-chain-logger-go" "github.com/multiversx/mx-chain-storage-go/common" - "github.com/multiversx/mx-chain-storage-go/factory" "github.com/multiversx/mx-chain-storage-go/types" ) @@ -50,32 +49,6 @@ func NewStorageUnit(c types.Cacher, p types.Persister) (*Unit, error) { return sUnit, nil } -// NewStorageUnitFromConf creates a new storage unit from a storage unit config -func NewStorageUnitFromConf(cacheConf common.CacheConfig, dbConf common.DBConfig, persisterFactory PersisterFactoryHandler) (*Unit, error) { - var cache types.Cacher - var db types.Persister - var err error - - // TODO: if there will be a differentiation between the creation or opening of a DB, the DB could be destroyed - // in case of a failure while creating (not opening). - - if dbConf.MaxBatchSize > int(cacheConf.Capacity) { - return nil, common.ErrCacheSizeIsLowerThanBatchSize - } - - cache, err = factory.NewCache(cacheConf) - if err != nil { - return nil, err - } - - db, err = persisterFactory.CreateWithRetries(dbConf.FilePath) - if err != nil { - return nil, err - } - - return NewStorageUnit(cache, db) -} - // Put adds data to both cache and persistence medium func (u *Unit) Put(key, data []byte) error { u.lock.Lock() diff --git a/storageUnit/storageunit_test.go b/storageUnit/storageunit_test.go index 6014a52b..dd219dce 100644 --- a/storageUnit/storageunit_test.go +++ b/storageUnit/storageunit_test.go @@ -7,7 +7,6 @@ import ( "github.com/multiversx/mx-chain-storage-go/lrucache" "github.com/multiversx/mx-chain-storage-go/memorydb" "github.com/multiversx/mx-chain-storage-go/storageUnit" - "github.com/multiversx/mx-chain-storage-go/testscommon" "github.com/stretchr/testify/assert" ) @@ -262,104 +261,3 @@ func TestDestroyUnitNoError(t *testing.T) { err := s.DestroyUnit() assert.Nil(t, err, "no error expected, but got %s", err) } - -func TestNewStorageUnit_FromConfWrongCacheSizeVsBatchSize(t *testing.T) { - t.Parallel() - - storer, err := storageUnit.NewStorageUnitFromConf(common.CacheConfig{ - Capacity: 10, - Type: common.LRUCache, - }, common.DBConfig{ - FilePath: "Blocks", - Type: common.LvlDB, - MaxBatchSize: 11, - BatchDelaySeconds: 1, - MaxOpenFiles: 10, - }, - testscommon.NewPersisterFactoryHandlerMock(common.LvlDB, 11, 1, 10), - ) - - assert.NotNil(t, err, "error expected") - assert.Nil(t, storer, "storer expected to be nil but got %s", storer) -} - -func TestNewStorageUnit_FromConfWrongCacheConfig(t *testing.T) { - t.Parallel() - - storer, err := storageUnit.NewStorageUnitFromConf(common.CacheConfig{ - Capacity: 10, - Type: "NotLRU", - }, common.DBConfig{ - FilePath: "Blocks", - Type: common.LvlDB, - BatchDelaySeconds: 1, - MaxBatchSize: 1, - MaxOpenFiles: 10, - }, - testscommon.NewPersisterFactoryHandlerMock(common.LvlDB, 1, 1, 10), - ) - - assert.NotNil(t, err, "error expected") - assert.Nil(t, storer, "storer expected to be nil but got %s", storer) -} - -func TestNewStorageUnit_FromConfWrongDBConfig(t *testing.T) { - t.Parallel() - - storer, err := storageUnit.NewStorageUnitFromConf(common.CacheConfig{ - Capacity: 10, - Type: common.LRUCache, - }, common.DBConfig{ - FilePath: "Blocks", - Type: "NotLvlDB", - }, - testscommon.NewPersisterFactoryHandlerMock("NotLvlDB", 0, 0, 0), - ) - - assert.NotNil(t, err, "error expected") - assert.Nil(t, storer, "storer expected to be nil but got %s", storer) -} - -func TestNewStorageUnit_FromConfLvlDBOk(t *testing.T) { - t.Parallel() - - storer, err := storageUnit.NewStorageUnitFromConf(common.CacheConfig{ - Capacity: 10, - Type: common.LRUCache, - }, common.DBConfig{ - FilePath: "Blocks", - Type: common.LvlDB, - MaxBatchSize: 1, - BatchDelaySeconds: 1, - MaxOpenFiles: 10, - }, - testscommon.NewPersisterFactoryHandlerMock(common.LvlDB, 1, 1, 10), - ) - - assert.Nil(t, err, "no error expected but got %s", err) - assert.NotNil(t, storer, "valid storer expected but got nil") - err = storer.DestroyUnit() - assert.Nil(t, err, "no error expected destroying the persister") -} - -func TestNewStorageUnit_ShouldWorkLvlDB(t *testing.T) { - t.Parallel() - - storer, err := storageUnit.NewStorageUnitFromConf(common.CacheConfig{ - Capacity: 10, - Type: common.LRUCache, - }, common.DBConfig{ - FilePath: "Blocks", - Type: common.LvlDB, - BatchDelaySeconds: 1, - MaxBatchSize: 1, - MaxOpenFiles: 10, - }, - testscommon.NewPersisterFactoryHandlerMock(common.LvlDB, 1, 1, 10), - ) - - assert.Nil(t, err, "no error expected but got %s", err) - assert.NotNil(t, storer, "valid storer expected but got nil") - err = storer.DestroyUnit() - assert.Nil(t, err, "no error expected destroying the persister") -} From 5c126467749cc074cbf5b0fc468e951d68cf7ece Mon Sep 17 00:00:00 2001 From: ssd04 Date: Wed, 31 Jan 2024 16:26:08 +0200 Subject: [PATCH 05/13] remove persister factory handler --- factory/cache.go | 1 - storageUnit/storageunit.go | 7 ---- testscommon/persisterFactoryHandlerMock.go | 49 ---------------------- 3 files changed, 57 deletions(-) delete mode 100644 testscommon/persisterFactoryHandlerMock.go diff --git a/factory/cache.go b/factory/cache.go index 6843a3c5..634e8968 100644 --- a/factory/cache.go +++ b/factory/cache.go @@ -46,7 +46,6 @@ func NewCache(config common.CacheConfig) (types.Cacher, error) { if err != nil { return nil, err } - // add other implementations if required default: return nil, common.ErrNotSupportedCacheType } diff --git a/storageUnit/storageunit.go b/storageUnit/storageunit.go index cf01ad4b..38181b45 100644 --- a/storageUnit/storageunit.go +++ b/storageUnit/storageunit.go @@ -16,13 +16,6 @@ var _ types.Storer = (*Unit)(nil) var log = logger.GetOrCreate("storage/storageUnit") -// PersisterFactoryHandler defines the behaviour of a component which is able to create persisters -type PersisterFactoryHandler interface { - Create(path string) (types.Persister, error) - CreateWithRetries(path string) (types.Persister, error) - IsInterfaceNil() bool -} - // Unit represents a storer's data bank // holding the cache and persistence unit type Unit struct { diff --git a/testscommon/persisterFactoryHandlerMock.go b/testscommon/persisterFactoryHandlerMock.go deleted file mode 100644 index 6192100b..00000000 --- a/testscommon/persisterFactoryHandlerMock.go +++ /dev/null @@ -1,49 +0,0 @@ -package testscommon - -import ( - "github.com/multiversx/mx-chain-storage-go/common" - "github.com/multiversx/mx-chain-storage-go/leveldb" - "github.com/multiversx/mx-chain-storage-go/memorydb" - "github.com/multiversx/mx-chain-storage-go/types" -) - -type persisterFactoryHandlerMock struct { - dbType common.DBType - batchDelaySeconds int - maxBatchSize int - maxOpenFiles int -} - -// NewPersisterFactoryHandlerMock - -func NewPersisterFactoryHandlerMock(dbType common.DBType, batchDelaySeconds int, maxBatchSize int, maxOpenFiles int) *persisterFactoryHandlerMock { - return &persisterFactoryHandlerMock{ - dbType: dbType, - batchDelaySeconds: batchDelaySeconds, - maxBatchSize: maxBatchSize, - maxOpenFiles: maxOpenFiles, - } -} - -// CreateWithRetries - -func (mock *persisterFactoryHandlerMock) CreateWithRetries(path string) (types.Persister, error) { - return mock.Create(path) -} - -// Create - -func (mock *persisterFactoryHandlerMock) Create(path string) (types.Persister, error) { - switch mock.dbType { - case common.LvlDB: - return leveldb.NewDB(path, mock.batchDelaySeconds, mock.maxBatchSize, mock.maxOpenFiles) - case common.LvlDBSerial: - return leveldb.NewSerialDB(path, mock.batchDelaySeconds, mock.maxBatchSize, mock.maxOpenFiles) - case common.MemoryDB: - return memorydb.New(), nil - default: - return nil, common.ErrNotSupportedDBType - } -} - -// IsInterfaceNil - -func (mock *persisterFactoryHandlerMock) IsInterfaceNil() bool { - return mock == nil -} From d4bdb73e1abd119b6990f19347312dfccde8768b Mon Sep 17 00:00:00 2001 From: ssd04 Date: Thu, 22 Feb 2024 11:55:29 +0200 Subject: [PATCH 06/13] fixes after review --- factory/cache.go | 18 +----- factory/cache_test.go | 92 +++++++++++++++++++++++++---- factory/db_test.go | 132 +++++++++++++++++++++++++++--------------- 3 files changed, 170 insertions(+), 72 deletions(-) diff --git a/factory/cache.go b/factory/cache.go index 634e8968..77126272 100644 --- a/factory/cache.go +++ b/factory/cache.go @@ -21,16 +21,13 @@ func NewCache(config common.CacheConfig) (types.Cacher, error) { shards := config.Shards sizeInBytes := config.SizeInBytes - var cacher types.Cacher - var err error - switch cacheType { case common.LRUCache: if sizeInBytes != 0 { return nil, common.ErrLRUCacheWithProvidedSize } - cacher, err = lrucache.NewCache(int(capacity)) + return lrucache.NewCache(int(capacity)) case common.SizeLRUCache: if sizeInBytes < minimumSizeForLRUCache { return nil, fmt.Errorf("%w, provided %d, minimum %d", @@ -40,19 +37,10 @@ func NewCache(config common.CacheConfig) (types.Cacher, error) { ) } - cacher, err = lrucache.NewCacheWithSizeInBytes(int(capacity), int64(sizeInBytes)) + return lrucache.NewCacheWithSizeInBytes(int(capacity), int64(sizeInBytes)) case common.FIFOShardedCache: - cacher, err = fifocache.NewShardedCache(int(capacity), int(shards)) - if err != nil { - return nil, err - } + return fifocache.NewShardedCache(int(capacity), int(shards)) default: return nil, common.ErrNotSupportedCacheType } - - if err != nil { - return nil, err - } - - return cacher, nil } diff --git a/factory/cache_test.go b/factory/cache_test.go index 9e768c0a..18b4496c 100644 --- a/factory/cache_test.go +++ b/factory/cache_test.go @@ -1,25 +1,97 @@ package factory_test import ( + "errors" + "fmt" "testing" "github.com/multiversx/mx-chain-storage-go/common" "github.com/multiversx/mx-chain-storage-go/factory" - "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) -func TestCreateCacheFromConfWrongType(t *testing.T) { +func TestNewCache(t *testing.T) { + t.Parallel() - cacher, err := factory.NewCache(common.CacheConfig{Type: "NotLRU", Capacity: 100, Shards: 1, SizeInBytes: 0}) + t.Run("should fail if wrong cache type", func(t *testing.T) { + cacheConf := common.CacheConfig{ + Type: "NotLRU", + Capacity: 100, + Shards: 1, + SizeInBytes: 0, + } + cacher, err := factory.NewCache(cacheConf) + require.Equal(t, common.ErrNotSupportedCacheType, err) + require.Nil(t, cacher, "cacher expected to be nil, but got %s", cacher) + }) - assert.NotNil(t, err, "error expected") - assert.Nil(t, cacher, "cacher expected to be nil, but got %s", cacher) -} + t.Run("LRUCache type, with provided size, should fail", func(t *testing.T) { + t.Parallel() + + cacheConf := common.CacheConfig{ + Type: common.LRUCache, + Capacity: 100, + Shards: 1, + SizeInBytes: 1024, + } + cacher, err := factory.NewCache(cacheConf) + require.Equal(t, common.ErrLRUCacheWithProvidedSize, err) + require.Nil(t, cacher) + }) + + t.Run("LRUCache type, should work", func(t *testing.T) { + t.Parallel() + + cacheConf := common.CacheConfig{ + Type: common.LRUCache, + Capacity: 100, + Shards: 1, + SizeInBytes: 0, + } + cacher, err := factory.NewCache(cacheConf) + require.Nil(t, err) + require.Equal(t, "*lrucache.lruCache", fmt.Sprintf("%T", cacher)) + }) + + t.Run("SizeLRUCache type, invalid size, should fail", func(t *testing.T) { + t.Parallel() + + cacheConf := common.CacheConfig{ + Type: common.SizeLRUCache, + Capacity: 100, + Shards: 1, + SizeInBytes: 512, + } + cacher, err := factory.NewCache(cacheConf) + require.True(t, errors.Is(err, common.ErrLRUCacheInvalidSize)) + require.Nil(t, cacher) + }) + + t.Run("SizeLRUCache type, should work", func(t *testing.T) { + t.Parallel() -func TestCreateCacheFromConfOK(t *testing.T) { + cacheConf := common.CacheConfig{ + Type: common.SizeLRUCache, + Capacity: 100, + Shards: 1, + SizeInBytes: 1024, + } + cacher, err := factory.NewCache(cacheConf) + require.Nil(t, err) + require.Equal(t, "*lrucache.lruCache", fmt.Sprintf("%T", cacher)) + }) - cacher, err := factory.NewCache(common.CacheConfig{Type: common.LRUCache, Capacity: 10, Shards: 1, SizeInBytes: 0}) + t.Run("FIFOShardedCache type, should work", func(t *testing.T) { + t.Parallel() - assert.Nil(t, err, "no error expected but got %s", err) - assert.NotNil(t, cacher, "valid cacher expected but got nil") + cacheConf := common.CacheConfig{ + Type: common.FIFOShardedCache, + Capacity: 100, + Shards: 1, + SizeInBytes: 1024, + } + cacher, err := factory.NewCache(cacheConf) + require.Nil(t, err) + require.Equal(t, "*fifocache.FIFOShardedCache", fmt.Sprintf("%T", cacher)) + }) } diff --git a/factory/db_test.go b/factory/db_test.go index 73a65719..95baf557 100644 --- a/factory/db_test.go +++ b/factory/db_test.go @@ -1,63 +1,101 @@ package factory_test import ( + "fmt" "testing" "github.com/multiversx/mx-chain-storage-go/common" "github.com/multiversx/mx-chain-storage-go/factory" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) -func TestCreateDBFromConfWrongType(t *testing.T) { +func TestNewDB(t *testing.T) { t.Parallel() - argsDB := factory.ArgDB{ - DBType: "NotLvlDB", - Path: "test", - BatchDelaySeconds: 10, - MaxBatchSize: 10, - MaxOpenFiles: 10, - } - persister, err := factory.NewDB(argsDB) - - assert.NotNil(t, err, "error expected") - assert.Nil(t, persister, "persister expected to be nil, but got %s", persister) -} + t.Run("wrong db type, should fail", func(t *testing.T) { + t.Parallel() -func TestCreateDBFromConfWrongFileNameLvlDB(t *testing.T) { - if testing.Short() { - t.Skip("this is not a short test") - } - - path := "" - argsDB := factory.ArgDB{ - DBType: common.LvlDB, - Path: path, - BatchDelaySeconds: 10, - MaxBatchSize: 10, - MaxOpenFiles: 10, - } - persister, err := factory.NewDB(argsDB) - assert.NotNil(t, err, "error expected") - assert.Nil(t, persister, "persister expected to be nil, but got %s", persister) -} + argsDB := factory.ArgDB{ + DBType: "NotLvlDB", + Path: "test", + BatchDelaySeconds: 10, + MaxBatchSize: 10, + MaxOpenFiles: 10, + } + persister, err := factory.NewDB(argsDB) -func TestCreateDBFromConfLvlDBOk(t *testing.T) { - t.Parallel() + require.Equal(t, common.ErrNotSupportedDBType, err) + require.Nil(t, persister) + }) + + t.Run("wrong file path, should fail", func(t *testing.T) { + t.Parallel() + + path := "" + argsDB := factory.ArgDB{ + DBType: common.LvlDB, + Path: path, + BatchDelaySeconds: 10, + MaxBatchSize: 10, + MaxOpenFiles: 10, + } + persister, err := factory.NewDB(argsDB) + assert.NotNil(t, err) + assert.Nil(t, persister) + }) + + t.Run("LvlDB type, should work", func(t *testing.T) { + t.Parallel() + + path := t.TempDir() + + argsDB := factory.ArgDB{ + DBType: common.LvlDB, + Path: path, + BatchDelaySeconds: 10, + MaxBatchSize: 10, + MaxOpenFiles: 10, + } + persister, err := factory.NewDB(argsDB) + require.Nil(t, err) + require.Equal(t, "*leveldb.DB", fmt.Sprintf("%T", persister)) + }) + + t.Run("LvlDBSerial type, should work", func(t *testing.T) { + t.Parallel() + + path := t.TempDir() + + argsDB := factory.ArgDB{ + DBType: common.LvlDBSerial, + Path: path, + BatchDelaySeconds: 10, + MaxBatchSize: 10, + MaxOpenFiles: 10, + } + persister, err := factory.NewDB(argsDB) + require.Nil(t, err) + require.Equal(t, "*leveldb.SerialDB", fmt.Sprintf("%T", persister)) + }) + + t.Run("MemoryDB type, should work", func(t *testing.T) { + t.Parallel() + + path := t.TempDir() + + argsDB := factory.ArgDB{ + DBType: common.MemoryDB, + Path: path, + BatchDelaySeconds: 10, + MaxBatchSize: 10, + MaxOpenFiles: 10, + } + persister, err := factory.NewDB(argsDB) + require.Nil(t, err) + require.Equal(t, "*memorydb.DB", fmt.Sprintf("%T", persister)) + }) +} - path := t.TempDir() - - argsDB := factory.ArgDB{ - DBType: common.LvlDB, - Path: path, - BatchDelaySeconds: 10, - MaxBatchSize: 10, - MaxOpenFiles: 10, - } - persister, err := factory.NewDB(argsDB) - assert.Nil(t, err, "no error expected") - assert.NotNil(t, persister, "valid persister expected but got nil") - - err = persister.Destroy() - assert.Nil(t, err, "no error expected destroying the persister") +func TestNewDB_WrongFileNameLvlDB(t *testing.T) { } From 91deb2ae66b69b893b1137f13209038b70277ad9 Mon Sep 17 00:00:00 2001 From: ssd04 Date: Thu, 22 Feb 2024 11:56:38 +0200 Subject: [PATCH 07/13] remove unused test --- factory/db_test.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/factory/db_test.go b/factory/db_test.go index 95baf557..bde77d5e 100644 --- a/factory/db_test.go +++ b/factory/db_test.go @@ -96,6 +96,3 @@ func TestNewDB(t *testing.T) { require.Equal(t, "*memorydb.DB", fmt.Sprintf("%T", persister)) }) } - -func TestNewDB_WrongFileNameLvlDB(t *testing.T) { -} From f6bcc32e44f5892b95bf9c43b5377a9711d8fcc5 Mon Sep 17 00:00:00 2001 From: ssd04 Date: Thu, 22 Feb 2024 14:56:46 +0200 Subject: [PATCH 08/13] added persister close in tests --- factory/db_test.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/factory/db_test.go b/factory/db_test.go index bde77d5e..42353aa3 100644 --- a/factory/db_test.go +++ b/factory/db_test.go @@ -60,6 +60,9 @@ func TestNewDB(t *testing.T) { persister, err := factory.NewDB(argsDB) require.Nil(t, err) require.Equal(t, "*leveldb.DB", fmt.Sprintf("%T", persister)) + + err = persister.Close() + require.Nil(t, err) }) t.Run("LvlDBSerial type, should work", func(t *testing.T) { @@ -77,6 +80,9 @@ func TestNewDB(t *testing.T) { persister, err := factory.NewDB(argsDB) require.Nil(t, err) require.Equal(t, "*leveldb.SerialDB", fmt.Sprintf("%T", persister)) + + err = persister.Close() + require.Nil(t, err) }) t.Run("MemoryDB type, should work", func(t *testing.T) { @@ -94,5 +100,8 @@ func TestNewDB(t *testing.T) { persister, err := factory.NewDB(argsDB) require.Nil(t, err) require.Equal(t, "*memorydb.DB", fmt.Sprintf("%T", persister)) + + err = persister.Close() + require.Nil(t, err) }) } From 359c431fc0a30e5ca87f79129612c6ec387444e5 Mon Sep 17 00:00:00 2001 From: Sorin Stanculeanu Date: Thu, 21 Mar 2024 17:03:39 +0200 Subject: [PATCH 09/13] update mx-chain-core-go --- go.mod | 4 ++-- go.sum | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 3afd9b44..2d0abdc2 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.20 require ( github.com/hashicorp/golang-lru v0.6.0 github.com/multiversx/concurrent-map v0.1.4 - github.com/multiversx/mx-chain-core-go v1.2.19-0.20240129082057-a76d0c995cf2 + github.com/multiversx/mx-chain-core-go v1.2.19-0.20240307143014-b07a6b96d3e2 github.com/multiversx/mx-chain-logger-go v1.0.14-0.20240129144507-d00e967c890c github.com/stretchr/testify v1.7.2 github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d @@ -20,7 +20,7 @@ require ( github.com/mr-tron/base58 v1.2.0 // indirect github.com/pelletier/go-toml v1.9.3 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - golang.org/x/crypto v0.3.0 // indirect + golang.org/x/net v0.2.0 // indirect golang.org/x/sys v0.2.0 // indirect google.golang.org/protobuf v1.28.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index eb3db30d..29efdaad 100644 --- a/go.sum +++ b/go.sum @@ -41,8 +41,8 @@ github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/multiversx/concurrent-map v0.1.4 h1:hdnbM8VE4b0KYJaGY5yJS2aNIW9TFFsUYwbO0993uPI= github.com/multiversx/concurrent-map v0.1.4/go.mod h1:8cWFRJDOrWHOTNSqgYCUvwT7c7eFQ4U2vKMOp4A/9+o= -github.com/multiversx/mx-chain-core-go v1.2.19-0.20240129082057-a76d0c995cf2 h1:pFh9bwOTRgW173aHqA8Bmax+jYzLnRyXqRvi5alF7V4= -github.com/multiversx/mx-chain-core-go v1.2.19-0.20240129082057-a76d0c995cf2/go.mod h1:B5zU4MFyJezmEzCsAHE9YNULmGCm2zbPHvl9hazNxmE= +github.com/multiversx/mx-chain-core-go v1.2.19-0.20240307143014-b07a6b96d3e2 h1:GpVNeez6WEdebAkMjhLAKsjexo14HddlO/Q/gygNuRE= +github.com/multiversx/mx-chain-core-go v1.2.19-0.20240307143014-b07a6b96d3e2/go.mod h1:B5zU4MFyJezmEzCsAHE9YNULmGCm2zbPHvl9hazNxmE= github.com/multiversx/mx-chain-logger-go v1.0.14-0.20240129144507-d00e967c890c h1:QIUOn8FgNRa5cir4BCWHZi/Qcr6Gg0eGNhns4+jy6+k= github.com/multiversx/mx-chain-logger-go v1.0.14-0.20240129144507-d00e967c890c/go.mod h1:fH/fR/GEBsDjPkBoZDVJMoYo2HhlA7++DP6QfITJ1N8= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= @@ -75,7 +75,6 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.3.0 h1:a06MkbcxBrEFc0w0QIZWXrH/9cCX6KJyWbBOIwAn+7A= -golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -88,6 +87,7 @@ golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.2.0 h1:sZfSu1wtKLGlWI4ZZayP0ck9Y73K1ynO6gqzTdBVdPU= +golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= From 3974ec1d64747a33ca672982a1e3d084be789637 Mon Sep 17 00:00:00 2001 From: Sorin Stanculeanu Date: Thu, 21 Mar 2024 17:06:23 +0200 Subject: [PATCH 10/13] update mx-chain-core-go --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 2d0abdc2..1886bbb0 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.20 require ( github.com/hashicorp/golang-lru v0.6.0 github.com/multiversx/concurrent-map v0.1.4 - github.com/multiversx/mx-chain-core-go v1.2.19-0.20240307143014-b07a6b96d3e2 + github.com/multiversx/mx-chain-core-go v1.2.19-0.20240321150532-5960a8922b18 github.com/multiversx/mx-chain-logger-go v1.0.14-0.20240129144507-d00e967c890c github.com/stretchr/testify v1.7.2 github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d diff --git a/go.sum b/go.sum index 29efdaad..6ef9512f 100644 --- a/go.sum +++ b/go.sum @@ -41,8 +41,8 @@ github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/multiversx/concurrent-map v0.1.4 h1:hdnbM8VE4b0KYJaGY5yJS2aNIW9TFFsUYwbO0993uPI= github.com/multiversx/concurrent-map v0.1.4/go.mod h1:8cWFRJDOrWHOTNSqgYCUvwT7c7eFQ4U2vKMOp4A/9+o= -github.com/multiversx/mx-chain-core-go v1.2.19-0.20240307143014-b07a6b96d3e2 h1:GpVNeez6WEdebAkMjhLAKsjexo14HddlO/Q/gygNuRE= -github.com/multiversx/mx-chain-core-go v1.2.19-0.20240307143014-b07a6b96d3e2/go.mod h1:B5zU4MFyJezmEzCsAHE9YNULmGCm2zbPHvl9hazNxmE= +github.com/multiversx/mx-chain-core-go v1.2.19-0.20240321150532-5960a8922b18 h1:hytqre8g+NIHsq/Kxl/lwIykHna57Gv+E38tt4K5A9I= +github.com/multiversx/mx-chain-core-go v1.2.19-0.20240321150532-5960a8922b18/go.mod h1:B5zU4MFyJezmEzCsAHE9YNULmGCm2zbPHvl9hazNxmE= github.com/multiversx/mx-chain-logger-go v1.0.14-0.20240129144507-d00e967c890c h1:QIUOn8FgNRa5cir4BCWHZi/Qcr6Gg0eGNhns4+jy6+k= github.com/multiversx/mx-chain-logger-go v1.0.14-0.20240129144507-d00e967c890c/go.mod h1:fH/fR/GEBsDjPkBoZDVJMoYo2HhlA7++DP6QfITJ1N8= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= From dcb8e6e0370f3aa5773484abbd4faaad2c9f6ab6 Mon Sep 17 00:00:00 2001 From: Sorin Stanculeanu Date: Wed, 8 May 2024 10:35:49 +0300 Subject: [PATCH 11/13] updated dependencies after merge --- go.mod | 6 +++--- go.sum | 10 +++++----- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/go.mod b/go.mod index f5cfdc50..efd58012 100644 --- a/go.mod +++ b/go.mod @@ -5,8 +5,8 @@ go 1.20 require ( github.com/hashicorp/golang-lru v0.6.0 github.com/multiversx/concurrent-map v0.1.4 - github.com/multiversx/mx-chain-core-go v1.2.19 - github.com/multiversx/mx-chain-logger-go v1.0.14 + github.com/multiversx/mx-chain-core-go v1.2.21-0.20240508071047-fefea5737840 + github.com/multiversx/mx-chain-logger-go v1.0.15-0.20240508072523-3f00a726af57 github.com/stretchr/testify v1.7.2 github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d ) @@ -20,7 +20,7 @@ require ( github.com/mr-tron/base58 v1.2.0 // indirect github.com/pelletier/go-toml v1.9.3 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - golang.org/x/crypto v0.3.0 // indirect + golang.org/x/net v0.2.0 // indirect golang.org/x/sys v0.2.0 // indirect google.golang.org/protobuf v1.28.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index 56657cc6..8bc68f9a 100644 --- a/go.sum +++ b/go.sum @@ -41,10 +41,10 @@ github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/multiversx/concurrent-map v0.1.4 h1:hdnbM8VE4b0KYJaGY5yJS2aNIW9TFFsUYwbO0993uPI= github.com/multiversx/concurrent-map v0.1.4/go.mod h1:8cWFRJDOrWHOTNSqgYCUvwT7c7eFQ4U2vKMOp4A/9+o= -github.com/multiversx/mx-chain-core-go v1.2.19 h1:2BaVHkB0tro3cjs5ay2pmLup1loCV0e1p9jV5QW0xqc= -github.com/multiversx/mx-chain-core-go v1.2.19/go.mod h1:B5zU4MFyJezmEzCsAHE9YNULmGCm2zbPHvl9hazNxmE= -github.com/multiversx/mx-chain-logger-go v1.0.14 h1:PRMpAvXE7Nec2d//QNmbYfKVHMomOKmcN4UXurQWX9o= -github.com/multiversx/mx-chain-logger-go v1.0.14/go.mod h1:bDfHSdwqIimn7Gp8w+SH5KlDuGzJ//nlyEANAaTSc3o= +github.com/multiversx/mx-chain-core-go v1.2.21-0.20240508071047-fefea5737840 h1:2mCrTUmbbA+Xv4UifZY9xptrGjcJBcJ2wavSb4FwejU= +github.com/multiversx/mx-chain-core-go v1.2.21-0.20240508071047-fefea5737840/go.mod h1:B5zU4MFyJezmEzCsAHE9YNULmGCm2zbPHvl9hazNxmE= +github.com/multiversx/mx-chain-logger-go v1.0.15-0.20240508072523-3f00a726af57 h1:g9t410dqjcb7UUptbVd/H6Ua12sEzWU4v7VplyNvRZ0= +github.com/multiversx/mx-chain-logger-go v1.0.15-0.20240508072523-3f00a726af57/go.mod h1:cY6CIXpndW5g5PTPn4WzPwka/UBEf+mgw+PSY5pHGAU= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= @@ -75,7 +75,6 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.3.0 h1:a06MkbcxBrEFc0w0QIZWXrH/9cCX6KJyWbBOIwAn+7A= -golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -88,6 +87,7 @@ golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.2.0 h1:sZfSu1wtKLGlWI4ZZayP0ck9Y73K1ynO6gqzTdBVdPU= +golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= From aa7fb322ebdf867a3b48efeeebd74466ff52ccb2 Mon Sep 17 00:00:00 2001 From: Sorin Stanculeanu Date: Thu, 25 Jul 2024 10:07:53 +0300 Subject: [PATCH 12/13] updated deps after merge --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index efd58012..5037a5b7 100644 --- a/go.mod +++ b/go.mod @@ -5,8 +5,8 @@ go 1.20 require ( github.com/hashicorp/golang-lru v0.6.0 github.com/multiversx/concurrent-map v0.1.4 - github.com/multiversx/mx-chain-core-go v1.2.21-0.20240508071047-fefea5737840 - github.com/multiversx/mx-chain-logger-go v1.0.15-0.20240508072523-3f00a726af57 + github.com/multiversx/mx-chain-core-go v1.2.21-0.20240725065431-6e9bfee5a4c6 + github.com/multiversx/mx-chain-logger-go v1.0.15-0.20240725065747-176bd697c775 github.com/stretchr/testify v1.7.2 github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d ) diff --git a/go.sum b/go.sum index 8bc68f9a..d5036d80 100644 --- a/go.sum +++ b/go.sum @@ -41,10 +41,10 @@ github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/multiversx/concurrent-map v0.1.4 h1:hdnbM8VE4b0KYJaGY5yJS2aNIW9TFFsUYwbO0993uPI= github.com/multiversx/concurrent-map v0.1.4/go.mod h1:8cWFRJDOrWHOTNSqgYCUvwT7c7eFQ4U2vKMOp4A/9+o= -github.com/multiversx/mx-chain-core-go v1.2.21-0.20240508071047-fefea5737840 h1:2mCrTUmbbA+Xv4UifZY9xptrGjcJBcJ2wavSb4FwejU= -github.com/multiversx/mx-chain-core-go v1.2.21-0.20240508071047-fefea5737840/go.mod h1:B5zU4MFyJezmEzCsAHE9YNULmGCm2zbPHvl9hazNxmE= -github.com/multiversx/mx-chain-logger-go v1.0.15-0.20240508072523-3f00a726af57 h1:g9t410dqjcb7UUptbVd/H6Ua12sEzWU4v7VplyNvRZ0= -github.com/multiversx/mx-chain-logger-go v1.0.15-0.20240508072523-3f00a726af57/go.mod h1:cY6CIXpndW5g5PTPn4WzPwka/UBEf+mgw+PSY5pHGAU= +github.com/multiversx/mx-chain-core-go v1.2.21-0.20240725065431-6e9bfee5a4c6 h1:Q7uUjTYTrt8Mw9oq5JWPv+WHhpxHTv6lhZZlhPuNcoQ= +github.com/multiversx/mx-chain-core-go v1.2.21-0.20240725065431-6e9bfee5a4c6/go.mod h1:B5zU4MFyJezmEzCsAHE9YNULmGCm2zbPHvl9hazNxmE= +github.com/multiversx/mx-chain-logger-go v1.0.15-0.20240725065747-176bd697c775 h1:a8LOfz3p4MQfRtbF00rGDAJiebziwtSfVmBHIaHBDdY= +github.com/multiversx/mx-chain-logger-go v1.0.15-0.20240725065747-176bd697c775/go.mod h1:owPYyrK7RcsLx9eOCAZQ22fIyW6BE7ttJr4XIhFIbQw= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= From 47c2e058979d157760c8cd91124a2fe49ba32ee7 Mon Sep 17 00:00:00 2001 From: Sorin Stanculeanu Date: Mon, 29 Jul 2024 10:20:49 +0300 Subject: [PATCH 13/13] updated deps --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 5037a5b7..5e658ad6 100644 --- a/go.mod +++ b/go.mod @@ -5,8 +5,8 @@ go 1.20 require ( github.com/hashicorp/golang-lru v0.6.0 github.com/multiversx/concurrent-map v0.1.4 - github.com/multiversx/mx-chain-core-go v1.2.21-0.20240725065431-6e9bfee5a4c6 - github.com/multiversx/mx-chain-logger-go v1.0.15-0.20240725065747-176bd697c775 + github.com/multiversx/mx-chain-core-go v1.2.21 + github.com/multiversx/mx-chain-logger-go v1.0.15 github.com/stretchr/testify v1.7.2 github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d ) diff --git a/go.sum b/go.sum index d5036d80..7f61e942 100644 --- a/go.sum +++ b/go.sum @@ -41,10 +41,10 @@ github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/multiversx/concurrent-map v0.1.4 h1:hdnbM8VE4b0KYJaGY5yJS2aNIW9TFFsUYwbO0993uPI= github.com/multiversx/concurrent-map v0.1.4/go.mod h1:8cWFRJDOrWHOTNSqgYCUvwT7c7eFQ4U2vKMOp4A/9+o= -github.com/multiversx/mx-chain-core-go v1.2.21-0.20240725065431-6e9bfee5a4c6 h1:Q7uUjTYTrt8Mw9oq5JWPv+WHhpxHTv6lhZZlhPuNcoQ= -github.com/multiversx/mx-chain-core-go v1.2.21-0.20240725065431-6e9bfee5a4c6/go.mod h1:B5zU4MFyJezmEzCsAHE9YNULmGCm2zbPHvl9hazNxmE= -github.com/multiversx/mx-chain-logger-go v1.0.15-0.20240725065747-176bd697c775 h1:a8LOfz3p4MQfRtbF00rGDAJiebziwtSfVmBHIaHBDdY= -github.com/multiversx/mx-chain-logger-go v1.0.15-0.20240725065747-176bd697c775/go.mod h1:owPYyrK7RcsLx9eOCAZQ22fIyW6BE7ttJr4XIhFIbQw= +github.com/multiversx/mx-chain-core-go v1.2.21 h1:+XVKznPTlUU5EFS1A8chtS8fStW60upRIyF4Pgml19I= +github.com/multiversx/mx-chain-core-go v1.2.21/go.mod h1:B5zU4MFyJezmEzCsAHE9YNULmGCm2zbPHvl9hazNxmE= +github.com/multiversx/mx-chain-logger-go v1.0.15 h1:HlNdK8etyJyL9NQ+6mIXyKPEBo+wRqOwi3n+m2QIHXc= +github.com/multiversx/mx-chain-logger-go v1.0.15/go.mod h1:t3PRKaWB1M+i6gUfD27KXgzLJJC+mAQiN+FLlL1yoGQ= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=