Merge branch 'rc/v1.7.next1' into selection-by-ppu

common/config.go

@@ -0,0 +1,39 @@
+package common
+
+import (
+	"encoding/json"
+
+	logger "github.com/multiversx/mx-chain-logger-go"
+)
+
+var log = logger.GetOrCreate("common")
+
+// CacheConfig holds the configurable elements of a cache
+type CacheConfig struct {
+	Name                 string
+	Type                 CacheType
+	SizeInBytes          uint64
+	SizeInBytesPerSender uint32
+	Capacity             uint32
+	SizePerSender        uint32
+	Shards               uint32
+}
+
+// String returns a readable representation of the object
+func (config *CacheConfig) String() string {
+	bytes, err := json.Marshal(config)
+	if err != nil {
+		log.Error("CacheConfig.String()", "err", err)
+	}
+
+	return string(bytes)
+}
+
+// DBConfig holds the configurable elements of a database
+type DBConfig struct {
+	FilePath          string
+	Type              DBType
+	BatchDelaySeconds int
+	MaxBatchSize      int
+	MaxOpenFiles      int
+}

common/constants.go

@@ -0,0 +1,37 @@
+package common
+
+import "time"
+
+// CacheType represents the type of the supported caches
+type CacheType string
+
+// Cache types that are currently supported
+const (
+	LRUCache         CacheType = "LRU"
+	SizeLRUCache     CacheType = "SizeLRU"
+	FIFOShardedCache CacheType = "FIFOSharded"
+)
+
+// DBType represents the type of the supported databases
+type DBType string
+
+// DB types that are currently supported
+const (
+	LvlDB       DBType = "LvlDB"
+	LvlDBSerial DBType = "LvlDBSerial"
+	MemoryDB    DBType = "MemoryDB"
+)
+
+// ShardIDProviderType represents the type for the supported shard id provider
+type ShardIDProviderType string
+
+// Shard id provider types that are currently supported
+const (
+	BinarySplit ShardIDProviderType = "BinarySplit"
+)
+
+// MaxRetriesToCreateDB represents the maximum number of times to try to create DB if it failed
+const MaxRetriesToCreateDB = 10
+
+// SleepTimeBetweenCreateDBRetries represents the number of seconds to sleep between DB creates
+const SleepTimeBetweenCreateDBRetries = 5 * time.Second

factory/cache.go

@@ -0,0 +1,46 @@
+package factory
+
+import (
+	"fmt"
+
+	"github.com/multiversx/mx-chain-storage-go/common"
+	"github.com/multiversx/mx-chain-storage-go/fifocache"
+	"github.com/multiversx/mx-chain-storage-go/lrucache"
+	"github.com/multiversx/mx-chain-storage-go/monitoring"
+	"github.com/multiversx/mx-chain-storage-go/types"
+)
+
+const minimumSizeForLRUCache = 1024
+
+// NewCache creates a new cache from a cache config
+func NewCache(config common.CacheConfig) (types.Cacher, error) {
+	monitoring.MonitorNewCache(config.Name, config.SizeInBytes)
+
+	cacheType := config.Type
+	capacity := config.Capacity
+	shards := config.Shards
+	sizeInBytes := config.SizeInBytes
+
+	switch cacheType {
+	case common.LRUCache:
+		if sizeInBytes != 0 {
+			return nil, common.ErrLRUCacheWithProvidedSize
+		}
+
+		return lrucache.NewCache(int(capacity))
+	case common.SizeLRUCache:
+		if sizeInBytes < minimumSizeForLRUCache {
+			return nil, fmt.Errorf("%w, provided %d, minimum %d",
+				common.ErrLRUCacheInvalidSize,
+				sizeInBytes,
+				minimumSizeForLRUCache,
+			)
+		}
+
+		return lrucache.NewCacheWithSizeInBytes(int(capacity), int64(sizeInBytes))
+	case common.FIFOShardedCache:
+		return fifocache.NewShardedCache(int(capacity), int(shards))
+	default:
+		return nil, common.ErrNotSupportedCacheType
+	}
+}

factory/cache_test.go

@@ -0,0 +1,97 @@
+package factory_test
+
+import (
+	"errors"
+	"fmt"
+	"testing"
+
+	"github.com/multiversx/mx-chain-storage-go/common"
+	"github.com/multiversx/mx-chain-storage-go/factory"
+	"github.com/stretchr/testify/require"
+)
+
+func TestNewCache(t *testing.T) {
+	t.Parallel()
+
+	t.Run("should fail if wrong cache type", func(t *testing.T) {
+		cacheConf := common.CacheConfig{
+			Type:        "NotLRU",
+			Capacity:    100,
+			Shards:      1,
+			SizeInBytes: 0,
+		}
+		cacher, err := factory.NewCache(cacheConf)
+		require.Equal(t, common.ErrNotSupportedCacheType, err)
+		require.Nil(t, cacher, "cacher expected to be nil, but got %s", cacher)
+	})
+
+	t.Run("LRUCache type, with provided size, should fail", func(t *testing.T) {
+		t.Parallel()
+
+		cacheConf := common.CacheConfig{
+			Type:        common.LRUCache,
+			Capacity:    100,
+			Shards:      1,
+			SizeInBytes: 1024,
+		}
+		cacher, err := factory.NewCache(cacheConf)
+		require.Equal(t, common.ErrLRUCacheWithProvidedSize, err)
+		require.Nil(t, cacher)
+	})
+
+	t.Run("LRUCache type, should work", func(t *testing.T) {
+		t.Parallel()
+
+		cacheConf := common.CacheConfig{
+			Type:        common.LRUCache,
+			Capacity:    100,
+			Shards:      1,
+			SizeInBytes: 0,
+		}
+		cacher, err := factory.NewCache(cacheConf)
+		require.Nil(t, err)
+		require.Equal(t, "*lrucache.lruCache", fmt.Sprintf("%T", cacher))
+	})
+
+	t.Run("SizeLRUCache type, invalid size, should fail", func(t *testing.T) {
+		t.Parallel()
+
+		cacheConf := common.CacheConfig{
+			Type:        common.SizeLRUCache,
+			Capacity:    100,
+			Shards:      1,
+			SizeInBytes: 512,
+		}
+		cacher, err := factory.NewCache(cacheConf)
+		require.True(t, errors.Is(err, common.ErrLRUCacheInvalidSize))
+		require.Nil(t, cacher)
+	})
+
+	t.Run("SizeLRUCache type, should work", func(t *testing.T) {
+		t.Parallel()
+
+		cacheConf := common.CacheConfig{
+			Type:        common.SizeLRUCache,
+			Capacity:    100,
+			Shards:      1,
+			SizeInBytes: 1024,
+		}
+		cacher, err := factory.NewCache(cacheConf)
+		require.Nil(t, err)
+		require.Equal(t, "*lrucache.lruCache", fmt.Sprintf("%T", cacher))
+	})
+
+	t.Run("FIFOShardedCache type, should work", func(t *testing.T) {
+		t.Parallel()
+
+		cacheConf := common.CacheConfig{
+			Type:        common.FIFOShardedCache,
+			Capacity:    100,
+			Shards:      1,
+			SizeInBytes: 1024,
+		}
+		cacher, err := factory.NewCache(cacheConf)
+		require.Nil(t, err)
+		require.Equal(t, "*fifocache.FIFOShardedCache", fmt.Sprintf("%T", cacher))
+	})
+}

factory/db.go

@@ -0,0 +1,31 @@
+package factory
+
+import (
+	"github.com/multiversx/mx-chain-storage-go/common"
+	"github.com/multiversx/mx-chain-storage-go/leveldb"
+	"github.com/multiversx/mx-chain-storage-go/memorydb"
+	"github.com/multiversx/mx-chain-storage-go/types"
+)
+
+// ArgDB is a structure that is used to create a new storage.Persister implementation
+type ArgDB struct {
+	DBType            common.DBType
+	Path              string
+	BatchDelaySeconds int
+	MaxBatchSize      int
+	MaxOpenFiles      int
+}
+
+// NewDB creates a new database from database config
+func NewDB(argDB ArgDB) (types.Persister, error) {
+	switch argDB.DBType {
+	case common.LvlDB:
+		return leveldb.NewDB(argDB.Path, argDB.BatchDelaySeconds, argDB.MaxBatchSize, argDB.MaxOpenFiles)
+	case common.LvlDBSerial:
+		return leveldb.NewSerialDB(argDB.Path, argDB.BatchDelaySeconds, argDB.MaxBatchSize, argDB.MaxOpenFiles)
+	case common.MemoryDB:
+		return memorydb.New(), nil
+	default:
+		return nil, common.ErrNotSupportedDBType
+	}
+}

factory/db_test.go

@@ -0,0 +1,107 @@
+package factory_test
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/multiversx/mx-chain-storage-go/common"
+	"github.com/multiversx/mx-chain-storage-go/factory"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestNewDB(t *testing.T) {
+	t.Parallel()
+
+	t.Run("wrong db type, should fail", func(t *testing.T) {
+		t.Parallel()
+
+		argsDB := factory.ArgDB{
+			DBType:            "NotLvlDB",
+			Path:              "test",
+			BatchDelaySeconds: 10,
+			MaxBatchSize:      10,
+			MaxOpenFiles:      10,
+		}
+		persister, err := factory.NewDB(argsDB)
+
+		require.Equal(t, common.ErrNotSupportedDBType, err)
+		require.Nil(t, persister)
+	})
+
+	t.Run("wrong file path, should fail", func(t *testing.T) {
+		t.Parallel()
+
+		path := ""
+		argsDB := factory.ArgDB{
+			DBType:            common.LvlDB,
+			Path:              path,
+			BatchDelaySeconds: 10,
+			MaxBatchSize:      10,
+			MaxOpenFiles:      10,
+		}
+		persister, err := factory.NewDB(argsDB)
+		assert.NotNil(t, err)
+		assert.Nil(t, persister)
+	})
+
+	t.Run("LvlDB type, should work", func(t *testing.T) {
+		t.Parallel()
+
+		path := t.TempDir()
+
+		argsDB := factory.ArgDB{
+			DBType:            common.LvlDB,
+			Path:              path,
+			BatchDelaySeconds: 10,
+			MaxBatchSize:      10,
+			MaxOpenFiles:      10,
+		}
+		persister, err := factory.NewDB(argsDB)
+		require.Nil(t, err)
+		require.Equal(t, "*leveldb.DB", fmt.Sprintf("%T", persister))
+
+		err = persister.Close()
+		require.Nil(t, err)
+	})
+
+	t.Run("LvlDBSerial type, should work", func(t *testing.T) {
+		t.Parallel()
+
+		path := t.TempDir()
+
+		argsDB := factory.ArgDB{
+			DBType:            common.LvlDBSerial,
+			Path:              path,
+			BatchDelaySeconds: 10,
+			MaxBatchSize:      10,
+			MaxOpenFiles:      10,
+		}
+		persister, err := factory.NewDB(argsDB)
+		require.Nil(t, err)
+		require.Equal(t, "*leveldb.SerialDB", fmt.Sprintf("%T", persister))
+
+		err = persister.Close()
+		require.Nil(t, err)
+	})
+
+	t.Run("MemoryDB type, should work", func(t *testing.T) {
+		t.Parallel()
+
+		path := t.TempDir()
+
+		argsDB := factory.ArgDB{
+			DBType:            common.MemoryDB,
+			Path:              path,
+			BatchDelaySeconds: 10,
+			MaxBatchSize:      10,
+			MaxOpenFiles:      10,
+		}
+		persister, err := factory.NewDB(argsDB)
+		require.Nil(t, err)
+		require.Equal(t, "*memorydb.DB", fmt.Sprintf("%T", persister))
+
+		err = persister.Close()
+		require.Nil(t, err)
+	})
+}

factory/storageUnit.go

@@ -0,0 +1,32 @@
+package factory
+
+import (
+	"github.com/multiversx/mx-chain-storage-go/common"
+	"github.com/multiversx/mx-chain-storage-go/storageUnit"
+)
+
+// NewStorageUnitFromConf creates a new storage unit from a storage unit config
+func NewStorageUnitFromConf(cacheConf common.CacheConfig, dbConf common.DBConfig) (*storageUnit.Unit, error) {
+	if dbConf.MaxBatchSize > int(cacheConf.Capacity) {
+		return nil, common.ErrCacheSizeIsLowerThanBatchSize
+	}
+
+	cache, err := NewCache(cacheConf)
+	if err != nil {
+		return nil, err
+	}
+
+	argDB := ArgDB{
+		DBType:            dbConf.Type,
+		Path:              dbConf.FilePath,
+		BatchDelaySeconds: dbConf.BatchDelaySeconds,
+		MaxBatchSize:      dbConf.MaxBatchSize,
+		MaxOpenFiles:      dbConf.MaxOpenFiles,
+	}
+	db, err := NewDB(argDB)
+	if err != nil {
+		return nil, err
+	}
+
+	return storageUnit.NewStorageUnit(cache, db)
+}