feat: implement SQL lite storage (#72)

* feat: initial scaffolding

* feat: insert implementation

* feat: change column types based on the SQL lite supported types

* feat: changed schema, first version of CRUD operations implementation

* refactor: extract common parts to helper functions

* chore: add comments

* fix: linter config warning

* refactor: simplify in-memory storage and fix deadlocks in case of errors

* feat: instantiate either in memory or sql storage

* chore: add comment

* test: fully covered memstorage feat minor renames

* feat: add github.com/mattn/go-sqlite3 as direct dependency

* fix: adapt the DB schema, fix issues and add unit tests for sql storage

* feat: increase coverage

* test: add some more unit tests

* fix: group variables

* refactor: rename PersistenceFilename config parameter

* fix: rename helper function (address comment part 1)

* feat: introduce meddler

* chore: remove useless comments

* fix: revert update logic to manually constructing SQL and address comments from @joanestebanr (part 1)

* fix: change some columns types (address comment from @joanestebanr 2nd part)

* fix: use meddler to construct on fly update statement, address comments from @joanestebanr (part 3)

* feat: remove in memory storage, change block_number to BIGINT (address @joanestebanr part 4)

* fix: remove redundant code

* feat: simplify building of UPDATE clause

* rebase fix

* refactor: build base select and base delete statement dynamically (address comment from @arnaubennassar)

* test: concurrent write to the database unit test

---------

Co-authored-by: Goran Rojovic <[email protected]>

.golangci.yml

@@ -1,10 +1,7 @@
 ---
 run:
   timeout: 5m
-  skip-dirs:
-    - test
-    - mocks
-
+
 linters:
   enable:
   - whitespace # Tool for detection of leading and trailing whitespace
@@ -47,6 +44,8 @@ issues:
         linters:
           - gosec
           - lll
+  exclude-dirs:
+    - test
   include:
   - EXC0012  # EXC0012 revive: Annoying issue about not having a comment. The rare codebase has such comments
   - EXC0014  # EXC0014 revive: Annoying issue about not having a comment. The rare codebase has such comments

.mockery.yml

@@ -4,14 +4,7 @@ filename: "{{.InterfaceName | lower }}.generated.go"
 mockname: "{{.InterfaceName}}"
 outpkg: "mocks"
 packages:
-  github.com/0xPolygon/zkevm-ethtx-manager/ethtxmanager:
+  github.com/0xPolygon/zkevm-ethtx-manager/types:
     interfaces:
       EthermanInterface:
-        config:
-      StorageInterface:
-        config:
-          dir: "{{.InterfaceDir}}"
-          filename: "{{.InterfaceName | lower }}.generated.go"
-          mockname: "{{.InterfaceName}}Mock"
-          outpkg: "{{.PackageName}}"
-          inpackage: True
+        config:

common/common.go

@@ -1,8 +1,41 @@
 package common
 
+import "github.com/ethereum/go-ethereum/common"
+
 const (
 	// Base10 decimal base
 	Base10 = 10
 	// Gwei represents 1000000000 wei
 	Gwei = 1000000000
+
+	// SQLLiteDriverName is the name for the SQL lite driver
+	SQLLiteDriverName = "sqlite3"
 )
+
+// ToAddressPtr converts a string to a common.Address pointer or returns nil if empty.
+func ToAddressPtr(addr string) *common.Address {
+	if addr == "" {
+		return nil
+	}
+
+	address := common.HexToAddress(addr)
+	return &address
+}
+
+// ToUint64Ptr is a helper to create uint64 pointer
+func ToUint64Ptr(v uint64) *uint64 {
+	return &v
+}
+
+// SlicePtrsToSlice converts a slice of pointers to a slice of values.
+func SlicePtrsToSlice[T any](ptrSlice []*T) []T {
+	// Create a new slice to hold the values
+	res := make([]T, len(ptrSlice))
+	// Dereference each pointer and add the value to the result slice
+	for i, ptr := range ptrSlice {
+		if ptr != nil {
+			res[i] = *ptr
+		}
+	}
+	return res
+}

ethtxmanager/config.go

@@ -56,10 +56,10 @@ type Config struct {
 	// max gas price limit: 110
 	// tx gas price = 110
 	MaxGasPriceLimit uint64 `mapstructure:"MaxGasPriceLimit"`
-	// PersistenceFilename is the filename to store the memory storage
-	PersistenceFilename string `mapstructure:"PersistenceFilename"`
+	// DBPath is the path of the SQL database
+	DBPath string `mapstructure:"DBPath"`
 	// ReadPendingL1Txs is a flag to enable the reading of pending L1 txs
-	// It can only be enabled if PersistenceFilename is empty
+	// It can only be enabled if DBPath is empty
 	ReadPendingL1Txs bool `mapstructure:"ReadPendingL1Txs"`
 	// Etherman configuration
 	Etherman etherman.Config `mapstructure:"Etherman"`