From ea8379f2b9502f79ed4cf426ae73251535b545cd Mon Sep 17 00:00:00 2001 From: Arnau Bennassar Date: Fri, 13 Sep 2024 15:57:48 +0200 Subject: [PATCH] Ler2block (#61) * wip * wip * WIP * decoding direct and indeirecr assets and messages works * connect everything * fix building contract scripts * fix building contract scripts * wip * WIP * tree migrated to SQLite * wip * wip * bridgesync working with sqlite * pass tests * minor cleaning * add GetBlockByLER func * handle err not found * merge develop * use memory for sqlite on the tests * increase timestamp to pass UT * review * add callbacks on db tx * lint * fix compilation * fix linter II * fix linter III * fix linter * increase linter TO * fix UTs and lint * increase linter TO * add PR requests --- Makefile | 2 +- bridgesync/bridgesync.go | 23 +- bridgesync/claimcalldata_test.go | 2 + bridgesync/downloader.go | 8 +- bridgesync/e2e_test.go | 13 +- bridgesync/migrations/bridgesync0001.sql | 42 ++ bridgesync/migrations/bridgesync0001_test.go | 61 +++ bridgesync/migrations/migrations.go | 23 + bridgesync/processor.go | 349 ++++++------- bridgesync/processor_test.go | 215 ++++++-- claimsponsor/e2e_test.go | 3 +- .../datacommittee/datacommittee_test.go | 3 +- db/interface.go | 17 + db/meddler.go | 178 +++++++ db/migrations.go | 48 ++ db/sqlite.go | 27 + db/tx.go | 60 +++ db/types/types.go | 7 + go.mod | 15 +- go.sum | 51 +- l1bridge2infoindexsync/downloader.go | 5 +- l1bridge2infoindexsync/driver.go | 4 +- l1bridge2infoindexsync/e2e_test.go | 12 +- l1infotreesync/downloader.go | 23 +- l1infotreesync/e2e_test.go | 53 +- l1infotreesync/l1infotreesync.go | 31 +- .../migrations/l1infotreesync0001.sql | 22 + l1infotreesync/migrations/migrations.go | 39 ++ l1infotreesync/processor.go | 493 ++++++------------ lastgersync/e2e_test.go | 2 +- lastgersync/evmdownloader.go | 6 +- sync/evmdriver.go | 2 +- test/helpers/aggoracle_e2e.go | 16 +- tree/appendonlytree.go | 216 ++------ tree/migrations/migrations.go | 22 + tree/migrations/tree0001.sql | 17 + tree/tree.go | 262 ++++------ tree/tree_test.go | 94 ++-- tree/types/types.go | 27 + tree/updatabletree.go | 136 ++--- 40 files changed, 1442 insertions(+), 1187 deletions(-) create mode 100644 bridgesync/migrations/bridgesync0001.sql create mode 100644 bridgesync/migrations/bridgesync0001_test.go create mode 100644 bridgesync/migrations/migrations.go create mode 100644 db/interface.go create mode 100644 db/meddler.go create mode 100644 db/migrations.go create mode 100644 db/sqlite.go create mode 100644 db/tx.go create mode 100644 db/types/types.go create mode 100644 l1infotreesync/migrations/l1infotreesync0001.sql create mode 100644 l1infotreesync/migrations/migrations.go create mode 100644 tree/migrations/migrations.go create mode 100644 tree/migrations/tree0001.sql create mode 100644 tree/types/types.go diff --git a/Makefile b/Makefile index fa71f98a..7f8ff33b 100644 --- a/Makefile +++ b/Makefile @@ -95,7 +95,7 @@ install-linter: ## Installs the linter .PHONY: lint lint: ## Runs the linter - export "GOROOT=$$(go env GOROOT)" && $$(go env GOPATH)/bin/golangci-lint run + export "GOROOT=$$(go env GOROOT)" && $$(go env GOPATH)/bin/golangci-lint run --timeout 5m $(VENV_PYTHON): rm -rf $(VENV) diff --git a/bridgesync/bridgesync.go b/bridgesync/bridgesync.go index ef8e4541..e79fba2e 100644 --- a/bridgesync/bridgesync.go +++ b/bridgesync/bridgesync.go @@ -6,6 +6,7 @@ import ( "github.com/0xPolygon/cdk/etherman" "github.com/0xPolygon/cdk/sync" + tree "github.com/0xPolygon/cdk/tree/types" "github.com/ethereum/go-ethereum/common" ) @@ -98,7 +99,7 @@ func newBridgeSync( maxRetryAttemptsAfterError int, syncFullClaims bool, ) (*BridgeSync, error) { - processor, err := newProcessor(ctx, dbPath, l1OrL2ID) + processor, err := newProcessor(dbPath, l1OrL2ID) if err != nil { return nil, err } @@ -159,12 +160,16 @@ func (s *BridgeSync) GetLastProcessedBlock(ctx context.Context) (uint64, error) return s.processor.GetLastProcessedBlock(ctx) } -func (s *BridgeSync) GetBridgeIndexByRoot(ctx context.Context, root common.Hash) (uint32, error) { - return s.processor.exitTree.GetIndexByRoot(ctx, root) +func (s *BridgeSync) GetBridgeRootByHash(ctx context.Context, root common.Hash) (tree.Root, error) { + return s.processor.exitTree.GetRootByHash(ctx, root) } -func (s *BridgeSync) GetClaimsAndBridges(ctx context.Context, fromBlock, toBlock uint64) ([]Event, error) { - return s.processor.GetClaimsAndBridges(ctx, fromBlock, toBlock) +func (s *BridgeSync) GetClaims(ctx context.Context, fromBlock, toBlock uint64) ([]Claim, error) { + return s.processor.GetClaims(ctx, fromBlock, toBlock) +} + +func (s *BridgeSync) GetBridges(ctx context.Context, fromBlock, toBlock uint64) ([]Bridge, error) { + return s.processor.GetBridges(ctx, fromBlock, toBlock) } // GetProof retrieves the Merkle proof for the given deposit count and exit root. @@ -173,3 +178,11 @@ func (s *BridgeSync) GetProof( ) ([32]common.Hash, error) { return s.processor.exitTree.GetProof(ctx, depositCount, localExitRoot) } + +func (p *processor) GetBlockByLER(ctx context.Context, ler common.Hash) (uint64, error) { + root, err := p.exitTree.GetRootByHash(ctx, ler) + if err != nil { + return 0, err + } + return root.BlockNum, nil +} diff --git a/bridgesync/claimcalldata_test.go b/bridgesync/claimcalldata_test.go index d788c2c7..1319835b 100644 --- a/bridgesync/claimcalldata_test.go +++ b/bridgesync/claimcalldata_test.go @@ -28,6 +28,7 @@ type testCase struct { func TestClaimCalldata(t *testing.T) { testCases := []testCase{} // Setup Docker L1 + log.Debug("starting docker") ctx := context.Background() msg, err := exec.Command("bash", "-l", "-c", "docker compose up -d").CombinedOutput() require.NoError(t, err, string(msg)) @@ -36,6 +37,7 @@ func TestClaimCalldata(t *testing.T) { msg, err = exec.Command("bash", "-l", "-c", "docker compose down").CombinedOutput() require.NoError(t, err, string(msg)) }() + log.Debug("docker started") client, err := ethclient.Dial("http://localhost:8545") require.NoError(t, err) privateKey, err := crypto.HexToECDSA("ac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80") diff --git a/bridgesync/downloader.go b/bridgesync/downloader.go index 3599d7de..b34267ce 100644 --- a/bridgesync/downloader.go +++ b/bridgesync/downloader.go @@ -10,7 +10,7 @@ import ( "github.com/0xPolygon/cdk-contracts-tooling/contracts/etrog/polygonzkevmbridgev2" rpcTypes "github.com/0xPolygon/cdk-rpc/types" "github.com/0xPolygon/cdk/sync" - "github.com/0xPolygon/cdk/tree" + tree "github.com/0xPolygon/cdk/tree/types" "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/accounts/abi" "github.com/ethereum/go-ethereum/accounts/abi/bind" @@ -60,6 +60,8 @@ func buildAppender(client EthClienter, bridge common.Address, syncFullClaims boo ) } b.Events = append(b.Events, Event{Bridge: &Bridge{ + BlockNum: b.Num, + BlockPos: uint64(l.Index), LeafType: bridge.LeafType, OriginNetwork: bridge.OriginNetwork, OriginAddress: bridge.OriginAddress, @@ -82,6 +84,8 @@ func buildAppender(client EthClienter, bridge common.Address, syncFullClaims boo ) } claim := &Claim{ + BlockNum: b.Num, + BlockPos: uint64(l.Index), GlobalIndex: claimEvent.GlobalIndex, OriginNetwork: claimEvent.OriginNetwork, OriginAddress: claimEvent.OriginAddress, @@ -106,6 +110,8 @@ func buildAppender(client EthClienter, bridge common.Address, syncFullClaims boo ) } claim := &Claim{ + BlockNum: b.Num, + BlockPos: uint64(l.Index), GlobalIndex: big.NewInt(int64(claimEvent.Index)), OriginNetwork: claimEvent.OriginNetwork, OriginAddress: claimEvent.OriginAddress, diff --git a/bridgesync/e2e_test.go b/bridgesync/e2e_test.go index e0901509..a19afb8d 100644 --- a/bridgesync/e2e_test.go +++ b/bridgesync/e2e_test.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "math/big" + "path" "testing" "time" @@ -47,7 +48,7 @@ func newSimulatedClient(t *testing.T, auth *bind.TransactOpts) ( func TestBridgeEventE2E(t *testing.T) { ctx := context.Background() - dbPathSyncer := t.TempDir() + dbPathSyncer := path.Join(t.TempDir(), "file::memory:?cache=shared") dbPathReorg := t.TempDir() privateKey, err := crypto.GenerateKey() require.NoError(t, err) @@ -70,6 +71,7 @@ func TestBridgeEventE2E(t *testing.T) { for i := 0; i < 100; i++ { bridge := bridgesync.Bridge{ + BlockNum: uint64(2 + i), Amount: big.NewInt(0), DepositCount: uint32(i), DestinationNetwork: 3, @@ -116,15 +118,8 @@ func TestBridgeEventE2E(t *testing.T) { // Get bridges lastBlock, err := client.Client().BlockNumber(ctx) require.NoError(t, err) - events, err := syncer.GetClaimsAndBridges(ctx, 0, lastBlock) + actualBridges, err := syncer.GetBridges(ctx, 0, lastBlock) require.NoError(t, err) - actualBridges := []bridgesync.Bridge{} - - for _, event := range events { - if event.Bridge != nil { - actualBridges = append(actualBridges, *event.Bridge) - } - } // Assert bridges require.Equal(t, expectedBridges, actualBridges) diff --git a/bridgesync/migrations/bridgesync0001.sql b/bridgesync/migrations/bridgesync0001.sql new file mode 100644 index 00000000..de90910c --- /dev/null +++ b/bridgesync/migrations/bridgesync0001.sql @@ -0,0 +1,42 @@ +-- +migrate Down +DROP TABLE IF EXISTS block; +DROP TABLE IF EXISTS claim; +DROP TABLE IF EXISTS bridge; + +-- +migrate Up +CREATE TABLE block ( + num BIGINT PRIMARY KEY +); + +CREATE TABLE bridge ( + block_num INTEGER NOT NULL REFERENCES block(num) ON DELETE CASCADE, + block_pos INTEGER NOT NULL, + leaf_type INTEGER NOT NULL, + origin_network INTEGER NOT NULL, + origin_address VARCHAR NOT NULL, + destination_network INTEGER NOT NULL, + destination_address VARCHAR NOT NULL, + amount DECIMAL(78, 0) NOT NULL, + metadata BLOB, + deposit_count INTEGER NOT NULL, + PRIMARY KEY (block_num, block_pos) +); + +CREATE TABLE claim ( + block_num INTEGER NOT NULL REFERENCES block(num) ON DELETE CASCADE, + block_pos INTEGER NOT NULL, + global_index DECIMAL(78, 0) NOT NULL, + origin_network INTEGER NOT NULL, + origin_address VARCHAR NOT NULL, + destination_address VARCHAR NOT NULL, + amount DECIMAL(78, 0) NOT NULL, + proof_local_exit_root VARCHAR, + proof_rollup_exit_root VARCHAR, + mainnet_exit_root VARCHAR, + rollup_exit_root VARCHAR, + global_exit_root VARCHAR, + destination_network INTEGER NOT NULL, + metadata BLOB, + is_message BOOLEAN, + PRIMARY KEY (block_num, block_pos) +); \ No newline at end of file diff --git a/bridgesync/migrations/bridgesync0001_test.go b/bridgesync/migrations/bridgesync0001_test.go new file mode 100644 index 00000000..d117e0e2 --- /dev/null +++ b/bridgesync/migrations/bridgesync0001_test.go @@ -0,0 +1,61 @@ +package migrations + +import ( + "context" + "path" + "testing" + + "github.com/0xPolygon/cdk/db" + "github.com/stretchr/testify/require" +) + +func Test001(t *testing.T) { + dbPath := path.Join(t.TempDir(), "file::memory:?cache=shared") + + err := RunMigrations(dbPath) + require.NoError(t, err) + db, err := db.NewSQLiteDB(dbPath) + require.NoError(t, err) + + ctx := context.Background() + tx, err := db.BeginTx(ctx, nil) + require.NoError(t, err) + + _, err = tx.Exec(` + INSERT INTO block (num) VALUES (1); + + INSERT INTO bridge ( + block_num, + block_pos, + leaf_type, + origin_network, + origin_address, + destination_network, + destination_address, + amount, + metadata, + deposit_count + ) VALUES (1, 0, 0, 0, '0x0000', 0, '0x0000', 0, NULL, 0); + + INSERT INTO claim ( + block_num, + block_pos, + global_index, + origin_network, + origin_address, + destination_address, + amount, + proof_local_exit_root, + proof_rollup_exit_root, + mainnet_exit_root, + rollup_exit_root, + global_exit_root, + destination_network, + metadata, + is_message + ) VALUES (1, 0, 0, 0, '0x0000', '0x0000', 0, '0x000,0x000', '0x000,0x000', '0x000', '0x000', '0x0', 0, NULL, FALSE); + `) + require.NoError(t, err) + err = tx.Commit() + require.NoError(t, err) +} diff --git a/bridgesync/migrations/migrations.go b/bridgesync/migrations/migrations.go new file mode 100644 index 00000000..c500ee38 --- /dev/null +++ b/bridgesync/migrations/migrations.go @@ -0,0 +1,23 @@ +package migrations + +import ( + _ "embed" + + "github.com/0xPolygon/cdk/db" + "github.com/0xPolygon/cdk/db/types" + treeMigrations "github.com/0xPolygon/cdk/tree/migrations" +) + +//go:embed bridgesync0001.sql +var mig001 string + +func RunMigrations(dbPath string) error { + migrations := []types.Migration{ + { + ID: "bridgesync0001", + SQL: mig001, + }, + } + migrations = append(migrations, treeMigrations.Migrations...) + return db.RunMigrations(dbPath, migrations) +} diff --git a/bridgesync/processor.go b/bridgesync/processor.go index 824d4afd..47b26595 100644 --- a/bridgesync/processor.go +++ b/bridgesync/processor.go @@ -2,43 +2,42 @@ package bridgesync import ( "context" + "database/sql" "encoding/binary" - "encoding/json" "errors" + "fmt" "math/big" - dbCommon "github.com/0xPolygon/cdk/common" + "github.com/0xPolygon/cdk/bridgesync/migrations" + "github.com/0xPolygon/cdk/db" "github.com/0xPolygon/cdk/log" "github.com/0xPolygon/cdk/sync" "github.com/0xPolygon/cdk/tree" + "github.com/0xPolygon/cdk/tree/types" "github.com/ethereum/go-ethereum/common" "github.com/iden3/go-iden3-crypto/keccak256" - "github.com/ledgerwatch/erigon-lib/kv" - "github.com/ledgerwatch/erigon-lib/kv/mdbx" -) - -const ( - eventsTableSufix = "-events" - lastBlockTableSufix = "-lastBlock" + "github.com/russross/meddler" + _ "modernc.org/sqlite" ) var ( // ErrBlockNotProcessed indicates that the given block(s) have not been processed yet. ErrBlockNotProcessed = errors.New("given block(s) have not been processed yet") ErrNotFound = errors.New("not found") - lastBlockKey = []byte("lb") ) // Bridge is the representation of a bridge event type Bridge struct { - LeafType uint8 - OriginNetwork uint32 - OriginAddress common.Address - DestinationNetwork uint32 - DestinationAddress common.Address - Amount *big.Int - Metadata []byte - DepositCount uint32 + BlockNum uint64 `meddler:"block_num"` + BlockPos uint64 `meddler:"block_pos"` + LeafType uint8 `meddler:"leaf_type"` + OriginNetwork uint32 `meddler:"origin_network"` + OriginAddress common.Address `meddler:"origin_address"` + DestinationNetwork uint32 `meddler:"destination_network"` + DestinationAddress common.Address `meddler:"destination_address"` + Amount *big.Int `meddler:"amount,bigint"` + Metadata []byte `meddler:"metadata"` + DepositCount uint32 `meddler:"deposit_count"` } // Hash returns the hash of the bridge event as expected by the exit tree @@ -71,200 +70,178 @@ func (b *Bridge) Hash() common.Hash { // Claim representation of a claim event type Claim struct { - // From claim event - GlobalIndex *big.Int - OriginNetwork uint32 - OriginAddress common.Address - DestinationAddress common.Address - Amount *big.Int - // From call data - ProofLocalExitRoot [tree.DefaultHeight]common.Hash - ProofRollupExitRoot [tree.DefaultHeight]common.Hash - MainnetExitRoot common.Hash - RollupExitRoot common.Hash - DestinationNetwork uint32 - Metadata []byte - // Meta - IsMessage bool + BlockNum uint64 `meddler:"block_num"` + BlockPos uint64 `meddler:"block_pos"` + GlobalIndex *big.Int `meddler:"global_index,bigint"` + OriginNetwork uint32 `meddler:"origin_network"` + OriginAddress common.Address `meddler:"origin_address"` + DestinationAddress common.Address `meddler:"destination_address"` + Amount *big.Int `meddler:"amount,bigint"` + ProofLocalExitRoot types.Proof `meddler:"proof_local_exit_root,merkleproof"` + ProofRollupExitRoot types.Proof `meddler:"proof_rollup_exit_root,merkleproof"` + MainnetExitRoot common.Hash `meddler:"mainnet_exit_root,hash"` + RollupExitRoot common.Hash `meddler:"rollup_exit_root,hash"` + GlobalExitRoot common.Hash `meddler:"global_exit_root,hash"` + DestinationNetwork uint32 `meddler:"destination_network"` + Metadata []byte `meddler:"metadata"` + IsMessage bool `meddler:"is_message"` } // Event combination of bridge and claim events type Event struct { + Pos uint64 Bridge *Bridge Claim *Claim } type processor struct { - db kv.RwDB - eventsTable string - lastBlockTable string - exitTree *tree.AppendOnlyTree - log *log.Logger + db *sql.DB + exitTree *tree.AppendOnlyTree + log *log.Logger } -func newProcessor(ctx context.Context, dbPath, dbPrefix string) (*processor, error) { - eventsTable := dbPrefix + eventsTableSufix - lastBlockTable := dbPrefix + lastBlockTableSufix - logger := log.WithFields("bridge-syncer", dbPrefix) - tableCfgFunc := func(defaultBuckets kv.TableCfg) kv.TableCfg { - cfg := kv.TableCfg{ - eventsTable: {}, - lastBlockTable: {}, - } - tree.AddTables(cfg, dbPrefix) - - return cfg - } - db, err := mdbx.NewMDBX(nil). - Path(dbPath). - WithTableCfg(tableCfgFunc). - Open() +func newProcessor(dbPath, loggerPrefix string) (*processor, error) { + err := migrations.RunMigrations(dbPath) if err != nil { return nil, err } - exitTree, err := tree.NewAppendOnlyTree(ctx, db, dbPrefix) + db, err := db.NewSQLiteDB(dbPath) if err != nil { return nil, err } - + logger := log.WithFields("bridge-syncer", loggerPrefix) + exitTree := tree.NewAppendOnlyTree(db, "") return &processor{ - db: db, - eventsTable: eventsTable, - lastBlockTable: lastBlockTable, - exitTree: exitTree, - log: logger, + db: db, + exitTree: exitTree, + log: logger, }, nil } -// GetClaimsAndBridges returns the claims and bridges occurred between fromBlock, toBlock both included. -// If toBlock has not been porcessed yet, ErrBlockNotProcessed will be returned -func (p *processor) GetClaimsAndBridges( +func (p *processor) GetBridges( ctx context.Context, fromBlock, toBlock uint64, -) ([]Event, error) { - events := []Event{} - - tx, err := p.db.BeginRo(ctx) +) ([]Bridge, error) { + tx, err := p.db.BeginTx(ctx, &sql.TxOptions{ReadOnly: true}) if err != nil { return nil, err } - - defer tx.Rollback() - lpb, err := p.getLastProcessedBlockWithTx(tx) + defer func() { + if err := tx.Rollback(); err != nil { + log.Warnf("error rolling back tx: %v", err) + } + }() + rows, err := p.queryBlockRange(tx, fromBlock, toBlock, "bridge") if err != nil { return nil, err } - - if lpb < toBlock { - return nil, ErrBlockNotProcessed + bridgePtrs := []*Bridge{} + if err = meddler.ScanAll(rows, &bridgePtrs); err != nil { + return nil, err + } + bridgesIface := db.SlicePtrsToSlice(bridgePtrs) + bridges, ok := bridgesIface.([]Bridge) + if !ok { + return nil, errors.New("failed to convert from []*Bridge to []Bridge") } - c, err := tx.Cursor(p.eventsTable) + return bridges, nil +} + +func (p *processor) GetClaims( + ctx context.Context, fromBlock, toBlock uint64, +) ([]Claim, error) { + tx, err := p.db.BeginTx(ctx, &sql.TxOptions{ReadOnly: true}) if err != nil { return nil, err } - defer c.Close() - - for k, v, err := c.Seek(dbCommon.Uint64ToBytes(fromBlock)); k != nil; k, v, err = c.Next() { - if err != nil { - return nil, err + defer func() { + if err := tx.Rollback(); err != nil { + log.Warnf("error rolling back tx: %v", err) } + }() + rows, err := p.queryBlockRange(tx, fromBlock, toBlock, "claim") + if err != nil { + return nil, err + } + claimPtrs := []*Claim{} + if err = meddler.ScanAll(rows, &claimPtrs); err != nil { + return nil, err + } + claimsIface := db.SlicePtrsToSlice(claimPtrs) + claims, ok := claimsIface.([]Claim) + if !ok { + return nil, errors.New("failed to convert from []*Claim to []Claim") + } + return claims, nil +} - if dbCommon.BytesToUint64(k) > toBlock { - break - } - blockEvents := []Event{} - err := json.Unmarshal(v, &blockEvents) - if err != nil { - return nil, err +func (p *processor) queryBlockRange(tx db.Querier, fromBlock, toBlock uint64, table string) (*sql.Rows, error) { + if err := p.isBlockProcessed(tx, toBlock); err != nil { + return nil, err + } + rows, err := tx.Query(fmt.Sprintf(` + SELECT * FROM %s + WHERE block_num >= $1 AND block_num <= $2; + `, table), fromBlock, toBlock) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + return nil, ErrNotFound } - events = append(events, blockEvents...) + return nil, err } + return rows, nil +} - return events, nil +func (p *processor) isBlockProcessed(tx db.Querier, blockNum uint64) error { + lpb, err := p.getLastProcessedBlockWithTx(tx) + if err != nil { + return err + } + if lpb < blockNum { + return ErrBlockNotProcessed + } + return nil } // GetLastProcessedBlock returns the last processed block by the processor, including blocks // that don't have events func (p *processor) GetLastProcessedBlock(ctx context.Context) (uint64, error) { - tx, err := p.db.BeginRo(ctx) - if err != nil { - return 0, err - } - - defer tx.Rollback() - - return p.getLastProcessedBlockWithTx(tx) + return p.getLastProcessedBlockWithTx(p.db) } -func (p *processor) getLastProcessedBlockWithTx(tx kv.Tx) (uint64, error) { - if blockNumBytes, err := tx.GetOne(p.lastBlockTable, lastBlockKey); err != nil { - return 0, err - } else if blockNumBytes == nil { +func (p *processor) getLastProcessedBlockWithTx(tx db.Querier) (uint64, error) { + var lastProcessedBlock uint64 + row := tx.QueryRow("SELECT num FROM BLOCK ORDER BY num DESC LIMIT 1;") + err := row.Scan(&lastProcessedBlock) + if errors.Is(err, sql.ErrNoRows) { return 0, nil - } else { - return dbCommon.BytesToUint64(blockNumBytes), nil } + return lastProcessedBlock, err } // Reorg triggers a purge and reset process on the processor to leaf it on a state // as if the last block processed was firstReorgedBlock-1 func (p *processor) Reorg(ctx context.Context, firstReorgedBlock uint64) error { - tx, err := p.db.BeginRw(ctx) + tx, err := db.NewTx(ctx, p.db) if err != nil { return err } - defer tx.Rollback() - c, err := tx.Cursor(p.eventsTable) - if err != nil { - return err - } - defer c.Close() - firstKey := dbCommon.Uint64ToBytes(firstReorgedBlock) - firstDepositCountReorged := int64(-1) - for k, v, err := c.Seek(firstKey); k != nil; k, _, err = c.Next() { + defer func() { if err != nil { - tx.Rollback() - - return err - } - if err := tx.Delete(p.eventsTable, k); err != nil { - tx.Rollback() - - return err - } - if firstDepositCountReorged == -1 { - events := []Event{} - if err := json.Unmarshal(v, &events); err != nil { - tx.Rollback() - - return err - } - - for _, event := range events { - if event.Bridge != nil { - firstDepositCountReorged = int64(event.Bridge.DepositCount) - - break - } + if errRllbck := tx.Rollback(); errRllbck != nil { + log.Errorf("error while rolling back tx %v", errRllbck) } } - } - if err := p.updateLastProcessedBlock(tx, firstReorgedBlock-1); err != nil { - tx.Rollback() + }() + _, err = tx.Exec(`DELETE FROM block WHERE num >= $1;`, firstReorgedBlock) + if err != nil { return err } - exitTreeRollback := func() {} - if firstDepositCountReorged != -1 { - if exitTreeRollback, err = p.exitTree.Reorg(tx, uint32(firstDepositCountReorged)); err != nil { - tx.Rollback() - exitTreeRollback() - return err - } + if err = p.exitTree.Reorg(tx, firstReorgedBlock); err != nil { + return err } if err := tx.Commit(); err != nil { - exitTreeRollback() - return err } @@ -274,56 +251,45 @@ func (p *processor) Reorg(ctx context.Context, firstReorgedBlock uint64) error { // ProcessBlock process the events of the block to build the exit tree // and updates the last processed block (can be called without events for that purpose) func (p *processor) ProcessBlock(ctx context.Context, block sync.Block) error { - tx, err := p.db.BeginRw(ctx) + tx, err := db.NewTx(ctx, p.db) if err != nil { return err } - leaves := []tree.Leaf{} - if len(block.Events) > 0 { - events := []Event{} - - for _, e := range block.Events { - if event, ok := e.(Event); ok { - events = append(events, event) - if event.Bridge != nil { - leaves = append(leaves, tree.Leaf{ - Index: event.Bridge.DepositCount, - Hash: event.Bridge.Hash(), - }) - } - } else { - p.log.Errorf("unexpected type %T; expected Event", e) + defer func() { + if err != nil { + if errRllbck := tx.Rollback(); errRllbck != nil { + log.Errorf("error while rolling back tx %v", errRllbck) } } - value, err := json.Marshal(events) - if err != nil { - tx.Rollback() + }() - return err + if _, err := tx.Exec(`INSERT INTO block (num) VALUES ($1)`, block.Num); err != nil { + return err + } + for _, e := range block.Events { + event, ok := e.(Event) + if !ok { + return errors.New("failed to convert sync.Block.Event to Event") } - if err := tx.Put(p.eventsTable, dbCommon.Uint64ToBytes(block.Num), value); err != nil { - tx.Rollback() - - return err + if event.Bridge != nil { + if err = p.exitTree.AddLeaf(tx, block.Num, event.Pos, types.Leaf{ + Index: event.Bridge.DepositCount, + Hash: event.Bridge.Hash(), + }); err != nil { + return err + } + if err = meddler.Insert(tx, "bridge", event.Bridge); err != nil { + return err + } + } + if event.Claim != nil { + if err = meddler.Insert(tx, "claim", event.Claim); err != nil { + return err + } } } - if err := p.updateLastProcessedBlock(tx, block.Num); err != nil { - tx.Rollback() - - return err - } - - exitTreeRollback, err := p.exitTree.AddLeaves(tx, leaves) - if err != nil { - tx.Rollback() - exitTreeRollback() - - return err - } if err := tx.Commit(); err != nil { - exitTreeRollback() - return err } @@ -332,13 +298,6 @@ func (p *processor) ProcessBlock(ctx context.Context, block sync.Block) error { return nil } -func (p *processor) updateLastProcessedBlock(tx kv.RwTx, blockNum uint64) error { - blockNumBytes := dbCommon.Uint64ToBytes(blockNum) - - return tx.Put(p.lastBlockTable, lastBlockKey, blockNumBytes) -} - -// GenerateGlobalIndex creates a global index based on network type, rollup index, and local exit root index. func GenerateGlobalIndex(mainnetFlag bool, rollupIndex uint32, localExitRootIndex uint32) *big.Int { var ( globalIndexBytes []byte diff --git a/bridgesync/processor_test.go b/bridgesync/processor_test.go index 790955bf..2ff03c76 100644 --- a/bridgesync/processor_test.go +++ b/bridgesync/processor_test.go @@ -6,9 +6,11 @@ import ( "fmt" "math/big" "os" + "path" "slices" "testing" + migrationsBridge "github.com/0xPolygon/cdk/bridgesync/migrations" "github.com/0xPolygon/cdk/log" "github.com/0xPolygon/cdk/sync" "github.com/0xPolygon/cdk/tree/testvectors" @@ -17,8 +19,11 @@ import ( ) func TestProceessor(t *testing.T) { - path := t.TempDir() - p, err := newProcessor(context.Background(), path, "foo") + path := path.Join(t.TempDir(), "file::memory:?cache=shared") + log.Debugf("sqlite path: %s", path) + err := migrationsBridge.RunMigrations(path) + require.NoError(t, err) + p, err := newProcessor(path, "foo") require.NoError(t, err) actions := []processAction{ // processed: ~ @@ -41,15 +46,24 @@ func TestProceessor(t *testing.T) { firstReorgedBlock: 1, expectedErr: nil, }, - &getClaimsAndBridgesAction{ + &getClaims{ p: p, description: "on an empty processor", ctx: context.Background(), fromBlock: 0, toBlock: 2, - expectedEvents: nil, + expectedClaims: nil, expectedErr: ErrBlockNotProcessed, }, + &getBridges{ + p: p, + description: "on an empty processor", + ctx: context.Background(), + fromBlock: 0, + toBlock: 2, + expectedBridges: nil, + expectedErr: ErrBlockNotProcessed, + }, &processBlockAction{ p: p, description: "block1", @@ -64,24 +78,42 @@ func TestProceessor(t *testing.T) { expectedLastProcessedBlock: 1, expectedErr: nil, }, - &getClaimsAndBridgesAction{ + &getClaims{ p: p, description: "after block1: range 0, 2", ctx: context.Background(), fromBlock: 0, toBlock: 2, - expectedEvents: nil, + expectedClaims: nil, expectedErr: ErrBlockNotProcessed, }, - &getClaimsAndBridgesAction{ + &getBridges{ + p: p, + description: "after block1: range 0, 2", + ctx: context.Background(), + fromBlock: 0, + toBlock: 2, + expectedBridges: nil, + expectedErr: ErrBlockNotProcessed, + }, + &getClaims{ p: p, description: "after block1: range 1, 1", ctx: context.Background(), fromBlock: 1, toBlock: 1, - expectedEvents: eventsToBridgeEvents(block1.Events), + expectedClaims: eventsToClaims(block1.Events), expectedErr: nil, }, + &getBridges{ + p: p, + description: "after block1: range 1, 1", + ctx: context.Background(), + fromBlock: 1, + toBlock: 1, + expectedBridges: eventsToBridges(block1.Events), + expectedErr: nil, + }, &reorgAction{ p: p, description: "after block1", @@ -89,15 +121,24 @@ func TestProceessor(t *testing.T) { expectedErr: nil, }, // processed: ~ - &getClaimsAndBridgesAction{ + &getClaims{ p: p, description: "after block1 reorged", ctx: context.Background(), fromBlock: 0, toBlock: 2, - expectedEvents: nil, + expectedClaims: nil, expectedErr: ErrBlockNotProcessed, }, + &getBridges{ + p: p, + description: "after block1 reorged", + ctx: context.Background(), + fromBlock: 0, + toBlock: 2, + expectedBridges: nil, + expectedErr: ErrBlockNotProcessed, + }, &processBlockAction{ p: p, description: "block1 (after it's reorged)", @@ -119,24 +160,45 @@ func TestProceessor(t *testing.T) { expectedLastProcessedBlock: 3, expectedErr: nil, }, - &getClaimsAndBridgesAction{ + &getClaims{ p: p, description: "after block3: range 2, 2", ctx: context.Background(), fromBlock: 2, toBlock: 2, - expectedEvents: []Event{}, + expectedClaims: []Claim{}, expectedErr: nil, }, - &getClaimsAndBridgesAction{ + &getClaims{ p: p, description: "after block3: range 1, 3", ctx: context.Background(), fromBlock: 1, toBlock: 3, - expectedEvents: append( - eventsToBridgeEvents(block1.Events), - eventsToBridgeEvents(block3.Events)..., + expectedClaims: append( + eventsToClaims(block1.Events), + eventsToClaims(block3.Events)..., + ), + expectedErr: nil, + }, + &getBridges{ + p: p, + description: "after block3: range 2, 2", + ctx: context.Background(), + fromBlock: 2, + toBlock: 2, + expectedBridges: []Bridge{}, + expectedErr: nil, + }, + &getBridges{ + p: p, + description: "after block3: range 1, 3", + ctx: context.Background(), + fromBlock: 1, + toBlock: 3, + expectedBridges: append( + eventsToBridges(block1.Events), + eventsToBridges(block3.Events)..., ), expectedErr: nil, }, @@ -151,7 +213,7 @@ func TestProceessor(t *testing.T) { p: p, description: "after block3 reorged", ctx: context.Background(), - expectedLastProcessedBlock: 2, + expectedLastProcessedBlock: 1, expectedErr: nil, }, &reorgAction{ @@ -195,48 +257,49 @@ func TestProceessor(t *testing.T) { expectedLastProcessedBlock: 5, expectedErr: nil, }, - &getClaimsAndBridgesAction{ + &getClaims{ p: p, description: "after block5: range 1, 3", ctx: context.Background(), fromBlock: 1, toBlock: 3, - expectedEvents: append( - eventsToBridgeEvents(block1.Events), - eventsToBridgeEvents(block3.Events)..., + expectedClaims: append( + eventsToClaims(block1.Events), + eventsToClaims(block3.Events)..., ), expectedErr: nil, }, - &getClaimsAndBridgesAction{ + &getClaims{ p: p, description: "after block5: range 4, 5", ctx: context.Background(), fromBlock: 4, toBlock: 5, - expectedEvents: append( - eventsToBridgeEvents(block4.Events), - eventsToBridgeEvents(block5.Events)..., + expectedClaims: append( + eventsToClaims(block4.Events), + eventsToClaims(block5.Events)..., ), expectedErr: nil, }, - &getClaimsAndBridgesAction{ + &getClaims{ p: p, description: "after block5: range 0, 5", ctx: context.Background(), fromBlock: 0, toBlock: 5, - expectedEvents: slices.Concat( - eventsToBridgeEvents(block1.Events), - eventsToBridgeEvents(block3.Events), - eventsToBridgeEvents(block4.Events), - eventsToBridgeEvents(block5.Events), + expectedClaims: slices.Concat( + eventsToClaims(block1.Events), + eventsToClaims(block3.Events), + eventsToClaims(block4.Events), + eventsToClaims(block5.Events), ), expectedErr: nil, }, } for _, a := range actions { - t.Run(fmt.Sprintf("%s: %s", a.method(), a.desc()), a.execute) + log.Debugf("%s: %s", a.method(), a.desc()) + a.execute(t) } } @@ -249,6 +312,8 @@ var ( Num: 1, Events: []interface{}{ Event{Bridge: &Bridge{ + BlockNum: 1, + BlockPos: 0, LeafType: 1, OriginNetwork: 1, OriginAddress: common.HexToAddress("01"), @@ -259,6 +324,8 @@ var ( DepositCount: 0, }}, Event{Claim: &Claim{ + BlockNum: 1, + BlockPos: 1, GlobalIndex: big.NewInt(1), OriginNetwork: 1, OriginAddress: common.HexToAddress("01"), @@ -271,6 +338,8 @@ var ( Num: 3, Events: []interface{}{ Event{Bridge: &Bridge{ + BlockNum: 3, + BlockPos: 0, LeafType: 2, OriginNetwork: 2, OriginAddress: common.HexToAddress("02"), @@ -281,12 +350,14 @@ var ( DepositCount: 1, }}, Event{Bridge: &Bridge{ + BlockNum: 3, + BlockPos: 1, LeafType: 3, OriginNetwork: 3, OriginAddress: common.HexToAddress("03"), DestinationNetwork: 3, DestinationAddress: common.HexToAddress("03"), - Amount: nil, + Amount: big.NewInt(0), Metadata: common.Hex2Bytes("03"), DepositCount: 2, }}, @@ -300,6 +371,8 @@ var ( Num: 5, Events: []interface{}{ Event{Claim: &Claim{ + BlockNum: 4, + BlockPos: 0, GlobalIndex: big.NewInt(4), OriginNetwork: 4, OriginAddress: common.HexToAddress("04"), @@ -307,6 +380,8 @@ var ( Amount: big.NewInt(4), }}, Event{Claim: &Claim{ + BlockNum: 4, + BlockPos: 1, GlobalIndex: big.NewInt(5), OriginNetwork: 5, OriginAddress: common.HexToAddress("05"), @@ -325,31 +400,57 @@ type processAction interface { execute(t *testing.T) } -// GetClaimsAndBridges +// GetClaims -type getClaimsAndBridgesAction struct { +type getClaims struct { p *processor description string ctx context.Context fromBlock uint64 toBlock uint64 - expectedEvents []Event + expectedClaims []Claim expectedErr error } -func (a *getClaimsAndBridgesAction) method() string { - return "GetClaimsAndBridges" +func (a *getClaims) method() string { + return "GetClaims" } -func (a *getClaimsAndBridgesAction) desc() string { +func (a *getClaims) desc() string { return a.description } -func (a *getClaimsAndBridgesAction) execute(t *testing.T) { +func (a *getClaims) execute(t *testing.T) { t.Helper() + actualEvents, actualErr := a.p.GetClaims(a.ctx, a.fromBlock, a.toBlock) + require.Equal(t, a.expectedErr, actualErr) + require.Equal(t, a.expectedClaims, actualEvents) +} - actualEvents, actualErr := a.p.GetClaimsAndBridges(a.ctx, a.fromBlock, a.toBlock) - require.Equal(t, a.expectedEvents, actualEvents) +// GetBridges + +type getBridges struct { + p *processor + description string + ctx context.Context + fromBlock uint64 + toBlock uint64 + expectedBridges []Bridge + expectedErr error +} + +func (a *getBridges) method() string { + return "GetBridges" +} + +func (a *getBridges) desc() string { + return a.description +} + +func (a *getBridges) execute(t *testing.T) { + t.Helper() + actualEvents, actualErr := a.p.GetBridges(a.ctx, a.fromBlock, a.toBlock) + require.Equal(t, a.expectedBridges, actualEvents) require.Equal(t, a.expectedErr, actualErr) } @@ -427,18 +528,32 @@ func (a *processBlockAction) execute(t *testing.T) { require.Equal(t, a.expectedErr, actualErr) } -func eventsToBridgeEvents(events []interface{}) []Event { - bridgeEvents := []Event{} - +func eventsToBridges(events []interface{}) []Bridge { + bridges := []Bridge{} for _, event := range events { - if evt, ok := event.(Event); ok { - bridgeEvents = append(bridgeEvents, evt) - } else { - log.Errorf("unexpected type %T; expected Event", event) + e, ok := event.(Event) + if !ok { + panic("should be ok") + } + if e.Bridge != nil { + bridges = append(bridges, *e.Bridge) } } + return bridges +} - return bridgeEvents +func eventsToClaims(events []interface{}) []Claim { + claims := []Claim{} + for _, event := range events { + e, ok := event.(Event) + if !ok { + panic("should be ok") + } + if e.Claim != nil { + claims = append(claims, *e.Claim) + } + } + return claims } func TestHashBridge(t *testing.T) { diff --git a/claimsponsor/e2e_test.go b/claimsponsor/e2e_test.go index 29a939d7..904df8a3 100644 --- a/claimsponsor/e2e_test.go +++ b/claimsponsor/e2e_test.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "math/big" + "path" "testing" "time" @@ -21,7 +22,7 @@ func TestE2EL1toEVML2(t *testing.T) { // start other needed components ctx := context.Background() env := helpers.SetupAggoracleWithEVMChain(t) - dbPathBridgeSyncL1 := t.TempDir() + dbPathBridgeSyncL1 := path.Join(t.TempDir(), "file::memory:?cache=shared") testClient := helpers.TestClient{ClientRenamed: env.L1Client.Client()} bridgeSyncL1, err := bridgesync.NewL1(ctx, dbPathBridgeSyncL1, env.BridgeL1Addr, 10, etherman.LatestBlock, env.ReorgDetector, testClient, 0, time.Millisecond*10, 0, 0) require.NoError(t, err) diff --git a/dataavailability/datacommittee/datacommittee_test.go b/dataavailability/datacommittee/datacommittee_test.go index ad128324..fcacef3c 100644 --- a/dataavailability/datacommittee/datacommittee_test.go +++ b/dataavailability/datacommittee/datacommittee_test.go @@ -2,7 +2,6 @@ package datacommittee import ( "errors" - "fmt" "math/big" "testing" @@ -164,7 +163,7 @@ func deployDACProxy(auth *bind.TransactOpts, client bind.ContractBackend, dacImp if err != nil { return common.Address{}, err } - fmt.Println("DAC proxy deployed at", proxyAddr) + log.Debugf("DAC proxy deployed at", proxyAddr) return proxyAddr, nil } diff --git a/db/interface.go b/db/interface.go new file mode 100644 index 00000000..03f81aba --- /dev/null +++ b/db/interface.go @@ -0,0 +1,17 @@ +package db + +import ( + "context" + "database/sql" +) + +type Querier interface { + Exec(query string, args ...interface{}) (sql.Result, error) + Query(query string, args ...interface{}) (*sql.Rows, error) + QueryRow(query string, args ...interface{}) *sql.Row +} + +type DBer interface { + Querier + BeginTx(ctx context.Context, opts *sql.TxOptions) (*sql.Tx, error) +} diff --git a/db/meddler.go b/db/meddler.go new file mode 100644 index 00000000..90071916 --- /dev/null +++ b/db/meddler.go @@ -0,0 +1,178 @@ +package db + +import ( + "errors" + "fmt" + "math/big" + "reflect" + "strings" + + tree "github.com/0xPolygon/cdk/tree/types" + "github.com/ethereum/go-ethereum/common" + sqlite "github.com/mattn/go-sqlite3" + "github.com/russross/meddler" +) + +// initMeddler registers tags to be used to read/write from SQL DBs using meddler +func initMeddler() { + meddler.Default = meddler.SQLite + meddler.Register("bigint", BigIntMeddler{}) + meddler.Register("merkleproof", MerkleProofMeddler{}) + meddler.Register("hash", HashMeddler{}) +} + +func SQLiteErr(err error) (*sqlite.Error, bool) { + sqliteErr := &sqlite.Error{} + if ok := errors.As(err, sqliteErr); ok { + return sqliteErr, true + } + if driverErr, ok := meddler.DriverErr(err); ok { + return sqliteErr, errors.As(driverErr, sqliteErr) + } + return sqliteErr, false +} + +// SliceToSlicePtrs converts any []Foo to []*Foo +func SliceToSlicePtrs(slice interface{}) interface{} { + v := reflect.ValueOf(slice) + vLen := v.Len() + typ := v.Type().Elem() + res := reflect.MakeSlice(reflect.SliceOf(reflect.PtrTo(typ)), vLen, vLen) + for i := 0; i < vLen; i++ { + res.Index(i).Set(v.Index(i).Addr()) + } + return res.Interface() +} + +// SlicePtrsToSlice converts any []*Foo to []Foo +func SlicePtrsToSlice(slice interface{}) interface{} { + v := reflect.ValueOf(slice) + vLen := v.Len() + typ := v.Type().Elem().Elem() + res := reflect.MakeSlice(reflect.SliceOf(typ), vLen, vLen) + for i := 0; i < vLen; i++ { + res.Index(i).Set(v.Index(i).Elem()) + } + return res.Interface() +} + +// BigIntMeddler encodes or decodes the field value to or from JSON +type BigIntMeddler struct{} + +// PreRead is called before a Scan operation for fields that have the BigIntMeddler +func (b BigIntMeddler) PreRead(fieldAddr interface{}) (scanTarget interface{}, err error) { + // give a pointer to a byte buffer to grab the raw data + return new(string), nil +} + +// PostRead is called after a Scan operation for fields that have the BigIntMeddler +func (b BigIntMeddler) PostRead(fieldPtr, scanTarget interface{}) error { + ptr, ok := scanTarget.(*string) + if !ok { + return errors.New("scanTarget is not *string") + } + if ptr == nil { + return fmt.Errorf("BigIntMeddler.PostRead: nil pointer") + } + field, ok := fieldPtr.(**big.Int) + if !ok { + return errors.New("fieldPtr is not *big.Int") + } + decimal := 10 + *field, ok = new(big.Int).SetString(*ptr, decimal) + if !ok { + return fmt.Errorf("big.Int.SetString failed on \"%v\"", *ptr) + } + return nil +} + +// PreWrite is called before an Insert or Update operation for fields that have the BigIntMeddler +func (b BigIntMeddler) PreWrite(fieldPtr interface{}) (saveValue interface{}, err error) { + field, ok := fieldPtr.(*big.Int) + if !ok { + return nil, errors.New("fieldPtr is not *big.Int") + } + + return field.String(), nil +} + +// MerkleProofMeddler encodes or decodes the field value to or from JSON +type MerkleProofMeddler struct{} + +// PreRead is called before a Scan operation for fields that have the ProofMeddler +func (b MerkleProofMeddler) PreRead(fieldAddr interface{}) (scanTarget interface{}, err error) { + // give a pointer to a byte buffer to grab the raw data + return new(string), nil +} + +// PostRead is called after a Scan operation for fields that have the ProofMeddler +func (b MerkleProofMeddler) PostRead(fieldPtr, scanTarget interface{}) error { + ptr, ok := scanTarget.(*string) + if !ok { + return errors.New("scanTarget is not *string") + } + if ptr == nil { + return errors.New("ProofMeddler.PostRead: nil pointer") + } + field, ok := fieldPtr.(*tree.Proof) + if !ok { + return errors.New("fieldPtr is not tree.Proof") + } + strHashes := strings.Split(*ptr, ",") + if len(strHashes) != int(tree.DefaultHeight) { + return fmt.Errorf("unexpected len of hashes: expected %d actual %d", tree.DefaultHeight, len(strHashes)) + } + for i, strHash := range strHashes { + field[i] = common.HexToHash(strHash) + } + return nil +} + +// PreWrite is called before an Insert or Update operation for fields that have the ProofMeddler +func (b MerkleProofMeddler) PreWrite(fieldPtr interface{}) (saveValue interface{}, err error) { + field, ok := fieldPtr.(tree.Proof) + if !ok { + return nil, errors.New("fieldPtr is not tree.Proof") + } + var s string + for _, f := range field { + s += f.Hex() + "," + } + s = strings.TrimSuffix(s, ",") + return s, nil +} + +// HashMeddler encodes or decodes the field value to or from JSON +type HashMeddler struct{} + +// PreRead is called before a Scan operation for fields that have the ProofMeddler +func (b HashMeddler) PreRead(fieldAddr interface{}) (scanTarget interface{}, err error) { + // give a pointer to a byte buffer to grab the raw data + return new(string), nil +} + +// PostRead is called after a Scan operation for fields that have the ProofMeddler +func (b HashMeddler) PostRead(fieldPtr, scanTarget interface{}) error { + ptr, ok := scanTarget.(*string) + if !ok { + return errors.New("scanTarget is not *string") + } + if ptr == nil { + return fmt.Errorf("HashMeddler.PostRead: nil pointer") + } + field, ok := fieldPtr.(*common.Hash) + if !ok { + return errors.New("fieldPtr is not common.Hash") + } + *field = common.HexToHash(*ptr) + return nil +} + +// PreWrite is called before an Insert or Update operation for fields that have the ProofMeddler +func (b HashMeddler) PreWrite(fieldPtr interface{}) (saveValue interface{}, err error) { + field, ok := fieldPtr.(common.Hash) + if !ok { + return nil, errors.New("fieldPtr is not common.Hash") + } + return field.Hex(), nil +} diff --git a/db/migrations.go b/db/migrations.go new file mode 100644 index 00000000..1a56874e --- /dev/null +++ b/db/migrations.go @@ -0,0 +1,48 @@ +package db + +import ( + "fmt" + "strings" + + "github.com/0xPolygon/cdk/db/types" + "github.com/0xPolygon/cdk/log" + _ "github.com/mattn/go-sqlite3" + migrate "github.com/rubenv/sql-migrate" +) + +const ( + upDownSeparator = "-- +migrate Up" + dbPrefixReplacer = "/*dbprefix*/" +) + +// RunMigrations will execute pending migrations if needed to keep +// the database updated with the latest changes in either direction, +// up or down. +func RunMigrations(dbPath string, migrations []types.Migration) error { + db, err := NewSQLiteDB(dbPath) + if err != nil { + return fmt.Errorf("error creating DB %w", err) + } + migs := &migrate.MemoryMigrationSource{Migrations: []*migrate.Migration{}} + for _, m := range migrations { + prefixed := strings.ReplaceAll(m.SQL, dbPrefixReplacer, m.Prefix) + splitted := strings.Split(prefixed, upDownSeparator) + migs.Migrations = append(migs.Migrations, &migrate.Migration{ + Id: m.Prefix + m.ID, + Up: []string{splitted[1]}, + Down: []string{splitted[0]}, + }) + } + + log.Debugf("running migrations:") + for _, m := range migs.Migrations { + log.Debugf("%+v", m.Id) + } + nMigrations, err := migrate.Exec(db, "sqlite3", migs, migrate.Up) + if err != nil { + return fmt.Errorf("error executing migration %w", err) + } + + log.Infof("successfully ran %d migrations", nMigrations) + return nil +} diff --git a/db/sqlite.go b/db/sqlite.go new file mode 100644 index 00000000..e30e9e26 --- /dev/null +++ b/db/sqlite.go @@ -0,0 +1,27 @@ +package db + +import ( + "database/sql" + + _ "github.com/mattn/go-sqlite3" +) + +const ( + UniqueConstrain = 1555 +) + +// NewSQLiteDB creates a new SQLite DB +func NewSQLiteDB(dbPath string) (*sql.DB, error) { + initMeddler() + db, err := sql.Open("sqlite3", dbPath) + if err != nil { + return nil, err + } + _, err = db.Exec(` + PRAGMA foreign_keys = ON; + pragma journal_mode = WAL; + pragma synchronous = normal; + pragma journal_size_limit = 6144000; + `) + return db, err +} diff --git a/db/tx.go b/db/tx.go new file mode 100644 index 00000000..926da07c --- /dev/null +++ b/db/tx.go @@ -0,0 +1,60 @@ +package db + +import ( + "context" +) + +type SQLTxer interface { + Querier + Commit() error + Rollback() error +} + +type Txer interface { + SQLTxer + AddRollbackCallback(cb func()) + AddCommitCallback(cb func()) +} + +type Tx struct { + SQLTxer + rollbackCallbacks []func() + commitCallbacks []func() +} + +func NewTx(ctx context.Context, db DBer) (Txer, error) { + tx, err := db.BeginTx(ctx, nil) + if err != nil { + return nil, err + } + return &Tx{ + SQLTxer: tx, + }, nil +} + +func (s *Tx) AddRollbackCallback(cb func()) { + s.rollbackCallbacks = append(s.rollbackCallbacks, cb) +} +func (s *Tx) AddCommitCallback(cb func()) { + s.commitCallbacks = append(s.commitCallbacks, cb) +} + +func (s *Tx) Commit() error { + if err := s.SQLTxer.Commit(); err != nil { + return err + } + for _, cb := range s.commitCallbacks { + cb() + } + return nil +} + +func (s *Tx) Rollback() error { + if err := s.SQLTxer.Rollback(); err != nil { + return err + } + for _, cb := range s.rollbackCallbacks { + cb() + } + return nil +} diff --git a/db/types/types.go b/db/types/types.go new file mode 100644 index 00000000..ade19092 --- /dev/null +++ b/db/types/types.go @@ -0,0 +1,7 @@ +package types + +type Migration struct { + ID string + SQL string + Prefix string +} diff --git a/go.mod b/go.mod index 0f229b12..f64ec3bb 100644 --- a/go.mod +++ b/go.mod @@ -17,8 +17,10 @@ require ( github.com/jackc/pgconn v1.14.3 github.com/jackc/pgx/v4 v4.18.3 github.com/ledgerwatch/erigon-lib v1.0.0 + github.com/mattn/go-sqlite3 v1.14.23 github.com/mitchellh/mapstructure v1.5.0 github.com/rubenv/sql-migrate v1.6.1 + github.com/russross/meddler v1.0.1 github.com/spf13/viper v1.19.0 github.com/stretchr/testify v1.9.0 github.com/urfave/cli/v2 v2.27.2 @@ -30,6 +32,7 @@ require ( golang.org/x/sync v0.7.0 google.golang.org/grpc v1.64.0 google.golang.org/protobuf v1.34.2 + modernc.org/sqlite v1.32.0 ) require ( @@ -59,6 +62,7 @@ require ( github.com/deckarep/golang-set/v2 v2.6.0 // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 // indirect github.com/didip/tollbooth/v6 v6.1.2 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect github.com/erigontech/mdbx-go v0.27.14 // indirect github.com/ethereum/c-kzg-4844 v1.0.0 // indirect github.com/ethereum/go-verkle v0.1.1-0.20240306133620-7d920df305f0 // indirect @@ -79,6 +83,7 @@ require ( github.com/google/uuid v1.6.0 // indirect github.com/gorilla/websocket v1.5.1 // indirect github.com/hashicorp/go-bexpr v0.1.10 // indirect + github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect github.com/hashicorp/hcl v1.0.1-0.20180906183839-65a6292f0157 // indirect github.com/holiman/billy v0.0.0-20240216141850-2abb0c79d3c4 // indirect github.com/holiman/bloomfilter/v2 v2.0.3 // indirect @@ -106,6 +111,7 @@ require ( github.com/miguelmota/go-solidity-sha3 v0.1.1 // indirect github.com/mitchellh/pointerstructure v1.2.0 // indirect github.com/mmcloughlin/addchain v0.4.0 // indirect + github.com/ncruces/go-strftime v0.1.9 // indirect github.com/olekukonko/tablewriter v0.0.5 // indirect github.com/onsi/gomega v1.27.10 // indirect github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect @@ -116,6 +122,7 @@ require ( github.com/prometheus/client_model v0.6.1 // indirect github.com/prometheus/common v0.48.0 // indirect github.com/prometheus/procfs v0.12.0 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/rivo/uniseg v0.2.0 // indirect github.com/rogpeppe/go-internal v1.11.0 // indirect github.com/rs/cors v1.7.0 // indirect @@ -142,12 +149,18 @@ require ( go.opentelemetry.io/otel/trace v1.24.0 // indirect go.uber.org/multierr v1.10.0 // indirect golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa // indirect - golang.org/x/sys v0.21.0 // indirect + golang.org/x/sys v0.22.0 // indirect golang.org/x/text v0.16.0 // indirect golang.org/x/time v0.5.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect + modernc.org/gc/v3 v3.0.0-20240801135723-a856999a2e4a // indirect + modernc.org/libc v1.60.0 // indirect + modernc.org/mathutil v1.6.0 // indirect + modernc.org/memory v1.8.0 // indirect + modernc.org/strutil v1.2.0 // indirect + modernc.org/token v1.1.0 // indirect rsc.io/tmplfunc v0.0.3 // indirect ) diff --git a/go.sum b/go.sum index 92207638..bc9eb188 100644 --- a/go.sum +++ b/go.sum @@ -89,6 +89,8 @@ github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 h1:HbphB4TFFXpv7MNrT52FGrrgVXF1 github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0/go.mod h1:DZGJHZMqrU4JJqFAWUS2UO1+lbSKsdiOoYi9Zzey7Fc= github.com/didip/tollbooth/v6 v6.1.2 h1:Kdqxmqw9YTv0uKajBUiWQg+GURL/k4vy9gmLCL01PjQ= github.com/didip/tollbooth/v6 v6.1.2/go.mod h1:xjcse6CTHCLuOkzsWrEgdy9WPJFv+p/x6v+MyfP+O9s= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/erigontech/mdbx-go v0.27.14 h1:IVVeQVCAjZRpAR8bThlP2ISxrOwdV35NZdGwAgotaRw= github.com/erigontech/mdbx-go v0.27.14/go.mod h1:FAMxbOgqOnRDx51j8HjuJZIgznbDwjX7LItd+/UWyA4= github.com/ethereum/c-kzg-4844 v1.0.0 h1:0X1LBXxaEtYD9xsyj9B9ctQEZIpnvVDeoBx8aHEwTNA= @@ -164,6 +166,8 @@ github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeN github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd h1:gbpYu9NMq8jhDVbvlGkMFWCjLFlqqEZjEmObmhUy6Vo= +github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/subcommands v1.2.0/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= @@ -174,6 +178,8 @@ github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 h1:UH//fgunKIs4JdUbpDl1VZCDa github.com/grpc-ecosystem/go-grpc-middleware v1.4.0/go.mod h1:g5qyo/la0ALbONm6Vbp88Yd8NsDy6rZz+RcrMPxvld8= github.com/hashicorp/go-bexpr v0.1.10 h1:9kuI5PFotCboP3dkDYFr/wi0gg0QVbSNz5oFRpxn4uE= github.com/hashicorp/go-bexpr v0.1.10/go.mod h1:oxlubA2vC/gFVfX1A6JGp7ls7uCDlfJn732ehYYg+g0= +github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= +github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hashicorp/hcl v1.0.1-0.20180906183839-65a6292f0157 h1:uyodBE3xDz0ynKs1tLBU26wOQoEkAqqiY18DbZ+FZrA= github.com/hashicorp/hcl v1.0.1-0.20180906183839-65a6292f0157/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hermeznetwork/tracerr v0.3.2 h1:QB3TlQxO/4XHyixsg+nRZPuoel/FFQlQ7oAoHDD5l1c= @@ -296,8 +302,9 @@ github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU= github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= -github.com/mattn/go-sqlite3 v1.14.19 h1:fhGleo2h1p8tVChob4I9HpmVFIAkKGpiukdrgQbWfGI= -github.com/mattn/go-sqlite3 v1.14.19/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= +github.com/mattn/go-sqlite3 v1.14.7/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= +github.com/mattn/go-sqlite3 v1.14.23 h1:gbShiuAP1W5j9UOksQ06aiiqPMxYecovVGwmTxWtuw0= +github.com/mattn/go-sqlite3 v1.14.23/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/miguelmota/go-solidity-sha3 v0.1.1 h1:3Y08sKZDtudtE5kbTBPC9RYJznoSYyWI9VD6mghU0CA= github.com/miguelmota/go-solidity-sha3 v0.1.1/go.mod h1:sax1FvQF+f71j8W1uUHMZn8NxKyl5rYLks2nqj8RFEw= github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= @@ -308,6 +315,8 @@ github.com/mitchellh/pointerstructure v1.2.0/go.mod h1:BRAsLI5zgXmw97Lf6s25bs8oh github.com/mmcloughlin/addchain v0.4.0 h1:SobOdjm2xLj1KkXN5/n0xTIWyZA2+s99UCY1iPfkHRY= github.com/mmcloughlin/addchain v0.4.0/go.mod h1:A86O+tHqZLMNO4w6ZZ4FlVQEadcoqkyU72HC5wJ4RlU= github.com/mmcloughlin/profile v0.1.1/go.mod h1:IhHD7q1ooxgwTgjxQYkACGA77oFTDdFVejUS1/tS/qU= +github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= +github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= @@ -348,6 +357,8 @@ github.com/prometheus/common v0.48.0 h1:QO8U2CdOzSn1BBsmXJXduaaW+dY/5QLjfB8svtSz github.com/prometheus/common v0.48.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5EC6ILDTlAPc= github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= @@ -363,6 +374,8 @@ github.com/rubenv/sql-migrate v1.6.1 h1:bo6/sjsan9HaXAsNxYP/jCEDUGibHp8JmOBw7NTG github.com/rubenv/sql-migrate v1.6.1/go.mod h1:tPzespupJS0jacLfhbwto/UjSX+8h2FdWB7ar+QlHa0= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/russross/meddler v1.0.1 h1:JLR7Z4M4iGm1nr7DIURBq18UW8cTrm+qArUFgOhELo8= +github.com/russross/meddler v1.0.1/go.mod h1:GzGDChbFHuzxlFwt8gnJMRRNyFSQDSudmy2kHh7GYnQ= github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ= github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4= github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= @@ -472,6 +485,8 @@ golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKG golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.19.0 h1:fEdghXQSo20giMthA7cd28ZC+jts4amQ3YMXiP5oMQ8= +golang.org/x/mod v0.19.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -524,8 +539,8 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= -golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= +golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -552,6 +567,8 @@ golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapK golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.23.0 h1:SGsXPZ+2l4JsgaCKkx+FQ9YZ5XEtA1GZYuoDjenLjvg= +golang.org/x/tools v0.23.0/go.mod h1:pnu6ufv6vQkll6szChhK3C3L/ruaIv5eBeztNG8wtsI= golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -595,5 +612,31 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +modernc.org/cc/v4 v4.21.4 h1:3Be/Rdo1fpr8GrQ7IVw9OHtplU4gWbb+wNgeoBMmGLQ= +modernc.org/cc/v4 v4.21.4/go.mod h1:HM7VJTZbUCR3rV8EYBi9wxnJ0ZBRiGE5OeGXNA0IsLQ= +modernc.org/ccgo/v4 v4.21.0 h1:kKPI3dF7RIag8YcToh5ZwDcVMIv6VGa0ED5cvh0LMW4= +modernc.org/ccgo/v4 v4.21.0/go.mod h1:h6kt6H/A2+ew/3MW/p6KEoQmrq/i3pr0J/SiwiaF/g0= +modernc.org/fileutil v1.3.0 h1:gQ5SIzK3H9kdfai/5x41oQiKValumqNTDXMvKo62HvE= +modernc.org/fileutil v1.3.0/go.mod h1:XatxS8fZi3pS8/hKG2GH/ArUogfxjpEKs3Ku3aK4JyQ= +modernc.org/gc/v2 v2.5.0 h1:bJ9ChznK1L1mUtAQtxi0wi5AtAs5jQuw4PrPHO5pb6M= +modernc.org/gc/v2 v2.5.0/go.mod h1:wzN5dK1AzVGoH6XOzc3YZ+ey/jPgYHLuVckd62P0GYU= +modernc.org/gc/v3 v3.0.0-20240801135723-a856999a2e4a h1:CfbpOLEo2IwNzJdMvE8aiRbPMxoTpgAJeyePh0SmO8M= +modernc.org/gc/v3 v3.0.0-20240801135723-a856999a2e4a/go.mod h1:Qz0X07sNOR1jWYCrJMEnbW/X55x206Q7Vt4mz6/wHp4= +modernc.org/libc v1.60.0 h1:XeRF1gXky7JE5E8IErtYAdKj+ykZPdYUsgJNQ8RFWIA= +modernc.org/libc v1.60.0/go.mod h1:xJuobKuNxKH3RUatS7GjR+suWj+5c2K7bi4m/S5arOY= +modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4= +modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo= +modernc.org/memory v1.8.0 h1:IqGTL6eFMaDZZhEWwcREgeMXYwmW83LYW8cROZYkg+E= +modernc.org/memory v1.8.0/go.mod h1:XPZ936zp5OMKGWPqbD3JShgd/ZoQ7899TUuQqxY+peU= +modernc.org/opt v0.1.3 h1:3XOZf2yznlhC+ibLltsDGzABUGVx8J6pnFMS3E4dcq4= +modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0= +modernc.org/sortutil v1.2.0 h1:jQiD3PfS2REGJNzNCMMaLSp/wdMNieTbKX920Cqdgqc= +modernc.org/sortutil v1.2.0/go.mod h1:TKU2s7kJMf1AE84OoiGppNHJwvB753OYfNl2WRb++Ss= +modernc.org/sqlite v1.32.0 h1:6BM4uGza7bWypsw4fdLRsLxut6bHe4c58VeqjRgST8s= +modernc.org/sqlite v1.32.0/go.mod h1:UqoylwmTb9F+IqXERT8bW9zzOWN8qwAIcLdzeBZs4hA= +modernc.org/strutil v1.2.0 h1:agBi9dp1I+eOnxXeiZawM8F4LawKv4NzGWSaLfyeNZA= +modernc.org/strutil v1.2.0/go.mod h1:/mdcBmfOibveCTBxUl5B5l6W+TTH1FXPLHZE6bTosX0= +modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= rsc.io/tmplfunc v0.0.3 h1:53XFQh69AfOa8Tw0Jm7t+GV7KZhOi6jzsCzTtKbMvzU= rsc.io/tmplfunc v0.0.3/go.mod h1:AG3sTPzElb1Io3Yg4voV9AGZJuleGAwaVRxL9M49PhA= diff --git a/l1bridge2infoindexsync/downloader.go b/l1bridge2infoindexsync/downloader.go index 0a609cb9..f4db8422 100644 --- a/l1bridge2infoindexsync/downloader.go +++ b/l1bridge2infoindexsync/downloader.go @@ -6,6 +6,7 @@ import ( "github.com/0xPolygon/cdk/bridgesync" "github.com/0xPolygon/cdk/l1infotreesync" + "github.com/0xPolygon/cdk/tree/types" "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/rpc" @@ -64,6 +65,6 @@ func (d *downloader) getMainnetExitRootAtL1InfoTreeIndex(ctx context.Context, in return leaf.MainnetExitRoot, nil } -func (d *downloader) getBridgeIndex(ctx context.Context, mainnetExitRoot common.Hash) (uint32, error) { - return d.l1Bridge.GetBridgeIndexByRoot(ctx, mainnetExitRoot) +func (d *downloader) getBridgeIndex(ctx context.Context, mainnetExitRoot common.Hash) (types.Root, error) { + return d.l1Bridge.GetBridgeRootByHash(ctx, mainnetExitRoot) } diff --git a/l1bridge2infoindexsync/driver.go b/l1bridge2infoindexsync/driver.go index d014e2c5..921a0c41 100644 --- a/l1bridge2infoindexsync/driver.go +++ b/l1bridge2infoindexsync/driver.go @@ -209,13 +209,13 @@ func (d *driver) getRelation(ctx context.Context, l1InfoIndex uint32) (bridge2L1 return bridge2L1InfoRelation{}, err } - bridgeIndex, err := d.downloader.getBridgeIndex(ctx, mer) + bridgeRoot, err := d.downloader.getBridgeIndex(ctx, mer) if err != nil { return bridge2L1InfoRelation{}, err } return bridge2L1InfoRelation{ - bridgeIndex: bridgeIndex, + bridgeIndex: bridgeRoot.Index, l1InfoTreeIndex: l1InfoIndex, }, nil } diff --git a/l1bridge2infoindexsync/e2e_test.go b/l1bridge2infoindexsync/e2e_test.go index e757be51..e134c1ab 100644 --- a/l1bridge2infoindexsync/e2e_test.go +++ b/l1bridge2infoindexsync/e2e_test.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "math/big" + "path" "strconv" "testing" "time" @@ -98,7 +99,8 @@ func newSimulatedClient(authDeployer, authCaller *bind.TransactOpts) ( return nil, common.Address{}, common.Address{}, nil, nil, err } if precalculatedAddr != checkGERAddr { - return nil, common.Address{}, common.Address{}, nil, nil, errors.New("error deploying bridge") + err = errors.New("error deploying bridge") + return } gerAddr, _, gerContract, err = polygonzkevmglobalexitrootv2.DeployPolygonzkevmglobalexitrootv2( @@ -118,8 +120,8 @@ func newSimulatedClient(authDeployer, authCaller *bind.TransactOpts) ( func TestE2E(t *testing.T) { ctx := context.Background() - dbPathBridgeSync := t.TempDir() - dbPathL1Sync := t.TempDir() + dbPathBridgeSync := path.Join(t.TempDir(), "file::memory:?cache=shared") + dbPathL1Sync := path.Join(t.TempDir(), "file::memory:?cache=shared") dbPathReorg := t.TempDir() dbPathL12InfoSync := t.TempDir() @@ -200,11 +202,11 @@ func TestE2E(t *testing.T) { // Wait for syncer to catch up syncerUpToDate := false var errMsg string + lb, err := client.Client().BlockByNumber(ctx, big.NewInt(int64(rpc.FinalizedBlockNumber))) + require.NoError(t, err) for i := 0; i < 10; i++ { lpb, err := bridge2InfoSync.GetLastProcessedBlock(ctx) require.NoError(t, err) - lb, err := client.Client().BlockByNumber(ctx, big.NewInt(int64(rpc.FinalizedBlockNumber))) - require.NoError(t, err) if lpb == lb.NumberU64() { syncerUpToDate = true diff --git a/l1infotreesync/downloader.go b/l1infotreesync/downloader.go index 060fa6ae..2051f7b5 100644 --- a/l1infotreesync/downloader.go +++ b/l1infotreesync/downloader.go @@ -67,6 +67,7 @@ func buildAppender(client EthClienter, globalExitRoot, rollupManager common.Addr ) } b.Events = append(b.Events, Event{UpdateL1InfoTree: &UpdateL1InfoTree{ + BlockPosition: uint64(l.Index), MainnetExitRoot: l1InfoTreeUpdate.MainnetExitRoot, RollupExitRoot: l1InfoTreeUpdate.RollupExitRoot, ParentHash: b.ParentHash, @@ -98,11 +99,12 @@ func buildAppender(client EthClienter, globalExitRoot, rollupManager common.Addr ) } b.Events = append(b.Events, Event{VerifyBatches: &VerifyBatches{ - RollupID: verifyBatches.RollupID, - NumBatch: verifyBatches.NumBatch, - StateRoot: verifyBatches.StateRoot, - ExitRoot: verifyBatches.ExitRoot, - Aggregator: verifyBatches.Aggregator, + BlockPosition: uint64(l.Index), + RollupID: verifyBatches.RollupID, + NumBatch: verifyBatches.NumBatch, + StateRoot: verifyBatches.StateRoot, + ExitRoot: verifyBatches.ExitRoot, + Aggregator: verifyBatches.Aggregator, }}) return nil @@ -116,11 +118,12 @@ func buildAppender(client EthClienter, globalExitRoot, rollupManager common.Addr ) } b.Events = append(b.Events, Event{VerifyBatches: &VerifyBatches{ - RollupID: verifyBatches.RollupID, - NumBatch: verifyBatches.NumBatch, - StateRoot: verifyBatches.StateRoot, - ExitRoot: verifyBatches.ExitRoot, - Aggregator: verifyBatches.Aggregator, + BlockPosition: uint64(l.Index), + RollupID: verifyBatches.RollupID, + NumBatch: verifyBatches.NumBatch, + StateRoot: verifyBatches.StateRoot, + ExitRoot: verifyBatches.ExitRoot, + Aggregator: verifyBatches.Aggregator, }}) return nil diff --git a/l1infotreesync/e2e_test.go b/l1infotreesync/e2e_test.go index e0437d66..90f7f091 100644 --- a/l1infotreesync/e2e_test.go +++ b/l1infotreesync/e2e_test.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "math/big" + "path" "strconv" "testing" "time" @@ -69,7 +70,7 @@ func newSimulatedClient(auth *bind.TransactOpts) ( func TestE2E(t *testing.T) { ctx := context.Background() - dbPath := t.TempDir() + dbPath := path.Join(t.TempDir(), "file::memory:?cache=shared") privateKey, err := crypto.GenerateKey() require.NoError(t, err) auth, err := bind.NewKeyedTransactorWithChainID(privateKey, big.NewInt(1337)) @@ -108,7 +109,7 @@ func TestE2E(t *testing.T) { require.Equal(t, g, expectedRoot) actualRoot, err := syncer.GetL1InfoTreeRootByIndex(ctx, uint32(i)) require.NoError(t, err) - require.Equal(t, common.Hash(expectedRoot), actualRoot) + require.Equal(t, common.Hash(expectedRoot), actualRoot.Hash) } // Update 3 rollups (verify batches event) 3 times @@ -129,41 +130,11 @@ func TestE2E(t *testing.T) { require.NoError(t, err) actualRollupExitRoot, err := syncer.GetLastRollupExitRoot(ctx) require.NoError(t, err) - require.Equal(t, common.Hash(expectedRollupExitRoot), actualRollupExitRoot, fmt.Sprintf("rollupID: %d, i: %d", rollupID, i)) + require.Equal(t, common.Hash(expectedRollupExitRoot), actualRollupExitRoot.Hash, fmt.Sprintf("rollupID: %d, i: %d", rollupID, i)) } } } -func TestFinalised(t *testing.T) { - ctx := context.Background() - privateKey, err := crypto.GenerateKey() - require.NoError(t, err) - auth, err := bind.NewKeyedTransactorWithChainID(privateKey, big.NewInt(1337)) - require.NoError(t, err) - client, _, _, _, _, err := newSimulatedClient(auth) //nolint:dogsled - require.NoError(t, err) - for i := 0; i < 100; i++ { - client.Commit() - } - - n4, err := client.Client().HeaderByNumber(ctx, big.NewInt(-4)) - require.NoError(t, err) - fmt.Println("-4", n4.Number) - n3, err := client.Client().HeaderByNumber(ctx, big.NewInt(-3)) - require.NoError(t, err) - fmt.Println("-3", n3.Number) - n2, err := client.Client().HeaderByNumber(ctx, big.NewInt(-2)) - require.NoError(t, err) - fmt.Println("-2", n2.Number) - n1, err := client.Client().HeaderByNumber(ctx, big.NewInt(-1)) - require.NoError(t, err) - fmt.Println("-1", n1.Number) - n0, err := client.Client().HeaderByNumber(ctx, nil) - require.NoError(t, err) - fmt.Println("0", n0.Number) - fmt.Printf("amount of blocks latest - finalised: %d", n0.Number.Uint64()-n3.Number.Uint64()) -} - func TestStressAndReorgs(t *testing.T) { const ( totalIterations = 200 // Have tested with much larger number (+10k) @@ -175,7 +146,7 @@ func TestStressAndReorgs(t *testing.T) { ) ctx := context.Background() - dbPathSyncer := t.TempDir() + dbPathSyncer := path.Join(t.TempDir(), "file::memory:?cache=shared") dbPathReorg := t.TempDir() privateKey, err := crypto.GenerateKey() require.NoError(t, err) @@ -224,11 +195,11 @@ func TestStressAndReorgs(t *testing.T) { syncerUpToDate := false var errMsg string + lb, err := client.Client().BlockNumber(ctx) + require.NoError(t, err) for i := 0; i < 50; i++ { lpb, err := syncer.GetLastProcessedBlock(ctx) require.NoError(t, err) - lb, err := client.Client().BlockNumber(ctx) - require.NoError(t, err) if lpb == lb { syncerUpToDate = true @@ -244,18 +215,18 @@ func TestStressAndReorgs(t *testing.T) { require.NoError(t, err) actualRollupExitRoot, err := syncer.GetLastRollupExitRoot(ctx) require.NoError(t, err) - require.Equal(t, common.Hash(expectedRollupExitRoot), actualRollupExitRoot) + require.Equal(t, common.Hash(expectedRollupExitRoot), actualRollupExitRoot.Hash) // Assert L1 Info tree root expectedL1InfoRoot, err := gerSc.GetRoot(&bind.CallOpts{Pending: false}) require.NoError(t, err) expectedGER, err := gerSc.GetLastGlobalExitRoot(&bind.CallOpts{Pending: false}) require.NoError(t, err) - index, actualL1InfoRoot, err := syncer.GetLastL1InfoTreeRootAndIndex(ctx) + lastRoot, err := syncer.GetLastL1InfoTreeRoot(ctx) require.NoError(t, err) - info, err := syncer.GetInfoByIndex(ctx, index) - require.NoError(t, err, fmt.Sprintf("index: %d", index)) + info, err := syncer.GetInfoByIndex(ctx, lastRoot.Index) + require.NoError(t, err, fmt.Sprintf("index: %d", lastRoot.Index)) - require.Equal(t, common.Hash(expectedL1InfoRoot), actualL1InfoRoot) + require.Equal(t, common.Hash(expectedL1InfoRoot), lastRoot.Hash) require.Equal(t, common.Hash(expectedGER), info.GlobalExitRoot, fmt.Sprintf("%+v", info)) } diff --git a/l1infotreesync/l1infotreesync.go b/l1infotreesync/l1infotreesync.go index 69868b55..546a8ead 100644 --- a/l1infotreesync/l1infotreesync.go +++ b/l1infotreesync/l1infotreesync.go @@ -8,6 +8,7 @@ import ( "github.com/0xPolygon/cdk/etherman" "github.com/0xPolygon/cdk/sync" "github.com/0xPolygon/cdk/tree" + "github.com/0xPolygon/cdk/tree/types" "github.com/ethereum/go-ethereum/common" ) @@ -36,7 +37,7 @@ func New( retryAfterErrorPeriod time.Duration, maxRetryAttemptsAfterError int, ) (*L1InfoTreeSync, error) { - processor, err := newProcessor(ctx, dbPath) + processor, err := newProcessor(dbPath) if err != nil { return nil, err } @@ -93,16 +94,16 @@ func (s *L1InfoTreeSync) Start(ctx context.Context) { } // GetL1InfoTreeMerkleProof creates a merkle proof for the L1 Info tree -func (s *L1InfoTreeSync) GetL1InfoTreeMerkleProof( - ctx context.Context, index uint32, -) ([32]common.Hash, common.Hash, error) { +func (s *L1InfoTreeSync) GetL1InfoTreeMerkleProof(ctx context.Context, index uint32) (types.Proof, types.Root, error) { return s.processor.GetL1InfoTreeMerkleProof(ctx, index) } // GetRollupExitTreeMerkleProof creates a merkle proof for the rollup exit tree func (s *L1InfoTreeSync) GetRollupExitTreeMerkleProof( - ctx context.Context, networkID uint32, root common.Hash, -) ([32]common.Hash, error) { + ctx context.Context, + networkID uint32, + root common.Hash, +) (types.Proof, error) { if networkID == 0 { return tree.EmptyProof, nil } @@ -122,24 +123,18 @@ func (s *L1InfoTreeSync) GetInfoByIndex(ctx context.Context, index uint32) (*L1I } // GetL1InfoTreeRootByIndex returns the root of the L1 info tree at the moment the leaf with the given index was added -func (s *L1InfoTreeSync) GetL1InfoTreeRootByIndex(ctx context.Context, index uint32) (common.Hash, error) { - tx, err := s.processor.db.BeginRo(ctx) - if err != nil { - return common.Hash{}, err - } - defer tx.Rollback() - - return s.processor.l1InfoTree.GetRootByIndex(tx, index) +func (s *L1InfoTreeSync) GetL1InfoTreeRootByIndex(ctx context.Context, index uint32) (types.Root, error) { + return s.processor.l1InfoTree.GetRootByIndex(ctx, index) } // GetLastRollupExitRoot return the last rollup exit root processed -func (s *L1InfoTreeSync) GetLastRollupExitRoot(ctx context.Context) (common.Hash, error) { +func (s *L1InfoTreeSync) GetLastRollupExitRoot(ctx context.Context) (types.Root, error) { return s.processor.rollupExitTree.GetLastRoot(ctx) } -// GetLastL1InfoTreeRootAndIndex return the last root and index processed from the L1 Info tree -func (s *L1InfoTreeSync) GetLastL1InfoTreeRootAndIndex(ctx context.Context) (uint32, common.Hash, error) { - return s.processor.l1InfoTree.GetLastIndexAndRoot(ctx) +// GetLastL1InfoTreeRoot return the last root and index processed from the L1 Info tree +func (s *L1InfoTreeSync) GetLastL1InfoTreeRoot(ctx context.Context) (types.Root, error) { + return s.processor.l1InfoTree.GetLastRoot(ctx) } // GetLastProcessedBlock return the last processed block diff --git a/l1infotreesync/migrations/l1infotreesync0001.sql b/l1infotreesync/migrations/l1infotreesync0001.sql new file mode 100644 index 00000000..39a45dd4 --- /dev/null +++ b/l1infotreesync/migrations/l1infotreesync0001.sql @@ -0,0 +1,22 @@ +-- +migrate Down +DROP TABLE IF EXISTS block; +DROP TABLE IF EXISTS claim; +DROP TABLE IF EXISTS bridge; + +-- +migrate Up +CREATE TABLE block ( + num BIGINT PRIMARY KEY +); + +CREATE TABLE l1info_leaf ( + block_num INTEGER NOT NULL REFERENCES block(num) ON DELETE CASCADE, + block_pos INTEGER NOT NULL, + position INTEGER NOT NULL, + previous_block_hash VARCHAR NOT NULL, + timestamp INTEGER NOT NULL, + mainnet_exit_root VARCHAR NOT NULL, + rollup_exit_root VARCHAR NOT NULL, + global_exit_root VARCHAR NOT NULL, + hash VARCHAR NOT NULL, + PRIMARY KEY (block_num, block_pos) +); diff --git a/l1infotreesync/migrations/migrations.go b/l1infotreesync/migrations/migrations.go new file mode 100644 index 00000000..768dde37 --- /dev/null +++ b/l1infotreesync/migrations/migrations.go @@ -0,0 +1,39 @@ +package migrations + +import ( + _ "embed" + + "github.com/0xPolygon/cdk/db" + "github.com/0xPolygon/cdk/db/types" + treeMigrations "github.com/0xPolygon/cdk/tree/migrations" +) + +const ( + RollupExitTreePrefix = "rollup_exit_" + L1InfoTreePrefix = "l1_info_" +) + +//go:embed l1infotreesync0001.sql +var mig001 string + +func RunMigrations(dbPath string) error { + migrations := []types.Migration{ + { + ID: "l1infotreesync0001", + SQL: mig001, + }, + } + for _, tm := range treeMigrations.Migrations { + migrations = append(migrations, types.Migration{ + ID: tm.ID, + SQL: tm.SQL, + Prefix: RollupExitTreePrefix, + }) + migrations = append(migrations, types.Migration{ + ID: tm.ID, + SQL: tm.SQL, + Prefix: L1InfoTreePrefix, + }) + } + return db.RunMigrations(dbPath, migrations) +} diff --git a/l1infotreesync/processor.go b/l1infotreesync/processor.go index a6fa28a1..781d4478 100644 --- a/l1infotreesync/processor.go +++ b/l1infotreesync/processor.go @@ -2,55 +2,38 @@ package l1infotreesync import ( "context" + "database/sql" "encoding/binary" - "encoding/json" "errors" "fmt" - "github.com/0xPolygon/cdk/common" + "github.com/0xPolygon/cdk/db" + "github.com/0xPolygon/cdk/l1infotreesync/migrations" "github.com/0xPolygon/cdk/log" "github.com/0xPolygon/cdk/sync" "github.com/0xPolygon/cdk/tree" + treeTypes "github.com/0xPolygon/cdk/tree/types" ethCommon "github.com/ethereum/go-ethereum/common" "github.com/iden3/go-iden3-crypto/keccak256" - "github.com/ledgerwatch/erigon-lib/kv" - "github.com/ledgerwatch/erigon-lib/kv/mdbx" + "github.com/russross/meddler" "golang.org/x/crypto/sha3" ) -const ( - dbPrefix = "l1infotreesync" - l1InfoTreeSuffix = "-l1infotree" - rollupExitTreeSuffix = "-rollupexittree" - - // infoTable stores the information of L1 Info Tree (the leaves) - // Key: index (uint32 converted to bytes) - // Value: JSON of storeLeaf struct - infoTable = dbPrefix + "-info" - // blockTable stores the first and last index of L1 Info Tree that have been updated on - // Value: JSON of blockWithLeafs - blockTable = dbPrefix + "-block" - // lastBlockTable used to store the last block processed. This is needed to know the last processed blcok - lastBlockTable = dbPrefix + "-lastBlock" - - treeHeight uint8 = 32 -) - var ( ErrBlockNotProcessed = errors.New("given block(s) have not been processed yet") ErrNotFound = errors.New("not found") ErrNoBlock0 = errors.New("blockNum must be greater than 0") - lastBlockKey = []byte("lb") ) type processor struct { - db kv.RwDB + db *sql.DB l1InfoTree *tree.AppendOnlyTree rollupExitTree *tree.UpdatableTree } // UpdateL1InfoTree representation of the UpdateL1InfoTree event type UpdateL1InfoTree struct { + BlockPosition uint64 MainnetExitRoot ethCommon.Hash RollupExitRoot ethCommon.Hash ParentHash ethCommon.Hash @@ -59,11 +42,12 @@ type UpdateL1InfoTree struct { // VerifyBatches representation of the VerifyBatches and VerifyBatchesTrustedAggregator events type VerifyBatches struct { - RollupID uint32 - NumBatch uint64 - StateRoot ethCommon.Hash - ExitRoot ethCommon.Hash - Aggregator ethCommon.Address + BlockPosition uint64 + RollupID uint32 + NumBatch uint64 + StateRoot ethCommon.Hash + ExitRoot ethCommon.Hash + Aggregator ethCommon.Address } type InitL1InfoRootMap struct { @@ -79,43 +63,28 @@ type Event struct { // L1InfoTreeLeaf representation of a leaf of the L1 Info tree type L1InfoTreeLeaf struct { - L1InfoTreeIndex uint32 - PreviousBlockHash ethCommon.Hash - BlockNumber uint64 - Timestamp uint64 - MainnetExitRoot ethCommon.Hash - RollupExitRoot ethCommon.Hash - GlobalExitRoot ethCommon.Hash -} - -type storeLeaf struct { - BlockNumber uint64 - MainnetExitRoot ethCommon.Hash - RollupExitRoot ethCommon.Hash - ParentHash ethCommon.Hash - Index uint32 - Timestamp uint64 + BlockNumber uint64 `meddler:"block_num"` + BlockPosition uint64 `meddler:"block_pos"` + L1InfoTreeIndex uint32 `meddler:"position"` + PreviousBlockHash ethCommon.Hash `meddler:"previous_block_hash,hash"` + Timestamp uint64 `meddler:"timestamp"` + MainnetExitRoot ethCommon.Hash `meddler:"mainnet_exit_root,hash"` + RollupExitRoot ethCommon.Hash `meddler:"rollup_exit_root,hash"` + GlobalExitRoot ethCommon.Hash `meddler:"global_exit_root,hash"` + Hash ethCommon.Hash `meddler:"hash,hash"` } // Hash as expected by the tree -func (l *storeLeaf) Hash() ethCommon.Hash { +func (l *L1InfoTreeLeaf) hash() ethCommon.Hash { var res [32]byte t := make([]byte, 8) //nolint:gomnd binary.BigEndian.PutUint64(t, l.Timestamp) - copy(res[:], keccak256.Hash(l.GlobalExitRoot().Bytes(), l.ParentHash.Bytes(), t)) - + copy(res[:], keccak256.Hash(l.globalExitRoot().Bytes(), l.PreviousBlockHash.Bytes(), t)) return res } -type blockWithLeafs struct { - // inclusive - FirstIndex uint32 - // not inclusive - LastIndex uint32 -} - // GlobalExitRoot returns the GER -func (l *storeLeaf) GlobalExitRoot() ethCommon.Hash { +func (l *L1InfoTreeLeaf) globalExitRoot() ethCommon.Hash { var gerBytes [32]byte hasher := sha3.NewLegacyKeccak256() hasher.Write(l.MainnetExitRoot[:]) @@ -125,65 +94,32 @@ func (l *storeLeaf) GlobalExitRoot() ethCommon.Hash { return gerBytes } -func newProcessor(ctx context.Context, dbPath string) (*processor, error) { - tableCfgFunc := func(defaultBuckets kv.TableCfg) kv.TableCfg { - cfg := kv.TableCfg{ - infoTable: {}, - blockTable: {}, - lastBlockTable: {}, - } - tree.AddTables(cfg, dbPrefix+rollupExitTreeSuffix) - tree.AddTables(cfg, dbPrefix+l1InfoTreeSuffix) - - return cfg - } - db, err := mdbx.NewMDBX(nil). - Path(dbPath). - WithTableCfg(tableCfgFunc). - Open() - if err != nil { - return nil, err - } - p := &processor{ - db: db, - } - - l1InfoTree, err := tree.NewAppendOnlyTree(ctx, db, dbPrefix+l1InfoTreeSuffix) +func newProcessor(dbPath string) (*processor, error) { + err := migrations.RunMigrations(dbPath) if err != nil { return nil, err } - p.l1InfoTree = l1InfoTree - rollupExitTree, err := tree.NewUpdatableTree(ctx, db, dbPrefix+rollupExitTreeSuffix) + db, err := db.NewSQLiteDB(dbPath) if err != nil { return nil, err } - p.rollupExitTree = rollupExitTree - - return p, nil + return &processor{ + db: db, + l1InfoTree: tree.NewAppendOnlyTree(db, migrations.L1InfoTreePrefix), + rollupExitTree: tree.NewUpdatableTree(db, migrations.RollupExitTreePrefix), + }, nil } // GetL1InfoTreeMerkleProof creates a merkle proof for the L1 Info tree func (p *processor) GetL1InfoTreeMerkleProof( ctx context.Context, index uint32, -) ([32]ethCommon.Hash, ethCommon.Hash, error) { - tx, err := p.db.BeginRo(ctx) +) (treeTypes.Proof, treeTypes.Root, error) { + root, err := p.l1InfoTree.GetRootByIndex(ctx, index) if err != nil { - return tree.EmptyProof, ethCommon.Hash{}, err + return treeTypes.Proof{}, treeTypes.Root{}, err } - defer tx.Rollback() - - root, err := p.l1InfoTree.GetRootByIndex(tx, index) - if err != nil { - return tree.EmptyProof, ethCommon.Hash{}, err - } - - proof, err := p.l1InfoTree.GetProof(ctx, index, root) - if err != nil { - return tree.EmptyProof, ethCommon.Hash{}, err - } - - // TODO: check if we need to return root or wat - return proof, root, nil + proof, err := p.l1InfoTree.GetProof(ctx, root.Index, root.Hash) + return proof, root, err } // GetLatestInfoUntilBlock returns the most recent L1InfoTreeLeaf that occurred before or at blockNum. @@ -192,11 +128,16 @@ func (p *processor) GetLatestInfoUntilBlock(ctx context.Context, blockNum uint64 if blockNum == 0 { return nil, ErrNoBlock0 } - tx, err := p.db.BeginRo(ctx) + tx, err := p.db.BeginTx(ctx, &sql.TxOptions{ReadOnly: true}) if err != nil { return nil, err } - defer tx.Rollback() + defer func() { + if err := tx.Rollback(); err != nil { + log.Warnf("error rolling back tx: %v", err) + } + }() + lpb, err := p.getLastProcessedBlockWithTx(tx) if err != nil { return nil, err @@ -204,154 +145,78 @@ func (p *processor) GetLatestInfoUntilBlock(ctx context.Context, blockNum uint64 if lpb < blockNum { return nil, ErrBlockNotProcessed } - iter, err := tx.RangeDescend(blockTable, common.Uint64ToBytes(blockNum), common.Uint64ToBytes(0), 1) - if err != nil { - return nil, fmt.Errorf( - "error calling RangeDescend(blockTable, %d, 0, 1): %w", blockNum, err, - ) - } - k, v, err := iter.Next() + + info := &L1InfoTreeLeaf{} + err = meddler.QueryRow( + tx, info, + `SELECT * FROM l1info_leaf ORDER BY block_num DESC, block_pos DESC LIMIT 1;`, + ) if err != nil { + if errors.Is(err, sql.ErrNoRows) { + return nil, ErrNotFound + } return nil, err } - if k == nil { - return nil, ErrNotFound - } - blk := blockWithLeafs{} - if err := json.Unmarshal(v, &blk); err != nil { - return nil, err - } - - return p.getInfoByIndexWithTx(tx, blk.LastIndex-1) + return info, nil } // GetInfoByIndex returns the value of a leaf (not the hash) of the L1 info tree func (p *processor) GetInfoByIndex(ctx context.Context, index uint32) (*L1InfoTreeLeaf, error) { - tx, err := p.db.BeginRo(ctx) - if err != nil { - return nil, err - } - defer tx.Rollback() - - return p.getInfoByIndexWithTx(tx, index) + return p.getInfoByIndexWithTx(p.db, index) } -func (p *processor) getInfoByIndexWithTx(tx kv.Tx, index uint32) (*L1InfoTreeLeaf, error) { - infoBytes, err := tx.GetOne(infoTable, common.Uint32ToBytes(index)) - if err != nil { - return nil, err - } - if infoBytes == nil { - return nil, ErrNotFound - } - var info storeLeaf - if err := json.Unmarshal(infoBytes, &info); err != nil { - return nil, err - } - - return &L1InfoTreeLeaf{ - L1InfoTreeIndex: info.Index, - PreviousBlockHash: info.ParentHash, - BlockNumber: info.BlockNumber, - Timestamp: info.Timestamp, - MainnetExitRoot: info.MainnetExitRoot, - RollupExitRoot: info.RollupExitRoot, - GlobalExitRoot: info.GlobalExitRoot(), - }, nil +func (p *processor) getInfoByIndexWithTx(tx db.DBer, index uint32) (*L1InfoTreeLeaf, error) { + info := &L1InfoTreeLeaf{} + return info, meddler.QueryRow( + tx, info, + `SELECT * FROM l1info_leaf WHERE position = $1;`, index, + ) } // GetLastProcessedBlock returns the last processed block func (p *processor) GetLastProcessedBlock(ctx context.Context) (uint64, error) { - tx, err := p.db.BeginRo(ctx) - if err != nil { - return 0, err - } - defer tx.Rollback() - - return p.getLastProcessedBlockWithTx(tx) + return p.getLastProcessedBlockWithTx(p.db) } -func (p *processor) getLastProcessedBlockWithTx(tx kv.Tx) (uint64, error) { - blockNumBytes, err := tx.GetOne(lastBlockTable, lastBlockKey) - if err != nil { - return 0, err - } else if blockNumBytes == nil { +func (p *processor) getLastProcessedBlockWithTx(tx db.Querier) (uint64, error) { + var lastProcessedBlock uint64 + row := tx.QueryRow("SELECT num FROM BLOCK ORDER BY num DESC LIMIT 1;") + err := row.Scan(&lastProcessedBlock) + if errors.Is(err, sql.ErrNoRows) { return 0, nil } - - return common.BytesToUint64(blockNumBytes), nil + return lastProcessedBlock, err } // Reorg triggers a purge and reset process on the processor to leaf it on a state // as if the last block processed was firstReorgedBlock-1 func (p *processor) Reorg(ctx context.Context, firstReorgedBlock uint64) error { - tx, err := p.db.BeginRw(ctx) - if err != nil { - return err - } - - c, err := tx.Cursor(blockTable) + tx, err := db.NewTx(ctx, p.db) if err != nil { return err } - defer c.Close() - - firstKey := common.Uint64ToBytes(firstReorgedBlock) - firstReorgedL1InfoTreeIndex := int64(-1) - for blkKey, blkValue, err := c.Seek(firstKey); blkKey != nil; blkKey, blkValue, err = c.Next() { + defer func() { if err != nil { - tx.Rollback() - - return err - } - var blk blockWithLeafs - if err := json.Unmarshal(blkValue, &blk); err != nil { - tx.Rollback() - - return err - } - for i := blk.FirstIndex; i < blk.LastIndex; i++ { - if firstReorgedL1InfoTreeIndex == -1 { - firstReorgedL1InfoTreeIndex = int64(i) - } - if err := p.deleteLeaf(tx, i); err != nil { - tx.Rollback() - - return err + if errRllbck := tx.Rollback(); errRllbck != nil { + log.Errorf("error while rolling back tx %v", errRllbck) } } - if err := tx.Delete(blockTable, blkKey); err != nil { - tx.Rollback() - - return err - } - } - if err := p.updateLastProcessedBlock(tx, firstReorgedBlock-1); err != nil { - tx.Rollback() + }() + _, err = tx.Exec(`DELETE FROM block WHERE num >= $1;`, firstReorgedBlock) + if err != nil { return err } - var rollbackL1InfoTree func() - if firstReorgedL1InfoTreeIndex != -1 { - rollbackL1InfoTree, err = p.l1InfoTree.Reorg(tx, uint32(firstReorgedL1InfoTreeIndex)) - if err != nil { - tx.Rollback() - rollbackL1InfoTree() - return err - } + if err = p.l1InfoTree.Reorg(tx, firstReorgedBlock); err != nil { + return err } - if err := tx.Commit(); err != nil { - rollbackL1InfoTree() + if err = p.rollupExitTree.Reorg(tx, firstReorgedBlock); err != nil { return err } - return nil -} - -func (p *processor) deleteLeaf(tx kv.RwTx, index uint32) error { - if err := tx.Delete(infoTable, common.Uint32ToBytes(index)); err != nil { + if err := tx.Commit(); err != nil { return err } @@ -361,161 +226,95 @@ func (p *processor) deleteLeaf(tx kv.RwTx, index uint32) error { // ProcessBlock process the events of the block to build the rollup exit tree and the l1 info tree // and updates the last processed block (can be called without events for that purpose) func (p *processor) ProcessBlock(ctx context.Context, b sync.Block) error { - tx, err := p.db.BeginRw(ctx) + tx, err := db.NewTx(ctx, p.db) if err != nil { return err } - events := make([]Event, 0, len(b.Events)) - rollupExitTreeRollback := func() {} - l1InfoTreeRollback := func() {} - rollback := func() { - tx.Rollback() - rollupExitTreeRollback() - l1InfoTreeRollback() - } - l1InfoTreeLeavesToAdd := []tree.Leaf{} - rollupExitTreeLeavesToAdd := []tree.Leaf{} - if len(b.Events) > 0 { - var initialL1InfoIndex uint32 - var l1InfoLeavesAdded uint32 - lastIndex, err := p.getLastIndex(tx) - if errors.Is(err, ErrNotFound) { - initialL1InfoIndex = 0 - } else if err != nil { - rollback() - - return err - } else { - initialL1InfoIndex = lastIndex + 1 - } - for _, e := range b.Events { - event, ok := e.(Event) - if !ok { - log.Errorf("unexpected type %T in events", e) - } - events = append(events, event) - if event.UpdateL1InfoTree != nil { - index := initialL1InfoIndex + l1InfoLeavesAdded - leafToStore := storeLeaf{ - BlockNumber: b.Num, - Index: index, - MainnetExitRoot: event.UpdateL1InfoTree.MainnetExitRoot, - RollupExitRoot: event.UpdateL1InfoTree.RollupExitRoot, - ParentHash: event.UpdateL1InfoTree.ParentHash, - Timestamp: event.UpdateL1InfoTree.Timestamp, - } - if err := p.storeLeafInfo(tx, leafToStore); err != nil { - rollback() - - return err - } - l1InfoTreeLeavesToAdd = append(l1InfoTreeLeavesToAdd, tree.Leaf{ - Index: leafToStore.Index, - Hash: leafToStore.Hash(), - }) - l1InfoLeavesAdded++ - } - - if event.VerifyBatches != nil { - rollupExitTreeLeavesToAdd = append(rollupExitTreeLeavesToAdd, tree.Leaf{ - Index: event.VerifyBatches.RollupID - 1, - Hash: event.VerifyBatches.ExitRoot, - }) - } - - if event.InitL1InfoRootMap != nil { - // TODO: indicate that l1 Info tree indexes before the one on this - // event are not safe to use - log.Debugf("TODO: handle InitL1InfoRootMap event") + defer func() { + if err != nil { + if errRllbck := tx.Rollback(); errRllbck != nil { + log.Errorf("error while rolling back tx %v", errRllbck) } } - if l1InfoLeavesAdded > 0 { - bwl := blockWithLeafs{ - FirstIndex: initialL1InfoIndex, - LastIndex: initialL1InfoIndex + l1InfoLeavesAdded, + }() + + if _, err := tx.Exec(`INSERT INTO block (num) VALUES ($1)`, b.Num); err != nil { + return fmt.Errorf("err: %w", err) + } + + var initialL1InfoIndex uint32 + var l1InfoLeavesAdded uint32 + lastIndex, err := p.getLastIndex(tx) + if errors.Is(err, ErrNotFound) { + initialL1InfoIndex = 0 + err = nil + } else if err != nil { + return fmt.Errorf("err: %w", err) + } else { + initialL1InfoIndex = lastIndex + 1 + } + for _, e := range b.Events { + event, ok := e.(Event) + if !ok { + return errors.New("failed to convert from sync.Block.Event into Event") + } + if event.UpdateL1InfoTree != nil { + index := initialL1InfoIndex + l1InfoLeavesAdded + info := &L1InfoTreeLeaf{ + BlockNumber: b.Num, + BlockPosition: event.UpdateL1InfoTree.BlockPosition, + L1InfoTreeIndex: index, + PreviousBlockHash: event.UpdateL1InfoTree.ParentHash, + Timestamp: event.UpdateL1InfoTree.Timestamp, + MainnetExitRoot: event.UpdateL1InfoTree.MainnetExitRoot, + RollupExitRoot: event.UpdateL1InfoTree.RollupExitRoot, } - blockValue, err := json.Marshal(bwl) + info.GlobalExitRoot = info.globalExitRoot() + info.Hash = info.hash() + err = meddler.Insert(tx, "l1info_leaf", info) if err != nil { - rollback() - - return err + return fmt.Errorf("err: %w", err) } - if err := tx.Put(blockTable, common.Uint64ToBytes(b.Num), blockValue); err != nil { - rollback() - - return err - } - l1InfoTreeRollback, err = p.l1InfoTree.AddLeaves(tx, l1InfoTreeLeavesToAdd) + err = p.l1InfoTree.AddLeaf(tx, info.BlockNumber, info.BlockPosition, treeTypes.Leaf{ + Index: info.L1InfoTreeIndex, + Hash: info.Hash, + }) if err != nil { - rollback() - - return err + return fmt.Errorf("err: %w", err) } + l1InfoLeavesAdded++ } - if len(rollupExitTreeLeavesToAdd) > 0 { - rollupExitTreeRollback, err = p.rollupExitTree.UpseartLeaves(tx, rollupExitTreeLeavesToAdd, b.Num) + if event.VerifyBatches != nil { + err = p.rollupExitTree.UpsertLeaf(tx, b.Num, event.VerifyBatches.BlockPosition, treeTypes.Leaf{ + Index: event.VerifyBatches.RollupID - 1, + Hash: event.VerifyBatches.ExitRoot, + }) if err != nil { - rollback() - - return err + return fmt.Errorf("err: %w", err) } } - } - if err := p.updateLastProcessedBlock(tx, b.Num); err != nil { - rollback() - return err + if event.InitL1InfoRootMap != nil { + // TODO: indicate that l1 Info tree indexes before the one on this + // event are not safe to use + log.Debugf("TODO: handle InitL1InfoRootMap event") + } } if err := tx.Commit(); err != nil { - rollback() - - return err + return fmt.Errorf("err: %w", err) } - log.Infof("block %d processed with events: %+v", b.Num, events) - + log.Infof("block %d processed with %d events", b.Num, len(b.Events)) return nil } -func (p *processor) getLastIndex(tx kv.Tx) (uint32, error) { - bNum, err := p.getLastProcessedBlockWithTx(tx) - if err != nil { - return 0, err - } - if bNum == 0 { - return 0, nil - } - iter, err := tx.RangeDescend(blockTable, common.Uint64ToBytes(bNum), common.Uint64ToBytes(0), 1) - if err != nil { - return 0, err - } - _, blkBytes, err := iter.Next() - if err != nil { - return 0, err - } - if blkBytes == nil { +func (p *processor) getLastIndex(tx db.Querier) (uint32, error) { + var lastProcessedIndex uint32 + row := tx.QueryRow("SELECT position FROM l1info_leaf ORDER BY block_num DESC, block_pos DESC LIMIT 1;") + err := row.Scan(&lastProcessedIndex) + if errors.Is(err, sql.ErrNoRows) { return 0, ErrNotFound } - var blk blockWithLeafs - if err := json.Unmarshal(blkBytes, &blk); err != nil { - return 0, err - } - - return blk.LastIndex - 1, nil -} - -func (p *processor) storeLeafInfo(tx kv.RwTx, leaf storeLeaf) error { - leafValue, err := json.Marshal(leaf) - if err != nil { - return err - } - - return tx.Put(infoTable, common.Uint32ToBytes(leaf.Index), leafValue) -} - -func (p *processor) updateLastProcessedBlock(tx kv.RwTx, blockNum uint64) error { - blockNumBytes := common.Uint64ToBytes(blockNum) - - return tx.Put(lastBlockTable, lastBlockKey, blockNumBytes) + return lastProcessedIndex, err } diff --git a/lastgersync/e2e_test.go b/lastgersync/e2e_test.go index 111ffa07..979d55a2 100644 --- a/lastgersync/e2e_test.go +++ b/lastgersync/e2e_test.go @@ -40,7 +40,7 @@ func TestE2E(t *testing.T) { _, err := env.GERL1Contract.UpdateExitRoot(env.AuthL1, common.HexToHash(strconv.Itoa(i))) require.NoError(t, err) env.L1Client.Commit() - time.Sleep(time.Millisecond * 50) + time.Sleep(time.Millisecond * 150) expectedGER, err := env.GERL1Contract.GetLastGlobalExitRoot(&bind.CallOpts{Pending: false}) require.NoError(t, err) isInjected, err := env.AggOracleSender.IsGERAlreadyInjected(expectedGER) diff --git a/lastgersync/evmdownloader.go b/lastgersync/evmdownloader.go index 10a8c4f2..91e05c7a 100644 --- a/lastgersync/evmdownloader.go +++ b/lastgersync/evmdownloader.go @@ -128,16 +128,16 @@ func (d *downloader) Download(ctx context.Context, fromBlock uint64, downloadedC } func (d *downloader) getGERsFromIndex(ctx context.Context, fromL1InfoTreeIndex uint32) ([]Event, error) { - lastIndex, _, err := d.l1InfoTreesync.GetLastL1InfoTreeRootAndIndex(ctx) + lastRoot, err := d.l1InfoTreesync.GetLastL1InfoTreeRoot(ctx) if errors.Is(err, tree.ErrNotFound) { return nil, nil } if err != nil { - return nil, fmt.Errorf("error calling GetLastL1InfoTreeRootAndIndex: %w", err) + return nil, fmt.Errorf("error calling GetLastL1InfoTreeRoot: %w", err) } gers := []Event{} - for i := fromL1InfoTreeIndex; i <= lastIndex; i++ { + for i := fromL1InfoTreeIndex; i <= lastRoot.Index; i++ { info, err := d.l1InfoTreesync.GetInfoByIndex(ctx, i) if err != nil { return nil, fmt.Errorf("error calling GetInfoByIndex: %w", err) diff --git a/sync/evmdriver.go b/sync/evmdriver.go index f42d040e..ae7388e0 100644 --- a/sync/evmdriver.go +++ b/sync/evmdriver.go @@ -122,7 +122,7 @@ func (d *EVMDriver) handleNewBlock(ctx context.Context, b EVMBlock) { err := d.processor.ProcessBlock(ctx, blockToProcess) if err != nil { attempts++ - d.log.Errorf("error processing events for blcok %d, err: ", b.Num, err) + d.log.Errorf("error processing events for block %d, err: ", b.Num, err) d.rh.Handle("handleNewBlock", attempts) continue } diff --git a/test/helpers/aggoracle_e2e.go b/test/helpers/aggoracle_e2e.go index 311ba189..77f5d99a 100644 --- a/test/helpers/aggoracle_e2e.go +++ b/test/helpers/aggoracle_e2e.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "math/big" + "path" "testing" "time" @@ -26,10 +27,13 @@ import ( ) const ( - NetworkIDL2 = uint32(1) - chainID = 1337 - initialBalance = "10000000000000000000000000" - blockGasLimit = uint64(999999999999999999) + NetworkIDL2 = uint32(1) + chainID = 1337 + initialBalance = "10000000000000000000000000" + blockGasLimit = uint64(999999999999999999) + syncBlockChunkSize = 10 + retries = 3 + periodRetry = time.Millisecond * 100 ) type AggoracleWithEVMChainEnv struct { @@ -111,8 +115,8 @@ func CommonSetup(t *testing.T) ( reorg, err := reorgdetector.New(l1Client.Client(), reorgdetector.Config{DBPath: dbPathReorgDetector}) require.NoError(t, err) // Syncer - dbPathSyncer := t.TempDir() - syncer, err := l1infotreesync.New(ctx, dbPathSyncer, gerL1Addr, common.Address{}, 10, etherman.LatestBlock, reorg, l1Client.Client(), time.Millisecond, 0, 100*time.Millisecond, 3) //nolint:gomnd + dbPathSyncer := path.Join(t.TempDir(), "file::memory:?cache=shared") + syncer, err := l1infotreesync.New(ctx, dbPathSyncer, gerL1Addr, common.Address{}, syncBlockChunkSize, etherman.LatestBlock, reorg, l1Client.Client(), time.Millisecond, 0, periodRetry, retries) require.NoError(t, err) go syncer.Start(ctx) diff --git a/tree/appendonlytree.go b/tree/appendonlytree.go index 67a1a8f8..20d22ec1 100644 --- a/tree/appendonlytree.go +++ b/tree/appendonlytree.go @@ -1,187 +1,114 @@ package tree import ( - "context" + "database/sql" + "errors" "fmt" - dbCommon "github.com/0xPolygon/cdk/common" + "github.com/0xPolygon/cdk/db" + "github.com/0xPolygon/cdk/tree/types" "github.com/ethereum/go-ethereum/common" - "github.com/ledgerwatch/erigon-lib/kv" ) // AppendOnlyTree is a tree where leaves are added sequentially (by index) type AppendOnlyTree struct { *Tree - lastLeftCache [DefaultHeight]common.Hash + lastLeftCache [types.DefaultHeight]common.Hash lastIndex int64 } // NewAppendOnlyTree creates a AppendOnlyTree -func NewAppendOnlyTree(ctx context.Context, db kv.RwDB, dbPrefix string) (*AppendOnlyTree, error) { +func NewAppendOnlyTree(db *sql.DB, dbPrefix string) *AppendOnlyTree { t := newTree(db, dbPrefix) - at := &AppendOnlyTree{Tree: t} - if err := at.initLastLeftCacheAndLastDepositCount(ctx); err != nil { - return nil, err + return &AppendOnlyTree{ + Tree: t, + // -1 is used to indicate no leafs, 0 means the first leaf is added (at index 0) and so on. + // In order to differentiate the "cache not initialised" we need any value smaller than -1 + lastIndex: -2, } - return at, nil } -// AddLeaves adds a list leaves into the tree. The indexes of the leaves must be consecutive, -// starting by the index of the last leaf added +1 -// It returns a function that must be called to rollback the changes done by this interaction -func (t *AppendOnlyTree) AddLeaves(tx kv.RwTx, leaves []Leaf) (func(), error) { - // Sanity check - if len(leaves) == 0 { - return func() {}, nil - } - - backupIndx := t.lastIndex - backupCache := [DefaultHeight]common.Hash{} - copy(backupCache[:], t.lastLeftCache[:]) - rollback := func() { - t.lastIndex = backupIndx - t.lastLeftCache = backupCache - } - - for _, leaf := range leaves { - if err := t.addLeaf(tx, leaf); err != nil { - return rollback, err - } - } - - return rollback, nil -} - -func (t *AppendOnlyTree) addLeaf(tx kv.RwTx, leaf Leaf) error { +func (t *AppendOnlyTree) AddLeaf(tx db.Txer, blockNum, blockPosition uint64, leaf types.Leaf) error { if int64(leaf.Index) != t.lastIndex+1 { - return fmt.Errorf( - "mismatched index. Expected: %d, actual: %d", - t.lastIndex+1, leaf.Index, - ) + // rebuild cache + if err := t.initCache(tx); err != nil { + return err + } + if int64(leaf.Index) != t.lastIndex+1 { + return fmt.Errorf( + "mismatched index. Expected: %d, actual: %d", + t.lastIndex+1, leaf.Index, + ) + } } // Calculate new tree nodes currentChildHash := leaf.Hash - newNodes := []treeNode{} - for h := uint8(0); h < DefaultHeight; h++ { - var parent treeNode + newNodes := []types.TreeNode{} + for h := uint8(0); h < types.DefaultHeight; h++ { + var parent types.TreeNode if leaf.Index&(1< 0 { // Add child to the right - parent = treeNode{ - left: t.lastLeftCache[h], - right: currentChildHash, - } + parent = newTreeNode(t.lastLeftCache[h], currentChildHash) } else { // Add child to the left - parent = treeNode{ - left: currentChildHash, - right: t.zeroHashes[h], - } + parent = newTreeNode(currentChildHash, t.zeroHashes[h]) // Update cache - // TODO: review this part of the logic, skipping? optimisation? - // from OG implementation t.lastLeftCache[h] = currentChildHash } - currentChildHash = parent.hash() + currentChildHash = parent.Hash newNodes = append(newNodes, parent) } // store root - if err := t.storeRoot(tx, uint64(leaf.Index), currentChildHash); err != nil { - return fmt.Errorf("failed to store root: %w", err) - } - root := currentChildHash - if err := tx.Put(t.rootTable, dbCommon.Uint64ToBytes(uint64(leaf.Index)), root[:]); err != nil { + if err := t.storeRoot(tx, types.Root{ + Hash: currentChildHash, + Index: leaf.Index, + BlockNum: blockNum, + BlockPosition: blockPosition, + }); err != nil { return err } + // store nodes if err := t.storeNodes(tx, newNodes); err != nil { return err } t.lastIndex++ + tx.AddRollbackCallback(func() { t.lastIndex-- }) return nil } -// GetRootByIndex returns the root of the tree as it was right after adding the leaf with index -func (t *AppendOnlyTree) GetRootByIndex(tx kv.Tx, index uint32) (common.Hash, error) { - return t.getRootByIndex(tx, uint64(index)) -} - -func (t *AppendOnlyTree) GetIndexByRoot(ctx context.Context, root common.Hash) (uint32, error) { - tx, err := t.db.BeginRo(ctx) - if err != nil { - return 0, err - } - defer tx.Rollback() - index, err := t.getIndexByRoot(tx, root) - return uint32(index), err -} - -// GetLastIndexAndRoot returns the last index and root added to the tree -func (t *AppendOnlyTree) GetLastIndexAndRoot(ctx context.Context) (uint32, common.Hash, error) { - tx, err := t.db.BeginRo(ctx) - if err != nil { - return 0, common.Hash{}, err - } - defer tx.Rollback() - i, root, err := t.getLastIndexAndRootWithTx(tx) - if err != nil { - return 0, common.Hash{}, err - } - if i == -1 { - return 0, common.Hash{}, ErrNotFound - } - return uint32(i), root, nil -} - -func (t *AppendOnlyTree) initLastLeftCacheAndLastDepositCount(ctx context.Context) error { - tx, err := t.db.BeginRw(ctx) - if err != nil { - return err - } - defer tx.Rollback() - - root, err := t.initLastIndex(tx) +func (t *AppendOnlyTree) initCache(tx db.Txer) error { + siblings := [types.DefaultHeight]common.Hash{} + lastRoot, err := t.getLastRootWithTx(tx) if err != nil { + if errors.Is(err, ErrNotFound) { + t.lastIndex = -1 + t.lastLeftCache = siblings + return nil + } return err } - return t.initLastLeftCache(tx, t.lastIndex, root) -} - -func (t *AppendOnlyTree) initLastIndex(tx kv.Tx) (common.Hash, error) { - lastIndex, root, err := t.getLastIndexAndRootWithTx(tx) - if err != nil { - return common.Hash{}, err - } - t.lastIndex = lastIndex - return root, nil -} - -func (t *AppendOnlyTree) initLastLeftCache(tx kv.Tx, lastIndex int64, lastRoot common.Hash) error { - siblings := [DefaultHeight]common.Hash{} - if lastIndex == -1 { - t.lastLeftCache = siblings - return nil - } - index := lastIndex - - currentNodeHash := lastRoot + t.lastIndex = int64(lastRoot.Index) + currentNodeHash := lastRoot.Hash + index := t.lastIndex // It starts in height-1 because 0 is the level of the leafs - for h := int(DefaultHeight - 1); h >= 0; h-- { + for h := int(types.DefaultHeight - 1); h >= 0; h-- { currentNode, err := t.getRHTNode(tx, currentNodeHash) if err != nil { return fmt.Errorf( "error getting node %s from the RHT at height %d with root %s: %w", - currentNodeHash.Hex(), h, lastRoot.Hex(), err, + currentNodeHash.Hex(), h, lastRoot.Hash.Hex(), err, ) } if currentNode == nil { return ErrNotFound } - siblings[h] = currentNode.left + siblings[h] = currentNode.Left if index&(1< 0 { - currentNodeHash = currentNode.right + currentNodeHash = currentNode.Right } else { - currentNodeHash = currentNode.left + currentNodeHash = currentNode.Left } } @@ -193,42 +120,3 @@ func (t *AppendOnlyTree) initLastLeftCache(tx kv.Tx, lastIndex int64, lastRoot c t.lastLeftCache = siblings return nil } - -// Reorg deletes all the data relevant from firstReorgedIndex (includded) and onwards -// and prepares the tree tfor being used as it was at firstReorgedIndex-1 -// It returns a function that must be called to rollback the changes done by this interaction -func (t *AppendOnlyTree) Reorg(tx kv.RwTx, firstReorgedIndex uint32) (func(), error) { - if t.lastIndex == -1 { - return func() {}, nil - } - // Clean root table - for i := firstReorgedIndex; i <= uint32(t.lastIndex); i++ { - if err := tx.Delete(t.rootTable, dbCommon.Uint64ToBytes(uint64(i))); err != nil { - return func() {}, err - } - } - - // Reset - root := common.Hash{} - if firstReorgedIndex > 0 { - rootBytes, err := tx.GetOne(t.rootTable, dbCommon.Uint64ToBytes(uint64(firstReorgedIndex)-1)) - if err != nil { - return func() {}, err - } - if rootBytes == nil { - return func() {}, ErrNotFound - } - root = common.Hash(rootBytes) - } - err := t.initLastLeftCache(tx, int64(firstReorgedIndex)-1, root) - if err != nil { - return func() {}, err - } - - // Note: not cleaning RHT, not worth it - backupLastIndex := t.lastIndex - t.lastIndex = int64(firstReorgedIndex) - 1 - return func() { - t.lastIndex = backupLastIndex - }, nil -} diff --git a/tree/migrations/migrations.go b/tree/migrations/migrations.go new file mode 100644 index 00000000..dd5847e7 --- /dev/null +++ b/tree/migrations/migrations.go @@ -0,0 +1,22 @@ +package migrations + +import ( + _ "embed" + + "github.com/0xPolygon/cdk/db" + "github.com/0xPolygon/cdk/db/types" +) + +//go:embed tree0001.sql +var mig001 string + +var Migrations = []types.Migration{ + { + ID: "tree001", + SQL: mig001, + }, +} + +func RunMigrations(dbPath string) error { + return db.RunMigrations(dbPath, Migrations) +} diff --git a/tree/migrations/tree0001.sql b/tree/migrations/tree0001.sql new file mode 100644 index 00000000..f70d048e --- /dev/null +++ b/tree/migrations/tree0001.sql @@ -0,0 +1,17 @@ +-- +migrate Down +DROP TABLE IF EXISTS /*dbprefix*/root; +DROP TABLE IF EXISTS /*dbprefix*/rht; + +-- +migrate Up +CREATE TABLE /*dbprefix*/root ( + hash VARCHAR PRIMARY KEY, + position INTEGER NOT NULL, + block_num BIGINT NOT NULL, + block_position BIGINT NOT NULL +); + +CREATE TABLE /*dbprefix*/rht ( + hash VARCHAR PRIMARY KEY, + left VARCHAR NOT NULL, + right VARCHAR NOT NULL +); diff --git a/tree/tree.go b/tree/tree.go index 5756df10..2107ba68 100644 --- a/tree/tree.go +++ b/tree/tree.go @@ -2,125 +2,62 @@ package tree import ( "context" + "database/sql" "errors" "fmt" - "math" - dbCommon "github.com/0xPolygon/cdk/common" + "github.com/0xPolygon/cdk/db" + "github.com/0xPolygon/cdk/tree/types" "github.com/ethereum/go-ethereum/common" - "github.com/ledgerwatch/erigon-lib/kv" + "github.com/russross/meddler" "golang.org/x/crypto/sha3" ) -const ( - DefaultHeight uint8 = 32 - rootTableSufix = "-root" - rhtTableSufix = "-rht" - indexTableSufix = "-index" -) - var ( - EmptyProof = [32]common.Hash{} + EmptyProof = types.Proof{} ErrNotFound = errors.New("not found") ) -type Leaf struct { - Index uint32 - Hash common.Hash -} - type Tree struct { - db kv.RwDB + db *sql.DB + zeroHashes []common.Hash rhtTable string rootTable string - indexTable string - zeroHashes []common.Hash -} - -type treeNode struct { - left common.Hash - right common.Hash } -func (n *treeNode) hash() common.Hash { +func newTreeNode(left, right common.Hash) types.TreeNode { var hash common.Hash hasher := sha3.NewLegacyKeccak256() - hasher.Write(n.left[:]) - hasher.Write(n.right[:]) + hasher.Write(left[:]) + hasher.Write(right[:]) copy(hash[:], hasher.Sum(nil)) - return hash -} - -func (n *treeNode) MarshalBinary() ([]byte, error) { - return append(n.left[:], n.right[:]...), nil -} - -func (n *treeNode) UnmarshalBinary(data []byte) error { - const nodeDataLength = 64 - if len(data) != nodeDataLength { - return fmt.Errorf("expected len %d, actual len %d", nodeDataLength, len(data)) + return types.TreeNode{ + Hash: hash, + Left: left, + Right: right, } - n.left = common.Hash(data[:32]) - n.right = common.Hash(data[32:]) - return nil -} - -// AddTables add the needed tables for the tree to work in a tableCfg -func AddTables(tableCfg map[string]kv.TableCfgItem, dbPrefix string) { - rootTable := dbPrefix + rootTableSufix - rhtTable := dbPrefix + rhtTableSufix - indexTable := dbPrefix + indexTableSufix - tableCfg[rootTable] = kv.TableCfgItem{} - tableCfg[rhtTable] = kv.TableCfgItem{} - tableCfg[indexTable] = kv.TableCfgItem{} } -func newTree(db kv.RwDB, dbPrefix string) *Tree { - rootTable := dbPrefix + rootTableSufix - rhtTable := dbPrefix + rhtTableSufix - indexTable := dbPrefix + indexTableSufix +func newTree(db *sql.DB, tablePrefix string) *Tree { t := &Tree{ - rhtTable: rhtTable, - rootTable: rootTable, - indexTable: indexTable, db: db, - zeroHashes: generateZeroHashes(DefaultHeight), + zeroHashes: generateZeroHashes(types.DefaultHeight), + rhtTable: tablePrefix + "rht", + rootTable: tablePrefix + "root", } return t } -func (t *Tree) getRootByIndex(tx kv.Tx, index uint64) (common.Hash, error) { - rootBytes, err := tx.GetOne(t.rootTable, dbCommon.Uint64ToBytes(index)) - if err != nil { - return common.Hash{}, err - } - if rootBytes == nil { - return common.Hash{}, ErrNotFound - } - return common.BytesToHash(rootBytes), nil -} - -func (t *Tree) getIndexByRoot(tx kv.Tx, root common.Hash) (uint64, error) { - indexBytes, err := tx.GetOne(t.indexTable, root[:]) - if err != nil { - return 0, err - } - if indexBytes == nil { - return 0, ErrNotFound - } - return dbCommon.BytesToUint64(indexBytes), nil -} - -func (t *Tree) getSiblings(tx kv.Tx, index uint32, root common.Hash) ( +func (t *Tree) getSiblings(tx db.Querier, index uint32, root common.Hash) ( siblings [32]common.Hash, hasUsedZeroHashes bool, err error, ) { currentNodeHash := root // It starts in height-1 because 0 is the level of the leafs - for h := int(DefaultHeight - 1); h >= 0; h-- { - var currentNode *treeNode + for h := int(types.DefaultHeight - 1); h >= 0; h-- { + var currentNode *types.TreeNode currentNode, err = t.getRHTNode(tx, currentNodeHash) if err != nil { if errors.Is(err, ErrNotFound) { @@ -158,11 +95,11 @@ func (t *Tree) getSiblings(tx kv.Tx, index uint32, root common.Hash) ( * Now, let's do AND operation => 100&100=100 which is higher than 0 so we need the left sibling (O5) */ if index&(1< 0 { - siblings[h] = currentNode.left - currentNodeHash = currentNode.right + siblings[h] = currentNode.Left + currentNodeHash = currentNode.Right } else { - siblings[h] = currentNode.right - currentNodeHash = currentNode.left + siblings[h] = currentNode.Right + currentNodeHash = currentNode.Left } } @@ -170,32 +107,30 @@ func (t *Tree) getSiblings(tx kv.Tx, index uint32, root common.Hash) ( } // GetProof returns the merkle proof for a given index and root. -func (t *Tree) GetProof(ctx context.Context, index uint32, root common.Hash) ([DefaultHeight]common.Hash, error) { - tx, err := t.db.BeginRw(ctx) +func (t *Tree) GetProof(ctx context.Context, index uint32, root common.Hash) (types.Proof, error) { + siblings, isErrNotFound, err := t.getSiblings(t.db, index, root) if err != nil { - return [DefaultHeight]common.Hash{}, err - } - defer tx.Rollback() - siblings, isErrNotFound, err := t.getSiblings(tx, index, root) - if err != nil { - return [DefaultHeight]common.Hash{}, err + return types.Proof{}, err } if isErrNotFound { - return [DefaultHeight]common.Hash{}, ErrNotFound + return types.Proof{}, ErrNotFound } return siblings, nil } -func (t *Tree) getRHTNode(tx kv.Tx, nodeHash common.Hash) (*treeNode, error) { - nodeBytes, err := tx.GetOne(t.rhtTable, nodeHash[:]) +func (t *Tree) getRHTNode(tx db.Querier, nodeHash common.Hash) (*types.TreeNode, error) { + node := &types.TreeNode{} + err := meddler.QueryRow( + tx, node, + fmt.Sprintf(`select * from %s where hash = $1`, t.rhtTable), + nodeHash.Hex(), + ) if err != nil { - return nil, err - } - if nodeBytes == nil { - return nil, ErrNotFound + if errors.Is(err, sql.ErrNoRows) { + return node, ErrNotFound + } + return node, err } - node := &treeNode{} - err = node.UnmarshalBinary(nodeBytes) return node, err } @@ -217,86 +152,101 @@ func generateZeroHashes(height uint8) []common.Hash { return zeroHashes } -func (t *Tree) storeNodes(tx kv.RwTx, nodes []treeNode) error { - for _, node := range nodes { - value, err := node.MarshalBinary() - if err != nil { - return err - } - if err := tx.Put(t.rhtTable, node.hash().Bytes(), value); err != nil { +func (t *Tree) storeNodes(tx db.Txer, nodes []types.TreeNode) error { + for i := 0; i < len(nodes); i++ { + if err := meddler.Insert(tx, t.rhtTable, &nodes[i]); err != nil { + if sqliteErr, ok := db.SQLiteErr(err); ok { + if sqliteErr.ExtendedCode == db.UniqueConstrain { + // ignore repeated entries. This is likely to happen due to not + // cleaning RHT when reorg + continue + } + } return err } } return nil } -func (t *Tree) storeRoot(tx kv.RwTx, rootIndex uint64, root common.Hash) error { - if err := tx.Put(t.rootTable, dbCommon.Uint64ToBytes(rootIndex), root[:]); err != nil { - return err - } - return tx.Put(t.indexTable, root[:], dbCommon.Uint64ToBytes(rootIndex)) +func (t *Tree) storeRoot(tx db.Txer, root types.Root) error { + return meddler.Insert(tx, t.rootTable, &root) } // GetLastRoot returns the last processed root -func (t *Tree) GetLastRoot(ctx context.Context) (common.Hash, error) { - tx, err := t.db.BeginRo(ctx) - if err != nil { - return common.Hash{}, err - } - defer tx.Rollback() +func (t *Tree) GetLastRoot(ctx context.Context) (types.Root, error) { + return t.getLastRootWithTx(t.db) +} - i, root, err := t.getLastIndexAndRootWithTx(tx) +func (t *Tree) getLastRootWithTx(tx db.Querier) (types.Root, error) { + var root types.Root + err := meddler.QueryRow( + tx, &root, + fmt.Sprintf(`SELECT * FROM %s ORDER BY block_num DESC, block_position DESC LIMIT 1;`, t.rootTable), + ) if err != nil { - return common.Hash{}, err - } - if i == -1 { - return common.Hash{}, ErrNotFound + if errors.Is(err, sql.ErrNoRows) { + return root, ErrNotFound + } + return root, err } return root, nil } -// getLastIndexAndRootWithTx return the index and the root associated to the last leaf inserted. -// If index == -1, it means no leaf added yet -func (t *Tree) getLastIndexAndRootWithTx(tx kv.Tx) (int64, common.Hash, error) { - iter, err := tx.RangeDescend( - t.rootTable, - dbCommon.Uint64ToBytes(math.MaxUint64), - dbCommon.Uint64ToBytes(0), - 1, - ) - if err != nil { - return 0, common.Hash{}, err +// GetRootByIndex returns the root associated to the index +func (t *Tree) GetRootByIndex(ctx context.Context, index uint32) (types.Root, error) { + var root types.Root + if err := meddler.QueryRow( + t.db, &root, + fmt.Sprintf(`SELECT * FROM %s WHERE position = $1;`, t.rootTable), + index, + ); err != nil { + if errors.Is(err, sql.ErrNoRows) { + return root, ErrNotFound + } + return root, err } + return root, nil +} - lastIndexBytes, rootBytes, err := iter.Next() - if err != nil { - return 0, common.Hash{}, err - } - if lastIndexBytes == nil { - return -1, common.Hash{}, nil +// GetRootByHash returns the root associated to the hash +func (t *Tree) GetRootByHash(ctx context.Context, hash common.Hash) (types.Root, error) { + var root types.Root + if err := meddler.QueryRow( + t.db, &root, + fmt.Sprintf(`SELECT * FROM %s WHERE hash = $1;`, t.rootTable), + hash.Hex(), + ); err != nil { + if errors.Is(err, sql.ErrNoRows) { + return root, ErrNotFound + } + return root, err } - return int64(dbCommon.BytesToUint64(lastIndexBytes)), common.Hash(rootBytes), nil + return root, nil } func (t *Tree) GetLeaf(ctx context.Context, index uint32, root common.Hash) (common.Hash, error) { - tx, err := t.db.BeginRo(ctx) - if err != nil { - return common.Hash{}, err - } - defer tx.Rollback() - currentNodeHash := root - for h := int(DefaultHeight - 1); h >= 0; h-- { - currentNode, err := t.getRHTNode(tx, currentNodeHash) + for h := int(types.DefaultHeight - 1); h >= 0; h-- { + currentNode, err := t.getRHTNode(t.db, currentNodeHash) if err != nil { return common.Hash{}, err } if index&(1< 0 { - currentNodeHash = currentNode.right + currentNodeHash = currentNode.Right } else { - currentNodeHash = currentNode.left + currentNodeHash = currentNode.Left } } return currentNodeHash, nil } + +// Reorg deletes all the data relevant from firstReorgedBlock (includded) and onwards +func (t *Tree) Reorg(tx db.Txer, firstReorgedBlock uint64) error { + _, err := tx.Exec( + fmt.Sprintf(`DELETE FROM %s WHERE block_num >= $1`, t.rootTable), + firstReorgedBlock, + ) + return err + // NOTE: rht is not cleaned, this could be done in the future as optimization +} diff --git a/tree/tree_test.go b/tree/tree_test.go index 3c27854f..b5278723 100644 --- a/tree/tree_test.go +++ b/tree/tree_test.go @@ -1,16 +1,20 @@ -package tree +package tree_test import ( "context" "encoding/json" "fmt" "os" + "path" "testing" + "github.com/0xPolygon/cdk/db" + "github.com/0xPolygon/cdk/log" + "github.com/0xPolygon/cdk/tree" + "github.com/0xPolygon/cdk/tree/migrations" "github.com/0xPolygon/cdk/tree/testvectors" + "github.com/0xPolygon/cdk/tree/types" "github.com/ethereum/go-ethereum/common" - "github.com/ledgerwatch/erigon-lib/kv" - "github.com/ledgerwatch/erigon-lib/kv/mdbx" "github.com/stretchr/testify/require" ) @@ -25,59 +29,46 @@ func TestMTAddLeaf(t *testing.T) { for ti, testVector := range mtTestVectors { t.Run(fmt.Sprintf("Test vector %d", ti), func(t *testing.T) { - path := t.TempDir() - dbPrefix := "foo" - tableCfgFunc := func(defaultBuckets kv.TableCfg) kv.TableCfg { - cfg := kv.TableCfg{} - AddTables(cfg, dbPrefix) - return cfg - } - db, err := mdbx.NewMDBX(nil). - Path(path). - WithTableCfg(tableCfgFunc). - Open() + dbPath := path.Join(t.TempDir(), "file::memory:?cache=shared") + log.Debug("DB created at: ", dbPath) + err := migrations.RunMigrations(dbPath) + require.NoError(t, err) + treeDB, err := db.NewSQLiteDB(dbPath) require.NoError(t, err) - tree, err := NewAppendOnlyTree(context.Background(), db, dbPrefix) + _, err = treeDB.Exec(`select * from root`) require.NoError(t, err) + merkletree := tree.NewAppendOnlyTree(treeDB, "") // Add exisiting leaves - leaves := []Leaf{} + tx, err := db.NewTx(ctx, treeDB) + require.NoError(t, err) for i, leaf := range testVector.ExistingLeaves { - leaves = append(leaves, Leaf{ + err = merkletree.AddLeaf(tx, uint64(i), 0, types.Leaf{ Index: uint32(i), Hash: common.HexToHash(leaf), }) + require.NoError(t, err) } - tx, err := db.BeginRw(ctx) - require.NoError(t, err) - _, err = tree.AddLeaves(tx, leaves) - require.NoError(t, err) require.NoError(t, tx.Commit()) if len(testVector.ExistingLeaves) > 0 { - txRo, err := tree.db.BeginRo(ctx) - require.NoError(t, err) - _, actualRoot, err := tree.getLastIndexAndRootWithTx(txRo) - txRo.Rollback() + root, err := merkletree.GetLastRoot(ctx) require.NoError(t, err) - require.Equal(t, common.HexToHash(testVector.CurrentRoot), actualRoot) + require.Equal(t, common.HexToHash(testVector.CurrentRoot), root.Hash) } // Add new bridge - tx, err = db.BeginRw(ctx) + tx, err = db.NewTx(ctx, treeDB) require.NoError(t, err) - _, err = tree.AddLeaves(tx, []Leaf{{ + err = merkletree.AddLeaf(tx, uint64(len(testVector.ExistingLeaves)), 0, types.Leaf{ Index: uint32(len(testVector.ExistingLeaves)), Hash: common.HexToHash(testVector.NewLeaf.CurrentHash), - }}) + }) require.NoError(t, err) require.NoError(t, tx.Commit()) - txRo, err := tree.db.BeginRo(ctx) + root, err := merkletree.GetLastRoot(ctx) require.NoError(t, err) - _, actualRoot, err := tree.getLastIndexAndRootWithTx(txRo) - txRo.Rollback() - require.NoError(t, err) - require.Equal(t, common.HexToHash(testVector.NewRoot), actualRoot) + require.Equal(t, common.HexToHash(testVector.NewRoot), root.Hash) }) } } @@ -93,43 +84,30 @@ func TestMTGetProof(t *testing.T) { for ti, testVector := range mtTestVectors { t.Run(fmt.Sprintf("Test vector %d", ti), func(t *testing.T) { - path := t.TempDir() - dbPrefix := "foo" - tableCfgFunc := func(defaultBuckets kv.TableCfg) kv.TableCfg { - cfg := kv.TableCfg{} - AddTables(cfg, dbPrefix) - return cfg - } - db, err := mdbx.NewMDBX(nil). - Path(path). - WithTableCfg(tableCfgFunc). - Open() + dbPath := path.Join(t.TempDir(), "file::memory:?cache=shared") + err := migrations.RunMigrations(dbPath) require.NoError(t, err) - tree, err := NewAppendOnlyTree(context.Background(), db, dbPrefix) + treeDB, err := db.NewSQLiteDB(dbPath) require.NoError(t, err) + tre := tree.NewAppendOnlyTree(treeDB, "") - leaves := []Leaf{} + tx, err := db.NewTx(ctx, treeDB) + require.NoError(t, err) for li, leaf := range testVector.Deposits { - leaves = append(leaves, Leaf{ + err = tre.AddLeaf(tx, uint64(li), 0, types.Leaf{ Index: uint32(li), Hash: leaf.Hash(), }) + require.NoError(t, err) } - tx, err := db.BeginRw(ctx) - require.NoError(t, err) - _, err = tree.AddLeaves(tx, leaves) - require.NoError(t, err) require.NoError(t, tx.Commit()) - txRo, err := tree.db.BeginRo(ctx) - require.NoError(t, err) - _, actualRoot, err := tree.getLastIndexAndRootWithTx(txRo) + root, err := tre.GetLastRoot(ctx) require.NoError(t, err) - txRo.Rollback() expectedRoot := common.HexToHash(testVector.ExpectedRoot) - require.Equal(t, expectedRoot, actualRoot) + require.Equal(t, expectedRoot, root.Hash) - proof, err := tree.GetProof(ctx, testVector.Index, expectedRoot) + proof, err := tre.GetProof(ctx, testVector.Index, expectedRoot) require.NoError(t, err) for i, sibling := range testVector.MerkleProof { require.Equal(t, common.HexToHash(sibling), proof[i]) diff --git a/tree/types/types.go b/tree/types/types.go new file mode 100644 index 00000000..bb117342 --- /dev/null +++ b/tree/types/types.go @@ -0,0 +1,27 @@ +package types + +import "github.com/ethereum/go-ethereum/common" + +const ( + DefaultHeight uint8 = 32 +) + +type Leaf struct { + Index uint32 + Hash common.Hash +} + +type Root struct { + Hash common.Hash `meddler:"hash,hash"` + Index uint32 `meddler:"position"` + BlockNum uint64 `meddler:"block_num"` + BlockPosition uint64 `meddler:"block_position"` +} + +type TreeNode struct { + Hash common.Hash `meddler:"hash,hash"` + Left common.Hash `meddler:"left,hash"` + Right common.Hash `meddler:"right,hash"` +} + +type Proof [DefaultHeight]common.Hash diff --git a/tree/updatabletree.go b/tree/updatabletree.go index 53f45889..3ed8b881 100644 --- a/tree/updatabletree.go +++ b/tree/updatabletree.go @@ -1,140 +1,68 @@ package tree import ( - "context" - "math" + "database/sql" + "errors" - dbCommon "github.com/0xPolygon/cdk/common" + "github.com/0xPolygon/cdk/db" + "github.com/0xPolygon/cdk/tree/types" "github.com/ethereum/go-ethereum/common" - "github.com/ledgerwatch/erigon-lib/kv" ) // UpdatableTree is a tree that have updatable leaves, and doesn't need to have sequential inserts type UpdatableTree struct { *Tree - lastRoot common.Hash } // NewUpdatableTree returns an UpdatableTree -func NewUpdatableTree(ctx context.Context, db kv.RwDB, dbPrefix string) (*UpdatableTree, error) { - // TODO: Load last root +func NewUpdatableTree(db *sql.DB, dbPrefix string) *UpdatableTree { t := newTree(db, dbPrefix) - tx, err := t.db.BeginRw(ctx) - if err != nil { - return nil, err - } - defer tx.Rollback() - rootIndex, root, err := t.getLastIndexAndRootWithTx(tx) - if err != nil { - return nil, err - } - if rootIndex == -1 { - root = t.zeroHashes[DefaultHeight] - } ut := &UpdatableTree{ - Tree: t, - lastRoot: root, + Tree: t, } - return ut, nil + return ut } -// UpseartLeaves inserts or updates a list of leaves. The root index will be used to index the resulting -// root after performing all the operations. Root index must be greater than the last used root index, -// but doesn't need to be sequential. Great for relating block nums and roots :) -// It returns a function that must be called to rollback the changes done by this interaction -func (t *UpdatableTree) UpseartLeaves(tx kv.RwTx, leaves []Leaf, rootIndex uint64) (func(), error) { - if len(leaves) == 0 { - return func() {}, nil - } - rootBackup := t.lastRoot - rollback := func() { - t.lastRoot = rootBackup - } - - for _, l := range leaves { - if err := t.upsertLeaf(tx, l); err != nil { - return rollback, err +func (t *UpdatableTree) UpsertLeaf(tx db.Txer, blockNum, blockPosition uint64, leaf types.Leaf) error { + var rootHash common.Hash + root, err := t.getLastRootWithTx(tx) + if err != nil { + if errors.Is(err, ErrNotFound) { + rootHash = t.zeroHashes[types.DefaultHeight] + } else { + return err } + } else { + rootHash = root.Hash } - - if err := t.storeRoot(tx, rootIndex, t.lastRoot); err != nil { - return rollback, err - } - return rollback, nil -} - -func (t *UpdatableTree) upsertLeaf(tx kv.RwTx, leaf Leaf) error { - siblings, _, err := t.getSiblings(tx, leaf.Index, t.lastRoot) + siblings, _, err := t.getSiblings(tx, leaf.Index, rootHash) if err != nil { return err } currentChildHash := leaf.Hash - newNodes := []treeNode{} - for h := uint8(0); h < DefaultHeight; h++ { - var parent treeNode + newNodes := []types.TreeNode{} + for h := uint8(0); h < types.DefaultHeight; h++ { + var parent types.TreeNode if leaf.Index&(1< 0 { // Add child to the right - parent = treeNode{ - left: siblings[h], - right: currentChildHash, - } + parent = newTreeNode(siblings[h], currentChildHash) } else { // Add child to the left - parent = treeNode{ - left: currentChildHash, - right: siblings[h], - } + parent = newTreeNode(currentChildHash, siblings[h]) } - currentChildHash = parent.hash() + currentChildHash = parent.Hash newNodes = append(newNodes, parent) } - + if err := t.storeRoot(tx, types.Root{ + Hash: currentChildHash, + Index: leaf.Index, + BlockNum: blockNum, + BlockPosition: blockPosition, + }); err != nil { + return err + } if err := t.storeNodes(tx, newNodes); err != nil { return err } - t.lastRoot = currentChildHash return nil } - -// Reorg deletes all the data relevant from firstReorgedIndex (includded) and onwards -// and prepares the tree tfor being used as it was at firstReorgedIndex-1. -// It returns a function that must be called to rollback the changes done by this interaction -func (t *UpdatableTree) Reorg(tx kv.RwTx, firstReorgedIndex uint64) (func(), error) { - iter, err := tx.RangeDescend( - t.rootTable, - dbCommon.Uint64ToBytes(math.MaxUint64), - dbCommon.Uint64ToBytes(0), - 0, - ) - if err != nil { - return func() {}, err - } - rootBackup := t.lastRoot - rollback := func() { - t.lastRoot = rootBackup - } - - for lastIndexBytes, rootBytes, err := iter.Next(); lastIndexBytes != nil; lastIndexBytes, rootBytes, err = iter.Next() { //nolint:lll - if err != nil { - return rollback, err - } - - if dbCommon.BytesToUint64(lastIndexBytes) >= firstReorgedIndex { - if err := tx.Delete(t.rootTable, lastIndexBytes); err != nil { - return rollback, err - } - } else { - t.lastRoot = common.Hash(rootBytes) - return rollback, nil - } - } - - // no root found after reorg, going back to empty tree - t.lastRoot = t.zeroHashes[DefaultHeight] - return rollback, nil -} - -// GetRootByRootIndex returns the root of the tree as it was right after adding the leaf with index -func (t *UpdatableTree) GetRootByRootIndex(tx kv.Tx, rootIndex uint64) (common.Hash, error) { - return t.getRootByIndex(tx, rootIndex) -}