From acd372e709ae63fb511654ffb5c82a9819584446 Mon Sep 17 00:00:00 2001 From: Arnau Date: Fri, 2 Aug 2024 11:54:16 +0200 Subject: [PATCH] pass E2E test --- common/common.go | 8 +- l1infotreesync/e2e_test.go | 140 +++++++++++++++++++++++++++++-- l1infotreesync/l1infotreesync.go | 8 ++ l1infotreesync/processor.go | 69 +++++++-------- tree/appendonlytree.go | 34 +++++--- tree/tree.go | 25 ++---- tree/tree_test.go | 6 +- tree/updatabletree.go | 10 +-- 8 files changed, 217 insertions(+), 83 deletions(-) diff --git a/common/common.go b/common/common.go index ebbafd69..f0f36575 100644 --- a/common/common.go +++ b/common/common.go @@ -12,26 +12,26 @@ import ( // Uint64ToBytes converts a uint64 to a byte slice func Uint64ToBytes(num uint64) []byte { bytes := make([]byte, 8) - binary.LittleEndian.PutUint64(bytes, num) + binary.BigEndian.PutUint64(bytes, num) return bytes } // BytesToUint64 converts a byte slice to a uint64 func BytesToUint64(bytes []byte) uint64 { - return binary.LittleEndian.Uint64(bytes) + return binary.BigEndian.Uint64(bytes) } // Uint32To2Bytes converts a uint32 to a byte slice func Uint32ToBytes(num uint32) []byte { key := make([]byte, 4) - binary.LittleEndian.PutUint32(key, num) + binary.BigEndian.PutUint32(key, num) return key } // BytesToUint32 converts a byte slice to a uint32 func BytesToUint32(bytes []byte) uint32 { - return binary.LittleEndian.Uint32(bytes) + return binary.BigEndian.Uint32(bytes) } func CalculateAccInputHash( diff --git a/l1infotreesync/e2e_test.go b/l1infotreesync/e2e_test.go index 416df057..e73576e7 100644 --- a/l1infotreesync/e2e_test.go +++ b/l1infotreesync/e2e_test.go @@ -1,4 +1,4 @@ -package l1infotreesync +package l1infotreesync_test import ( "context" @@ -11,6 +11,7 @@ import ( "github.com/0xPolygon/cdk-contracts-tooling/contracts/manual/globalexitrootnopush0" "github.com/0xPolygon/cdk/etherman" + "github.com/0xPolygon/cdk/l1infotreesync" "github.com/0xPolygon/cdk/reorgdetector" "github.com/0xPolygon/cdk/test/contracts/verifybatchesmock" "github.com/ethereum/go-ethereum/accounts/abi/bind" @@ -71,12 +72,12 @@ func TestE2E(t *testing.T) { require.NoError(t, err) auth, err := bind.NewKeyedTransactorWithChainID(privateKey, big.NewInt(1337)) require.NoError(t, err) - rdm := NewReorgDetectorMock(t) - rdm.On("Subscribe", reorgDetectorID).Return(&reorgdetector.Subscription{}, nil) + rdm := l1infotreesync.NewReorgDetectorMock(t) + rdm.On("Subscribe", mock.Anything).Return(&reorgdetector.Subscription{}, nil) rdm.On("AddBlockToTrack", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil) client, gerAddr, verifyAddr, gerSc, verifySC, err := newSimulatedClient(auth) require.NoError(t, err) - syncer, err := New(ctx, dbPath, gerAddr, verifyAddr, 10, etherman.LatestBlock, rdm, client.Client(), time.Millisecond, 0, 100*time.Millisecond, 3) + syncer, err := l1infotreesync.New(ctx, dbPath, gerAddr, verifyAddr, 10, etherman.LatestBlock, rdm, client.Client(), time.Millisecond, 0, 100*time.Millisecond, 3) require.NoError(t, err) go syncer.Start(ctx) @@ -105,19 +106,19 @@ func TestE2E(t *testing.T) { require.Equal(t, common.Hash(expectedRoot), actualRoot) } - // Update 10 rollups 10 times + // Update 10 rollups (verify batches event) 10 times for rollupID := uint32(1); rollupID < 10; rollupID++ { for i := 0; i < 10; i++ { newLocalExitRoot := common.HexToHash(strconv.Itoa(int(rollupID)) + "ffff" + strconv.Itoa(i)) - tx, err := verifySC.VerifyBatches(auth, rollupID, 0, newLocalExitRoot, common.Hash{}, true) + tx, err := verifySC.VerifyBatches(auth, rollupID, 0, newLocalExitRoot, common.Hash{}, i%2 != 0) require.NoError(t, err) client.Commit() // Let the processor catch up - time.Sleep(time.Millisecond * 100) + time.Sleep(time.Millisecond * 10) receipt, err := client.Client().TransactionReceipt(ctx, tx.Hash()) require.NoError(t, err) require.Equal(t, receipt.Status, types.ReceiptStatusSuccessful) - require.True(t, len(receipt.Logs) == 2) + require.True(t, len(receipt.Logs) == 1+i%2) expectedRollupExitRoot, err := verifySC.GetRollupExitRoot(&bind.CallOpts{Pending: false}) require.NoError(t, err) @@ -127,3 +128,126 @@ func TestE2E(t *testing.T) { } } } + +func TestFinalised(t *testing.T) { + ctx := context.Background() + privateKey, err := crypto.GenerateKey() + require.NoError(t, err) + auth, err := bind.NewKeyedTransactorWithChainID(privateKey, big.NewInt(1337)) + require.NoError(t, err) + client, _, _, _, _, err := newSimulatedClient(auth) + require.NoError(t, err) + for i := 0; i < 100; i++ { + client.Commit() + } + + n4, err := client.Client().HeaderByNumber(ctx, big.NewInt(-4)) + require.NoError(t, err) + fmt.Println("-4", n4.Number) + n3, err := client.Client().HeaderByNumber(ctx, big.NewInt(-3)) + require.NoError(t, err) + fmt.Println("-3", n3.Number) + n2, err := client.Client().HeaderByNumber(ctx, big.NewInt(-2)) + require.NoError(t, err) + fmt.Println("-2", n2.Number) + n1, err := client.Client().HeaderByNumber(ctx, big.NewInt(-1)) + require.NoError(t, err) + fmt.Println("-1", n1.Number) + n0, err := client.Client().HeaderByNumber(ctx, nil) + require.NoError(t, err) + fmt.Println("0", n0.Number) + fmt.Printf("amount of blocks latest - finalised: %d", n0.Number.Uint64()-n3.Number.Uint64()) +} + +func TestStressAndReorgs(t *testing.T) { + const ( + totalIterations = 10_124 + enableReorgs = false // test fails when set to true + reorgEveryXIterations = 53 + maxReorgDepth = 5 + maxEventsPerBlock = 7 + maxRollups = 31 + ) + + ctx := context.Background() + dbPathSyncer := t.TempDir() + dbPathReorg := t.TempDir() + privateKey, err := crypto.GenerateKey() + require.NoError(t, err) + auth, err := bind.NewKeyedTransactorWithChainID(privateKey, big.NewInt(1337)) + require.NoError(t, err) + client, gerAddr, verifyAddr, gerSc, verifySC, err := newSimulatedClient(auth) + require.NoError(t, err) + rd, err := reorgdetector.New(ctx, client.Client(), dbPathReorg) + go rd.Start(ctx) + syncer, err := l1infotreesync.New(ctx, dbPathSyncer, gerAddr, verifyAddr, 10, etherman.LatestBlock, rd, client.Client(), time.Millisecond, 0, 100*time.Millisecond, 3) + require.NoError(t, err) + go syncer.Start(ctx) + + for i := 0; i < totalIterations; i++ { + for j := 0; j < i%maxEventsPerBlock; j++ { + switch j % 3 { + case 0: // Update L1 Info Tree + _, err := gerSc.UpdateExitRoot(auth, common.HexToHash(strconv.Itoa(i))) + require.NoError(t, err) + case 1: // Update L1 Info Tree + Rollup Exit Tree + newLocalExitRoot := common.HexToHash(strconv.Itoa(i) + "ffff" + strconv.Itoa(j)) + _, err := verifySC.VerifyBatches(auth, 1+uint32(i%maxRollups), 0, newLocalExitRoot, common.Hash{}, true) + require.NoError(t, err) + case 2: // Update Rollup Exit Tree + newLocalExitRoot := common.HexToHash(strconv.Itoa(i) + "ffff" + strconv.Itoa(j)) + _, err := verifySC.VerifyBatches(auth, 1+uint32(i%maxRollups), 0, newLocalExitRoot, common.Hash{}, false) + require.NoError(t, err) + } + } + client.Commit() + time.Sleep(time.Microsecond * 10) // Sleep just enough for goroutine to switch + if enableReorgs && i%reorgEveryXIterations == 0 { + reorgDepth := i%maxReorgDepth + 1 + currentBlockNum, err := client.Client().BlockNumber(ctx) + require.NoError(t, err) + targetReorgBlockNum := currentBlockNum - uint64(reorgDepth) + if targetReorgBlockNum < currentBlockNum { // we are dealing with uints... + reorgBlock, err := client.Client().BlockByNumber(ctx, big.NewInt(int64(targetReorgBlockNum))) + require.NoError(t, err) + client.Fork(reorgBlock.Hash()) + } + } + } + + syncerUpToDate := false + var errMsg string + for i := 0; i < 50; i++ { + lpb, err := syncer.GetLastProcessedBlock(ctx) + require.NoError(t, err) + lb, err := client.Client().BlockNumber(ctx) + require.NoError(t, err) + if lpb == lb { + syncerUpToDate = true + break + } + time.Sleep(time.Millisecond * 100) + errMsg = fmt.Sprintf("last block from client: %d, last block from syncer: %d", lb, lpb) + } + require.True(t, syncerUpToDate, errMsg) + + // Assert rollup exit root + expectedRollupExitRoot, err := verifySC.GetRollupExitRoot(&bind.CallOpts{Pending: false}) + require.NoError(t, err) + actualRollupExitRoot, err := syncer.GetLastRollupExitRoot(ctx) + require.NoError(t, err) + require.Equal(t, common.Hash(expectedRollupExitRoot), actualRollupExitRoot) + + // Assert L1 Info tree root + expectedL1InfoRoot, err := gerSc.GetRoot(&bind.CallOpts{Pending: false}) + require.NoError(t, err) + expectedGER, err := gerSc.GetLastGlobalExitRoot(&bind.CallOpts{Pending: false}) + require.NoError(t, err) + index, actualL1InfoRoot, err := syncer.GetLastL1InfoTreeRootAndIndex(ctx) + require.NoError(t, err) + info, err := syncer.GetInfoByIndex(ctx, index) + require.NoError(t, err, fmt.Sprintf("index: %d", index)) + + require.Equal(t, common.Hash(expectedL1InfoRoot), actualL1InfoRoot) + require.Equal(t, common.Hash(expectedGER), info.GlobalExitRoot, fmt.Sprintf("%+v", info)) +} diff --git a/l1infotreesync/l1infotreesync.go b/l1infotreesync/l1infotreesync.go index 2a288704..1dea0bef 100644 --- a/l1infotreesync/l1infotreesync.go +++ b/l1infotreesync/l1infotreesync.go @@ -125,3 +125,11 @@ func (s *L1InfoTreeSync) GetL1InfoTreeRootByIndex(ctx context.Context, index uin func (s *L1InfoTreeSync) GetLastRollupExitRoot(ctx context.Context) (common.Hash, error) { return s.processor.rollupExitTree.GetLastRoot(ctx) } + +func (s *L1InfoTreeSync) GetLastL1InfoTreeRootAndIndex(ctx context.Context) (uint32, common.Hash, error) { + return s.processor.l1InfoTree.GetLastIndexAndRoot(ctx) +} + +func (s *L1InfoTreeSync) GetLastProcessedBlock(ctx context.Context) (uint64, error) { + return s.processor.GetLastProcessedBlock(ctx) +} diff --git a/l1infotreesync/processor.go b/l1infotreesync/processor.go index 67ffc26c..21f872ce 100644 --- a/l1infotreesync/processor.go +++ b/l1infotreesync/processor.go @@ -339,24 +339,25 @@ func (p *processor) ProcessBlock(ctx context.Context, b sync.Block) error { l1InfoTreeLeavesToAdd := []tree.Leaf{} rollupExitTreeLeavesToAdd := []tree.Leaf{} if len(b.Events) > 0 { - var initialIndex uint32 + var initialL1InfoIndex uint32 + var l1InfoLeavesAdded uint32 lastIndex, err := p.getLastIndex(tx) if err == ErrNotFound { - initialIndex = 0 + initialL1InfoIndex = 0 } else if err != nil { rollback() return err } else { - initialIndex = lastIndex + 1 + initialL1InfoIndex = lastIndex + 1 } - var nextExpectedRollupExitTreeRoot *ethCommon.Hash - for i, e := range b.Events { + for _, e := range b.Events { event := e.(Event) events = append(events, event) if event.UpdateL1InfoTree != nil { + index := initialL1InfoIndex + l1InfoLeavesAdded leafToStore := storeLeaf{ BlockNumber: b.Num, - Index: initialIndex + uint32(i), + Index: index, MainnetExitRoot: event.UpdateL1InfoTree.MainnetExitRoot, RollupExitRoot: event.UpdateL1InfoTree.RollupExitRoot, ParentHash: event.UpdateL1InfoTree.ParentHash, @@ -367,10 +368,10 @@ func (p *processor) ProcessBlock(ctx context.Context, b sync.Block) error { return err } l1InfoTreeLeavesToAdd = append(l1InfoTreeLeavesToAdd, tree.Leaf{ - Index: initialIndex + uint32(i), + Index: leafToStore.Index, Hash: leafToStore.Hash(), }) - nextExpectedRollupExitTreeRoot = &leafToStore.RollupExitRoot + l1InfoLeavesAdded++ } if event.VerifyBatches != nil { @@ -379,36 +380,38 @@ func (p *processor) ProcessBlock(ctx context.Context, b sync.Block) error { // that the computation of the tree is correct. However, there are some execution paths // on the contract that don't follow this (verifyBatches + pendingStateTimeout != 0) rollupExitTreeLeavesToAdd = append(rollupExitTreeLeavesToAdd, tree.Leaf{ - Index: event.VerifyBatches.RollupID, - Hash: event.VerifyBatches.ExitRoot, - ExpectedRoot: nextExpectedRollupExitTreeRoot, + Index: event.VerifyBatches.RollupID - 1, + Hash: event.VerifyBatches.ExitRoot, }) - nextExpectedRollupExitTreeRoot = nil } } - bwl := blockWithLeafs{ - FirstIndex: initialIndex, - LastIndex: initialIndex + uint32(len(b.Events)), - } - blockValue, err := json.Marshal(bwl) - if err != nil { - rollback() - return err - } - if err := tx.Put(blockTable, common.Uint64ToBytes(b.Num), blockValue); err != nil { - rollback() - return err - } - l1InfoTreeRollback, err = p.l1InfoTree.AddLeaves(tx, l1InfoTreeLeavesToAdd) - if err != nil { - rollback() - return err + if l1InfoLeavesAdded > 0 { + bwl := blockWithLeafs{ + FirstIndex: initialL1InfoIndex, + LastIndex: initialL1InfoIndex + l1InfoLeavesAdded, + } + blockValue, err := json.Marshal(bwl) + if err != nil { + rollback() + return err + } + if err := tx.Put(blockTable, common.Uint64ToBytes(b.Num), blockValue); err != nil { + rollback() + return err + } + l1InfoTreeRollback, err = p.l1InfoTree.AddLeaves(tx, l1InfoTreeLeavesToAdd) + if err != nil { + rollback() + return err + } } - rollupExitTreeRollback, err = p.rollupExitTree.UpseartLeaves(tx, rollupExitTreeLeavesToAdd, b.Num) - if err != nil { - rollback() - return err + if len(rollupExitTreeLeavesToAdd) > 0 { + rollupExitTreeRollback, err = p.rollupExitTree.UpseartLeaves(tx, rollupExitTreeLeavesToAdd, b.Num) + if err != nil { + rollback() + return err + } } } if err := p.updateLastProcessedBlock(tx, b.Num); err != nil { diff --git a/tree/appendonlytree.go b/tree/appendonlytree.go index 0d23a735..db78bb80 100644 --- a/tree/appendonlytree.go +++ b/tree/appendonlytree.go @@ -30,12 +30,7 @@ func (t *AppendOnlyTree) AddLeaves(tx kv.RwTx, leaves []Leaf) (func(), error) { if len(leaves) == 0 { return func() {}, nil } - if int64(leaves[0].Index) != t.lastIndex+1 { - return func() {}, fmt.Errorf( - "mismatched index. Expected: %d, actual: %d", - t.lastIndex+1, leaves[0].Index, - ) - } + backupIndx := t.lastIndex backupCache := make([]common.Hash, len(t.lastLeftCache)) copy(backupCache, t.lastLeftCache) @@ -54,6 +49,12 @@ func (t *AppendOnlyTree) AddLeaves(tx kv.RwTx, leaves []Leaf) (func(), error) { } func (t *AppendOnlyTree) addLeaf(tx kv.RwTx, leaf Leaf) error { + if int64(leaf.Index) != t.lastIndex+1 { + return fmt.Errorf( + "mismatched index. Expected: %d, actual: %d", + t.lastIndex+1, leaf.Index, + ) + } // Calculate new tree nodes currentChildHash := leaf.Hash newNodes := []treeNode{} @@ -80,10 +81,6 @@ func (t *AppendOnlyTree) addLeaf(tx kv.RwTx, leaf Leaf) error { newNodes = append(newNodes, parent) } - if err := assertRoot(leaf.ExpectedRoot, currentChildHash); err != nil { - return err - } - // store root t.storeRoot(tx, uint64(leaf.Index), currentChildHash) root := currentChildHash @@ -102,6 +99,21 @@ func (t *AppendOnlyTree) GetRootByIndex(tx kv.Tx, index uint32) (common.Hash, er return t.getRootByIndex(tx, uint64(index)) } +func (t *AppendOnlyTree) GetLastIndexAndRoot(ctx context.Context) (uint32, common.Hash, error) { + tx, err := t.db.BeginRo(ctx) + if err != nil { + return 0, common.Hash{}, err + } + i, root, err := t.getLastIndexAndRootWithTx(tx) + if err != nil { + return 0, common.Hash{}, err + } + if i == -1 { + return 0, common.Hash{}, ErrNotFound + } + return uint32(i), root, nil +} + func (t *AppendOnlyTree) initLastLeftCacheAndLastDepositCount(ctx context.Context) error { tx, err := t.db.BeginRw(ctx) if err != nil { @@ -117,7 +129,7 @@ func (t *AppendOnlyTree) initLastLeftCacheAndLastDepositCount(ctx context.Contex } func (t *AppendOnlyTree) initLastIndex(tx kv.Tx) (common.Hash, error) { - ldc, root, err := t.getLastIndexAndRoot(tx) + ldc, root, err := t.getLastIndexAndRootWithTx(tx) if err != nil { return common.Hash{}, err } diff --git a/tree/tree.go b/tree/tree.go index a93b8c28..8564db50 100644 --- a/tree/tree.go +++ b/tree/tree.go @@ -23,9 +23,8 @@ var ( ) type Leaf struct { - Index uint32 - Hash common.Hash - ExpectedRoot *common.Hash + Index uint32 + Hash common.Hash } type Tree struct { @@ -111,6 +110,7 @@ func (t *Tree) getSiblings(tx kv.Tx, index uint32, root common.Hash) ( if err == ErrNotFound { hasUsedZeroHashes = true siblings = append(siblings, t.zeroHashes[h]) + err = nil continue } else { err = fmt.Errorf( @@ -228,7 +228,7 @@ func (t *Tree) GetLastRoot(ctx context.Context) (common.Hash, error) { return common.Hash{}, err } - i, root, err := t.getLastIndexAndRoot(tx) + i, root, err := t.getLastIndexAndRootWithTx(tx) if err != nil { return common.Hash{}, err } @@ -238,9 +238,9 @@ func (t *Tree) GetLastRoot(ctx context.Context) (common.Hash, error) { return root, nil } -// getLastIndexAndRoot return the index and the root associated to the last leaf inserted. +// getLastIndexAndRootWithTx return the index and the root associated to the last leaf inserted. // If index == -1, it means no leaf added yet -func (t *Tree) getLastIndexAndRoot(tx kv.Tx) (int64, common.Hash, error) { +func (t *Tree) getLastIndexAndRootWithTx(tx kv.Tx) (int64, common.Hash, error) { iter, err := tx.RangeDescend( t.rootTable, dbCommon.Uint64ToBytes(math.MaxUint64), @@ -258,16 +258,5 @@ func (t *Tree) getLastIndexAndRoot(tx kv.Tx) (int64, common.Hash, error) { if lastIndexBytes == nil { return -1, common.Hash{}, nil } - return int64(dbCommon.BytesToUint32(lastIndexBytes)), common.Hash(rootBytes), nil -} - -func assertRoot(expected *common.Hash, actual common.Hash) error { - if expected != nil && *expected != actual { - return fmt.Errorf( - "root missmatch. Expected %s actual %s", - expected.Hex(), - actual.Hex(), - ) - } - return nil + return int64(dbCommon.BytesToUint64(lastIndexBytes)), common.Hash(rootBytes), nil } diff --git a/tree/tree_test.go b/tree/tree_test.go index cd1348dd..d826c5b8 100644 --- a/tree/tree_test.go +++ b/tree/tree_test.go @@ -56,7 +56,7 @@ func TestMTAddLeaf(t *testing.T) { if len(testVector.ExistingLeaves) > 0 { txRo, err := tree.db.BeginRo(ctx) require.NoError(t, err) - _, actualRoot, err := tree.getLastIndexAndRoot(txRo) + _, actualRoot, err := tree.getLastIndexAndRootWithTx(txRo) txRo.Rollback() require.NoError(t, err) require.Equal(t, common.HexToHash(testVector.CurrentRoot), actualRoot) @@ -74,7 +74,7 @@ func TestMTAddLeaf(t *testing.T) { txRo, err := tree.db.BeginRo(ctx) require.NoError(t, err) - _, actualRoot, err := tree.getLastIndexAndRoot(txRo) + _, actualRoot, err := tree.getLastIndexAndRootWithTx(txRo) txRo.Rollback() require.NoError(t, err) require.Equal(t, common.HexToHash(testVector.NewRoot), actualRoot) @@ -123,7 +123,7 @@ func TestMTGetProof(t *testing.T) { txRo, err := tree.db.BeginRo(ctx) require.NoError(t, err) - _, actualRoot, err := tree.getLastIndexAndRoot(txRo) + _, actualRoot, err := tree.getLastIndexAndRootWithTx(txRo) txRo.Rollback() expectedRoot := common.HexToHash(testVector.ExpectedRoot) require.Equal(t, expectedRoot, actualRoot) diff --git a/tree/updatabletree.go b/tree/updatabletree.go index 684350d0..0f77b259 100644 --- a/tree/updatabletree.go +++ b/tree/updatabletree.go @@ -22,7 +22,7 @@ func NewUpdatable(ctx context.Context, db kv.RwDB, dbPrefix string) (*UpdatableT return nil, err } defer tx.Rollback() - rootIndex, root, err := t.getLastIndexAndRoot(tx) + rootIndex, root, err := t.getLastIndexAndRootWithTx(tx) if err != nil { return nil, err } @@ -46,7 +46,9 @@ func (t *UpdatableTree) UpseartLeaves(tx kv.RwTx, leaves []Leaf, rootIndex uint6 } for _, l := range leaves { - t.upsertLeaf(tx, l) + if err := t.upsertLeaf(tx, l); err != nil { + return rollback, err + } } if err := t.storeRoot(tx, rootIndex, t.lastRoot); err != nil { @@ -81,10 +83,6 @@ func (t *UpdatableTree) upsertLeaf(tx kv.RwTx, leaf Leaf) error { newNodes = append(newNodes, parent) } - if err := assertRoot(leaf.ExpectedRoot, currentChildHash); err != nil { - return err - } - if err := t.storeNodes(tx, newNodes); err != nil { return err }