From 2c1e0e5bbbec868e711d3748d97eee46fb77ebbd Mon Sep 17 00:00:00 2001 From: Ganesh Vanahalli Date: Fri, 1 Nov 2024 20:40:27 +0530 Subject: [PATCH 01/14] Bulk sync missing blockMetadata --- arbnode/blockmetadata.go | 159 ++++++++++++++++++++++++++++++++ arbnode/node.go | 3 + arbnode/schema.go | 19 ++-- arbnode/transaction_streamer.go | 16 ++++ execution/gethexec/node.go | 3 + execution/interface.go | 2 + 6 files changed, 193 insertions(+), 9 deletions(-) create mode 100644 arbnode/blockmetadata.go diff --git a/arbnode/blockmetadata.go b/arbnode/blockmetadata.go new file mode 100644 index 0000000000..e2b0ad6b3c --- /dev/null +++ b/arbnode/blockmetadata.go @@ -0,0 +1,159 @@ +package arbnode + +import ( + "bytes" + "context" + "encoding/binary" + "time" + + "github.com/spf13/pflag" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/node" + "github.com/ethereum/go-ethereum/rpc" + "github.com/offchainlabs/nitro/arbutil" + "github.com/offchainlabs/nitro/execution" + "github.com/offchainlabs/nitro/execution/gethexec" + "github.com/offchainlabs/nitro/util/signature" + "github.com/offchainlabs/nitro/util/stopwaiter" +) + +type BlockMetadataRebuilderConfig struct { + Enable bool `koanf:"enable"` + Url string `koanf:"url"` + JWTSecret string `koanf:"jwt-secret"` + RebuildInterval time.Duration `koanf:"rebuild-interval"` + APIBlocksLimit int `koanf:"api-blocks-limit"` +} + +var DefaultBlockMetadataRebuilderConfig = BlockMetadataRebuilderConfig{ + Enable: false, + RebuildInterval: time.Minute * 5, + APIBlocksLimit: 100, +} + +func BlockMetadataRebuilderConfigAddOptions(prefix string, f *pflag.FlagSet) { + f.Bool(prefix+".enable", DefaultBlockMetadataRebuilderConfig.Enable, "enable syncing blockMetadata using a bulk metadata api") + f.String(prefix+".url", DefaultBlockMetadataRebuilderConfig.Url, "url for bulk blockMetadata api") + f.String(prefix+".jwt-secret", DefaultBlockMetadataRebuilderConfig.JWTSecret, "filepath of jwt secret") + f.Duration(prefix+".rebuild-interval", DefaultBlockMetadataRebuilderConfig.RebuildInterval, "interval at which blockMetadata is synced regularly") + f.Int(prefix+".api-blocks-limit", DefaultBlockMetadataRebuilderConfig.APIBlocksLimit, "maximum number of blocks allowed to be queried for blockMetadata per arb_getRawBlockMetadata query.\n"+ + "This should be set lesser than or equal to the value set on the api provider side") +} + +type BlockMetadataRebuilder struct { + stopwaiter.StopWaiter + config *BlockMetadataRebuilderConfig + db ethdb.Database + client *rpc.Client + exec execution.ExecutionClient +} + +func NewBlockMetadataRebuilder(ctx context.Context, c *BlockMetadataRebuilderConfig, db ethdb.Database, exec execution.ExecutionClient) (*BlockMetadataRebuilder, error) { + var err error + var jwt *common.Hash + if c.JWTSecret != "" { + jwt, err = signature.LoadSigningKey(c.JWTSecret) + if err != nil { + return nil, err + } + } + var client *rpc.Client + if jwt == nil { + client, err = rpc.DialOptions(ctx, c.Url) + } else { + client, err = rpc.DialOptions(ctx, c.Url, rpc.WithHTTPAuth(node.NewJWTAuth([32]byte(*jwt)))) + } + if err != nil { + return nil, err + } + return &BlockMetadataRebuilder{ + config: c, + db: db, + client: client, + exec: exec, + }, nil +} + +func (b *BlockMetadataRebuilder) Fetch(ctx context.Context, fromBlock, toBlock uint64) ([]gethexec.NumberAndBlockMetadata, error) { + var result []gethexec.NumberAndBlockMetadata + err := b.client.CallContext(ctx, &result, "arb_getRawBlockMetadata", rpc.BlockNumber(fromBlock), rpc.BlockNumber(toBlock)) + if err != nil { + return nil, err + } + return result, nil +} + +func ArrayToMap[T comparable](arr []T) map[T]struct{} { + ret := make(map[T]struct{}) + for _, elem := range arr { + ret[elem] = struct{}{} + } + return ret +} + +func (b *BlockMetadataRebuilder) PushBlockMetadataToDB(query []uint64, result []gethexec.NumberAndBlockMetadata) error { + batch := b.db.NewBatch() + queryMap := ArrayToMap(query) + for _, elem := range result { + pos, err := b.exec.BlockNumberToMessageIndex(elem.BlockNumber) + if err != nil { + return err + } + if _, ok := queryMap[uint64(pos)]; ok { + if err := batch.Put(dbKey(blockMetadataInputFeedPrefix, uint64(pos)), elem.RawMetadata); err != nil { + return err + } + if err := batch.Delete(dbKey(missingBlockMetadataInputFeedPrefix, uint64(pos))); err != nil { + return err + } + } + } + return batch.Write() +} + +func (b *BlockMetadataRebuilder) Update(ctx context.Context) time.Duration { + iter := b.db.NewIterator(missingBlockMetadataInputFeedPrefix, nil) + defer iter.Release() + var query []uint64 + for iter.Next() { + keyBytes := bytes.TrimPrefix(iter.Key(), missingBlockMetadataInputFeedPrefix) + query = append(query, binary.BigEndian.Uint64(keyBytes)) + end := len(query) - 1 + if query[end]-query[0] >= uint64(b.config.APIBlocksLimit) { + if query[end]-query[0] > uint64(b.config.APIBlocksLimit) { + if len(query) >= 2 { + end -= 1 + } else { + end = 0 + } + } + result, err := b.Fetch( + ctx, + b.exec.MessageIndexToBlockNumber(arbutil.MessageIndex(query[0])), + b.exec.MessageIndexToBlockNumber(arbutil.MessageIndex(query[end])), + ) + if err != nil { + log.Error("Error getting result from bulk blockMetadata API", "err", err) + return b.config.RebuildInterval // backoff + } + if err = b.PushBlockMetadataToDB(query[:end+1], result); err != nil { + log.Error("Error committing result from bulk blockMetadata API to ArbDB", "err", err) + return b.config.RebuildInterval // backoff + } + query = query[end+1:] + } + } + return b.config.RebuildInterval +} + +func (b *BlockMetadataRebuilder) Start(ctx context.Context) { + b.StopWaiter.Start(ctx, b) + b.CallIteratively(b.Update) +} + +func (b *BlockMetadataRebuilder) StopAndWait() { + b.StopWaiter.StopAndWait() +} diff --git a/arbnode/node.go b/arbnode/node.go index 705a48da08..0961a7ed43 100644 --- a/arbnode/node.go +++ b/arbnode/node.go @@ -129,6 +129,9 @@ func (c *Config) Validate() error { if err := c.Staker.Validate(); err != nil { return err } + if c.Sequencer && c.TransactionStreamer.TrackBlockMetadataFrom == 0 { + return errors.New("when sequencer is enabled track-missing-block-metadata should be enabled as well") + } return nil } diff --git a/arbnode/schema.go b/arbnode/schema.go index 486afb20ae..09554d6161 100644 --- a/arbnode/schema.go +++ b/arbnode/schema.go @@ -4,15 +4,16 @@ package arbnode var ( - messagePrefix []byte = []byte("m") // maps a message sequence number to a message - blockHashInputFeedPrefix []byte = []byte("b") // maps a message sequence number to a block hash received through the input feed - blockMetadataInputFeedPrefix []byte = []byte("t") // maps a message sequence number to a blockMetaData byte array received through the input feed - messageResultPrefix []byte = []byte("r") // maps a message sequence number to a message result - legacyDelayedMessagePrefix []byte = []byte("d") // maps a delayed sequence number to an accumulator and a message as serialized on L1 - rlpDelayedMessagePrefix []byte = []byte("e") // maps a delayed sequence number to an accumulator and an RLP encoded message - parentChainBlockNumberPrefix []byte = []byte("p") // maps a delayed sequence number to a parent chain block number - sequencerBatchMetaPrefix []byte = []byte("s") // maps a batch sequence number to BatchMetadata - delayedSequencedPrefix []byte = []byte("a") // maps a delayed message count to the first sequencer batch sequence number with this delayed count + messagePrefix []byte = []byte("m") // maps a message sequence number to a message + blockHashInputFeedPrefix []byte = []byte("b") // maps a message sequence number to a block hash received through the input feed + blockMetadataInputFeedPrefix []byte = []byte("t") // maps a message sequence number to a blockMetaData byte array received through the input feed + missingBlockMetadataInputFeedPrefix []byte = []byte("mt") // maps a message sequence number whose blockMetaData byte array is missing to nil + messageResultPrefix []byte = []byte("r") // maps a message sequence number to a message result + legacyDelayedMessagePrefix []byte = []byte("d") // maps a delayed sequence number to an accumulator and a message as serialized on L1 + rlpDelayedMessagePrefix []byte = []byte("e") // maps a delayed sequence number to an accumulator and an RLP encoded message + parentChainBlockNumberPrefix []byte = []byte("p") // maps a delayed sequence number to a parent chain block number + sequencerBatchMetaPrefix []byte = []byte("s") // maps a batch sequence number to BatchMetadata + delayedSequencedPrefix []byte = []byte("a") // maps a delayed message count to the first sequencer batch sequence number with this delayed count messageCountKey []byte = []byte("_messageCount") // contains the current message count delayedMessageCountKey []byte = []byte("_delayedMessageCount") // contains the current delayed message count diff --git a/arbnode/transaction_streamer.go b/arbnode/transaction_streamer.go index 1636e06bd3..f1747e879e 100644 --- a/arbnode/transaction_streamer.go +++ b/arbnode/transaction_streamer.go @@ -68,12 +68,15 @@ type TransactionStreamer struct { broadcastServer *broadcaster.Broadcaster inboxReader *InboxReader delayedBridge *DelayedBridge + + trackBlockMetadataFrom arbutil.MessageIndex } type TransactionStreamerConfig struct { MaxBroadcasterQueueSize int `koanf:"max-broadcaster-queue-size"` MaxReorgResequenceDepth int64 `koanf:"max-reorg-resequence-depth" reload:"hot"` ExecuteMessageLoopDelay time.Duration `koanf:"execute-message-loop-delay" reload:"hot"` + TrackBlockMetadataFrom uint64 `koanf:"track-block-metadata-from"` } type TransactionStreamerConfigFetcher func() *TransactionStreamerConfig @@ -82,18 +85,21 @@ var DefaultTransactionStreamerConfig = TransactionStreamerConfig{ MaxBroadcasterQueueSize: 50_000, MaxReorgResequenceDepth: 1024, ExecuteMessageLoopDelay: time.Millisecond * 100, + TrackBlockMetadataFrom: 0, } var TestTransactionStreamerConfig = TransactionStreamerConfig{ MaxBroadcasterQueueSize: 10_000, MaxReorgResequenceDepth: 128 * 1024, ExecuteMessageLoopDelay: time.Millisecond, + TrackBlockMetadataFrom: 0, } func TransactionStreamerConfigAddOptions(prefix string, f *flag.FlagSet) { f.Int(prefix+".max-broadcaster-queue-size", DefaultTransactionStreamerConfig.MaxBroadcasterQueueSize, "maximum cache of pending broadcaster messages") f.Int64(prefix+".max-reorg-resequence-depth", DefaultTransactionStreamerConfig.MaxReorgResequenceDepth, "maximum number of messages to attempt to resequence on reorg (0 = never resequence, -1 = always resequence)") f.Duration(prefix+".execute-message-loop-delay", DefaultTransactionStreamerConfig.ExecuteMessageLoopDelay, "delay when polling calls to execute messages") + f.Uint64(prefix+".track-block-metadata-from", DefaultTransactionStreamerConfig.TrackBlockMetadataFrom, "block number starting from which the missing of blockmetadata is being tracked in the local disk. Disabled by default") } func NewTransactionStreamer( @@ -119,6 +125,13 @@ func NewTransactionStreamer( if err != nil { return nil, err } + if config().TrackBlockMetadataFrom != 0 { + trackBlockMetadataFrom, err := exec.BlockNumberToMessageIndex(config().TrackBlockMetadataFrom) + if err != nil { + return nil, err + } + streamer.trackBlockMetadataFrom = trackBlockMetadataFrom + } return streamer, nil } @@ -1045,6 +1058,9 @@ func (s *TransactionStreamer) writeMessage(pos arbutil.MessageIndex, msg arbosty // This also allows update of BatchGasCost in message without mistakenly erasing BlockMetadata key = dbKey(blockMetadataInputFeedPrefix, uint64(pos)) return batch.Put(key, msg.BlockMetadata) + } else if s.trackBlockMetadataFrom != 0 && pos >= s.trackBlockMetadataFrom { + key = dbKey(missingBlockMetadataInputFeedPrefix, uint64(pos)) + return batch.Put(key, nil) } return nil } diff --git a/execution/gethexec/node.go b/execution/gethexec/node.go index 32e43874f2..54fbd0d429 100644 --- a/execution/gethexec/node.go +++ b/execution/gethexec/node.go @@ -439,6 +439,9 @@ func (n *ExecutionNode) SetConsensusClient(consensus execution.FullConsensusClie func (n *ExecutionNode) MessageIndexToBlockNumber(messageNum arbutil.MessageIndex) uint64 { return n.ExecEngine.MessageIndexToBlockNumber(messageNum) } +func (n *ExecutionNode) BlockNumberToMessageIndex(blockNum uint64) (arbutil.MessageIndex, error) { + return n.ExecEngine.BlockNumberToMessageIndex(blockNum) +} func (n *ExecutionNode) Maintenance() error { return n.ChainDB.Compact(nil, nil) diff --git a/execution/interface.go b/execution/interface.go index 01f71d4422..700ae61ecd 100644 --- a/execution/interface.go +++ b/execution/interface.go @@ -33,6 +33,8 @@ type ExecutionClient interface { HeadMessageNumber() (arbutil.MessageIndex, error) HeadMessageNumberSync(t *testing.T) (arbutil.MessageIndex, error) ResultAtPos(pos arbutil.MessageIndex) (*MessageResult, error) + MessageIndexToBlockNumber(messageNum arbutil.MessageIndex) uint64 + BlockNumberToMessageIndex(blockNum uint64) (arbutil.MessageIndex, error) } // needed for validators / stakers From c4f43162f2c65ea32ab6c167d5b4988cc507675b Mon Sep 17 00:00:00 2001 From: Ganesh Vanahalli Date: Wed, 6 Nov 2024 12:43:32 +0530 Subject: [PATCH 02/14] delete affected missing blockMetadata trackers from ArbDB in case of a Reorg --- arbnode/transaction_streamer.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/arbnode/transaction_streamer.go b/arbnode/transaction_streamer.go index f1747e879e..2d2192d4b9 100644 --- a/arbnode/transaction_streamer.go +++ b/arbnode/transaction_streamer.go @@ -399,6 +399,10 @@ func (s *TransactionStreamer) reorg(batch ethdb.Batch, count arbutil.MessageInde if err != nil { return err } + err = deleteStartingAt(s.db, batch, missingBlockMetadataInputFeedPrefix, uint64ToKey(uint64(count))) + if err != nil { + return err + } err = deleteStartingAt(s.db, batch, messagePrefix, uint64ToKey(uint64(count))) if err != nil { return err From 4722102980f35da0bb1b530724d7a6ab4ca14012 Mon Sep 17 00:00:00 2001 From: Ganesh Vanahalli Date: Wed, 6 Nov 2024 14:59:56 +0530 Subject: [PATCH 03/14] add plumbing to allow starting BlockMetadataRebuilder --- arbnode/blockmetadata.go | 17 ++++++-- arbnode/node.go | 86 +++++++++++++++++++++++++--------------- 2 files changed, 66 insertions(+), 37 deletions(-) diff --git a/arbnode/blockmetadata.go b/arbnode/blockmetadata.go index e2b0ad6b3c..35691fa064 100644 --- a/arbnode/blockmetadata.go +++ b/arbnode/blockmetadata.go @@ -4,6 +4,8 @@ import ( "bytes" "context" "encoding/binary" + "errors" + "fmt" "time" "github.com/spf13/pflag" @@ -28,6 +30,13 @@ type BlockMetadataRebuilderConfig struct { APIBlocksLimit int `koanf:"api-blocks-limit"` } +func (c *BlockMetadataRebuilderConfig) Validate() error { + if c.APIBlocksLimit < 0 { + return errors.New("api-blocks-limit cannot be negative") + } + return nil +} + var DefaultBlockMetadataRebuilderConfig = BlockMetadataRebuilderConfig{ Enable: false, RebuildInterval: time.Minute * 5, @@ -45,19 +54,19 @@ func BlockMetadataRebuilderConfigAddOptions(prefix string, f *pflag.FlagSet) { type BlockMetadataRebuilder struct { stopwaiter.StopWaiter - config *BlockMetadataRebuilderConfig + config BlockMetadataRebuilderConfig db ethdb.Database client *rpc.Client exec execution.ExecutionClient } -func NewBlockMetadataRebuilder(ctx context.Context, c *BlockMetadataRebuilderConfig, db ethdb.Database, exec execution.ExecutionClient) (*BlockMetadataRebuilder, error) { +func NewBlockMetadataRebuilder(ctx context.Context, c BlockMetadataRebuilderConfig, db ethdb.Database, exec execution.ExecutionClient) (*BlockMetadataRebuilder, error) { var err error var jwt *common.Hash if c.JWTSecret != "" { jwt, err = signature.LoadSigningKey(c.JWTSecret) if err != nil { - return nil, err + return nil, fmt.Errorf("BlockMetadataRebuilder: error loading jwt secret: %w", err) } } var client *rpc.Client @@ -67,7 +76,7 @@ func NewBlockMetadataRebuilder(ctx context.Context, c *BlockMetadataRebuilderCon client, err = rpc.DialOptions(ctx, c.Url, rpc.WithHTTPAuth(node.NewJWTAuth([32]byte(*jwt)))) } if err != nil { - return nil, err + return nil, fmt.Errorf("BlockMetadataRebuilder: error connecting to bulk blockMetadata API: %w", err) } return &BlockMetadataRebuilder{ config: c, diff --git a/arbnode/node.go b/arbnode/node.go index 0961a7ed43..80603cd8d1 100644 --- a/arbnode/node.go +++ b/arbnode/node.go @@ -77,22 +77,23 @@ func GenerateRollupConfig(prod bool, wasmModuleRoot common.Hash, rollupOwner com } type Config struct { - Sequencer bool `koanf:"sequencer"` - ParentChainReader headerreader.Config `koanf:"parent-chain-reader" reload:"hot"` - InboxReader InboxReaderConfig `koanf:"inbox-reader" reload:"hot"` - DelayedSequencer DelayedSequencerConfig `koanf:"delayed-sequencer" reload:"hot"` - BatchPoster BatchPosterConfig `koanf:"batch-poster" reload:"hot"` - MessagePruner MessagePrunerConfig `koanf:"message-pruner" reload:"hot"` - BlockValidator staker.BlockValidatorConfig `koanf:"block-validator" reload:"hot"` - Feed broadcastclient.FeedConfig `koanf:"feed" reload:"hot"` - Staker staker.L1ValidatorConfig `koanf:"staker" reload:"hot"` - SeqCoordinator SeqCoordinatorConfig `koanf:"seq-coordinator"` - DataAvailability das.DataAvailabilityConfig `koanf:"data-availability"` - SyncMonitor SyncMonitorConfig `koanf:"sync-monitor"` - Dangerous DangerousConfig `koanf:"dangerous"` - TransactionStreamer TransactionStreamerConfig `koanf:"transaction-streamer" reload:"hot"` - Maintenance MaintenanceConfig `koanf:"maintenance" reload:"hot"` - ResourceMgmt resourcemanager.Config `koanf:"resource-mgmt" reload:"hot"` + Sequencer bool `koanf:"sequencer"` + ParentChainReader headerreader.Config `koanf:"parent-chain-reader" reload:"hot"` + InboxReader InboxReaderConfig `koanf:"inbox-reader" reload:"hot"` + DelayedSequencer DelayedSequencerConfig `koanf:"delayed-sequencer" reload:"hot"` + BatchPoster BatchPosterConfig `koanf:"batch-poster" reload:"hot"` + MessagePruner MessagePrunerConfig `koanf:"message-pruner" reload:"hot"` + BlockValidator staker.BlockValidatorConfig `koanf:"block-validator" reload:"hot"` + Feed broadcastclient.FeedConfig `koanf:"feed" reload:"hot"` + Staker staker.L1ValidatorConfig `koanf:"staker" reload:"hot"` + SeqCoordinator SeqCoordinatorConfig `koanf:"seq-coordinator"` + DataAvailability das.DataAvailabilityConfig `koanf:"data-availability"` + SyncMonitor SyncMonitorConfig `koanf:"sync-monitor"` + Dangerous DangerousConfig `koanf:"dangerous"` + TransactionStreamer TransactionStreamerConfig `koanf:"transaction-streamer" reload:"hot"` + Maintenance MaintenanceConfig `koanf:"maintenance" reload:"hot"` + ResourceMgmt resourcemanager.Config `koanf:"resource-mgmt" reload:"hot"` + BlockMetadataRebuilder BlockMetadataRebuilderConfig `koanf:"block-metadata-rebuilder" reload:"hot"` // SnapSyncConfig is only used for testing purposes, these should not be configured in production. SnapSyncTest SnapSyncConfig } @@ -129,6 +130,9 @@ func (c *Config) Validate() error { if err := c.Staker.Validate(); err != nil { return err } + if err := c.BlockMetadataRebuilder.Validate(); err != nil { + return err + } if c.Sequencer && c.TransactionStreamer.TrackBlockMetadataFrom == 0 { return errors.New("when sequencer is enabled track-missing-block-metadata should be enabled as well") } @@ -161,26 +165,28 @@ func ConfigAddOptions(prefix string, f *flag.FlagSet, feedInputEnable bool, feed DangerousConfigAddOptions(prefix+".dangerous", f) TransactionStreamerConfigAddOptions(prefix+".transaction-streamer", f) MaintenanceConfigAddOptions(prefix+".maintenance", f) + BlockMetadataRebuilderConfigAddOptions(prefix+"block-metadata-rebuilder", f) } var ConfigDefault = Config{ - Sequencer: false, - ParentChainReader: headerreader.DefaultConfig, - InboxReader: DefaultInboxReaderConfig, - DelayedSequencer: DefaultDelayedSequencerConfig, - BatchPoster: DefaultBatchPosterConfig, - MessagePruner: DefaultMessagePrunerConfig, - BlockValidator: staker.DefaultBlockValidatorConfig, - Feed: broadcastclient.FeedConfigDefault, - Staker: staker.DefaultL1ValidatorConfig, - SeqCoordinator: DefaultSeqCoordinatorConfig, - DataAvailability: das.DefaultDataAvailabilityConfig, - SyncMonitor: DefaultSyncMonitorConfig, - Dangerous: DefaultDangerousConfig, - TransactionStreamer: DefaultTransactionStreamerConfig, - ResourceMgmt: resourcemanager.DefaultConfig, - Maintenance: DefaultMaintenanceConfig, - SnapSyncTest: DefaultSnapSyncConfig, + Sequencer: false, + ParentChainReader: headerreader.DefaultConfig, + InboxReader: DefaultInboxReaderConfig, + DelayedSequencer: DefaultDelayedSequencerConfig, + BatchPoster: DefaultBatchPosterConfig, + MessagePruner: DefaultMessagePrunerConfig, + BlockValidator: staker.DefaultBlockValidatorConfig, + Feed: broadcastclient.FeedConfigDefault, + Staker: staker.DefaultL1ValidatorConfig, + SeqCoordinator: DefaultSeqCoordinatorConfig, + DataAvailability: das.DefaultDataAvailabilityConfig, + SyncMonitor: DefaultSyncMonitorConfig, + Dangerous: DefaultDangerousConfig, + TransactionStreamer: DefaultTransactionStreamerConfig, + ResourceMgmt: resourcemanager.DefaultConfig, + Maintenance: DefaultMaintenanceConfig, + BlockMetadataRebuilder: DefaultBlockMetadataRebuilderConfig, + SnapSyncTest: DefaultSnapSyncConfig, } func ConfigDefaultL1Test() *Config { @@ -275,6 +281,7 @@ type Node struct { MaintenanceRunner *MaintenanceRunner DASLifecycleManager *das.LifecycleManager SyncMonitor *SyncMonitor + blockMetadataRebuilder *BlockMetadataRebuilder configFetcher ConfigFetcher ctx context.Context } @@ -483,6 +490,14 @@ func createNodeImpl( } } + var blockMetadataRebuilder *BlockMetadataRebuilder + if config.BlockMetadataRebuilder.Enable { + blockMetadataRebuilder, err = NewBlockMetadataRebuilder(ctx, config.BlockMetadataRebuilder, arbDb, exec) + if err != nil { + return nil, err + } + } + if !config.ParentChainReader.Enable { return &Node{ ArbDB: arbDb, @@ -506,6 +521,7 @@ func createNodeImpl( MaintenanceRunner: maintenanceRunner, DASLifecycleManager: nil, SyncMonitor: syncMonitor, + blockMetadataRebuilder: blockMetadataRebuilder, configFetcher: configFetcher, ctx: ctx, }, nil @@ -742,6 +758,7 @@ func createNodeImpl( MaintenanceRunner: maintenanceRunner, DASLifecycleManager: dasLifecycleManager, SyncMonitor: syncMonitor, + blockMetadataRebuilder: blockMetadataRebuilder, configFetcher: configFetcher, ctx: ctx, }, nil @@ -925,6 +942,9 @@ func (n *Node) Start(ctx context.Context) error { n.BroadcastClients.Start(ctx) }() } + if n.blockMetadataRebuilder != nil { + n.blockMetadataRebuilder.Start(ctx) + } if n.configFetcher != nil { n.configFetcher.Start(ctx) } From df96876ef09e26ef476b396684f27f6ae506a0e4 Mon Sep 17 00:00:00 2001 From: Ganesh Vanahalli Date: Wed, 6 Nov 2024 17:17:02 +0530 Subject: [PATCH 04/14] test bulk syncing of missing blockMetadata --- system_tests/timeboost_test.go | 130 ++++++++++++++++++++++++++++++--- 1 file changed, 121 insertions(+), 9 deletions(-) diff --git a/system_tests/timeboost_test.go b/system_tests/timeboost_test.go index 49d9d5fb16..276ff4c08a 100644 --- a/system_tests/timeboost_test.go +++ b/system_tests/timeboost_test.go @@ -27,6 +27,7 @@ import ( "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/rpc" "github.com/offchainlabs/nitro/arbnode" + "github.com/offchainlabs/nitro/arbos/arbostypes" "github.com/offchainlabs/nitro/arbos/util" "github.com/offchainlabs/nitro/arbutil" "github.com/offchainlabs/nitro/broadcastclient" @@ -42,9 +43,129 @@ import ( "github.com/offchainlabs/nitro/util/containers" "github.com/offchainlabs/nitro/util/redisutil" "github.com/offchainlabs/nitro/util/stopwaiter" + "github.com/offchainlabs/nitro/util/testhelpers" "github.com/stretchr/testify/require" ) +var blockMetadataInputFeedKey = func(pos uint64) []byte { + var key []byte + prefix := []byte("t") + key = append(key, prefix...) + data := make([]byte, 8) + binary.BigEndian.PutUint64(data, pos) + key = append(key, data...) + return key +} + +func TestTimeboostBulkBlockMetadataRebuilder(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + builder := NewNodeBuilder(ctx).DefaultConfig(t, true) + httpConfig := genericconf.HTTPConfigDefault + httpConfig.Addr = "127.0.0.1" + httpConfig.Apply(builder.l2StackConfig) + builder.execConfig.BlockMetadataApiCacheSize = 0 // Caching is disabled + builder.nodeConfig.TransactionStreamer.TrackBlockMetadataFrom = 1 + cleanupSeq := builder.Build(t) + defer cleanupSeq() + + // Generate blocks until current block is > 20 + arbDb := builder.L2.ConsensusNode.ArbDB + builder.L2Info.GenerateAccount("User") + user := builder.L2Info.GetDefaultTransactOpts("User", ctx) + var latestL2 uint64 + var err error + for i := 0; ; i++ { + builder.L2.TransferBalanceTo(t, "Owner", util.RemapL1Address(user.From), big.NewInt(1e18), builder.L2Info) + latestL2, err = builder.L2.Client.BlockNumber(ctx) + Require(t, err) + // Clean BlockMetadata from arbDB so that we can modify it at will + Require(t, arbDb.Delete(blockMetadataInputFeedKey(latestL2))) + if latestL2 > uint64(20) { + break + } + } + var sampleBulkData []arbostypes.BlockMetadata + for i := 1; i <= int(latestL2); i++ { + blockMetadata := []byte{0, uint8(i)} + sampleBulkData = append(sampleBulkData, blockMetadata) + arbDb.Put(blockMetadataInputFeedKey(uint64(i)), blockMetadata) + } + + ndcfg := arbnode.ConfigDefaultL1NonSequencerTest() + ndcfg.TransactionStreamer.TrackBlockMetadataFrom = 1 + newNode, cleanupNewNode := builder.Build2ndNode(t, &SecondNodeParams{ + nodeConfig: ndcfg, + stackConfig: testhelpers.CreateStackConfigForTest(t.TempDir()), + }) + defer cleanupNewNode() + + // Wait for second node to catchup via L1, since L1 doesn't have the blockMetadata, we ensure that messages are tracked with missingBlockMetadataInputFeedPrefix prefix + for { + current, err := newNode.Client.BlockNumber(ctx) + Require(t, err) + if current == latestL2 { + break + } + time.Sleep(time.Second) + } + + blockMetadataInputFeedPrefix := []byte("t") + missingBlockMetadataInputFeedPrefix := []byte("mt") + arbDb = newNode.ConsensusNode.ArbDB + + // Check if all block numbers with missingBlockMetadataInputFeedPrefix are present as keys in arbDB and that no keys with blockMetadataInputFeedPrefix + iter := arbDb.NewIterator(blockMetadataInputFeedPrefix, nil) + for iter.Next() { + keyBytes := bytes.TrimPrefix(iter.Key(), blockMetadataInputFeedPrefix) + t.Fatalf("unexpected presence of blockMetadata when blocks are synced via L1. msgSeqNum: %d", binary.BigEndian.Uint64(keyBytes)) + } + iter.Release() + iter = arbDb.NewIterator(missingBlockMetadataInputFeedPrefix, nil) + pos := uint64(1) + for iter.Next() { + keyBytes := bytes.TrimPrefix(iter.Key(), missingBlockMetadataInputFeedPrefix) + if pos != binary.BigEndian.Uint64(keyBytes) { + t.Fatalf("unexpected msgSeqNum with missingBlockMetadataInputFeedPrefix for blockMetadata. Want: %d, Got: %d", pos, binary.BigEndian.Uint64(keyBytes)) + } + pos++ + } + if pos-1 != latestL2 { + t.Fatalf("number of keys with missingBlockMetadataInputFeedPrefix doesn't match expected value. Want: %d, Got: %d", latestL2, pos-1) + } + iter.Release() + + // Rebuild blockMetadata and cleanup trackers from ArbDB + blockMetadataRebuilder, err := arbnode.NewBlockMetadataRebuilder(ctx, arbnode.BlockMetadataRebuilderConfig{Url: "http://127.0.0.1:8547"}, arbDb, newNode.ExecNode) + Require(t, err) + blockMetadataRebuilder.Update(ctx) + + // Check if all blockMetadata was synced from bulk BlockMetadata API via the blockMetadataRebuilder and that trackers for missing blockMetadata were cleared + iter = arbDb.NewIterator(blockMetadataInputFeedPrefix, nil) + pos = uint64(1) + for iter.Next() { + keyBytes := bytes.TrimPrefix(iter.Key(), blockMetadataInputFeedPrefix) + if binary.BigEndian.Uint64(keyBytes) != pos { + t.Fatalf("unexpected msgSeqNum with blockMetadataInputFeedPrefix for blockMetadata. Want: %d, Got: %d", pos, binary.BigEndian.Uint64(keyBytes)) + } + if !bytes.Equal(sampleBulkData[pos-1], iter.Value()) { + t.Fatalf("blockMetadata mismatch. Want: %v, Got: %v", sampleBulkData[pos-1], iter.Value()) + } + pos++ + } + if pos-1 != latestL2 { + t.Fatalf("number of keys with blockMetadataInputFeedPrefix doesn't match expected value. Want: %d, Got: %d", latestL2, pos-1) + } + iter.Release() + iter = arbDb.NewIterator(missingBlockMetadataInputFeedPrefix, nil) + for iter.Next() { + keyBytes := bytes.TrimPrefix(iter.Key(), missingBlockMetadataInputFeedPrefix) + t.Fatalf("unexpected presence of msgSeqNum with missingBlockMetadataInputFeedPrefix, indicating missing of some blockMetadata after rebuilding. msgSeqNum: %d", binary.BigEndian.Uint64(keyBytes)) + } + iter.Release() +} + func TestTimeboostBulkBlockMetadataAPI(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -55,15 +176,6 @@ func TestTimeboostBulkBlockMetadataAPI(t *testing.T) { defer cleanup() arbDb := builder.L2.ConsensusNode.ArbDB - blockMetadataInputFeedKey := func(pos uint64) []byte { - var key []byte - prefix := []byte("t") - key = append(key, prefix...) - data := make([]byte, 8) - binary.BigEndian.PutUint64(data, pos) - key = append(key, data...) - return key - } // Generate blocks until current block is end start := 1 From d8137ec50f3647870a444b512f408ea380a1434d Mon Sep 17 00:00:00 2001 From: Ganesh Vanahalli Date: Fri, 8 Nov 2024 16:06:46 +0530 Subject: [PATCH 05/14] minor bug fixes and improvements --- arbnode/blockmetadata.go | 57 ++++++++++++++++++++-------------------- arbnode/node.go | 3 --- 2 files changed, 28 insertions(+), 32 deletions(-) diff --git a/arbnode/blockmetadata.go b/arbnode/blockmetadata.go index 35691fa064..bbcbb3c045 100644 --- a/arbnode/blockmetadata.go +++ b/arbnode/blockmetadata.go @@ -4,7 +4,6 @@ import ( "bytes" "context" "encoding/binary" - "errors" "fmt" "time" @@ -27,14 +26,7 @@ type BlockMetadataRebuilderConfig struct { Url string `koanf:"url"` JWTSecret string `koanf:"jwt-secret"` RebuildInterval time.Duration `koanf:"rebuild-interval"` - APIBlocksLimit int `koanf:"api-blocks-limit"` -} - -func (c *BlockMetadataRebuilderConfig) Validate() error { - if c.APIBlocksLimit < 0 { - return errors.New("api-blocks-limit cannot be negative") - } - return nil + APIBlocksLimit uint64 `koanf:"api-blocks-limit"` } var DefaultBlockMetadataRebuilderConfig = BlockMetadataRebuilderConfig{ @@ -48,7 +40,7 @@ func BlockMetadataRebuilderConfigAddOptions(prefix string, f *pflag.FlagSet) { f.String(prefix+".url", DefaultBlockMetadataRebuilderConfig.Url, "url for bulk blockMetadata api") f.String(prefix+".jwt-secret", DefaultBlockMetadataRebuilderConfig.JWTSecret, "filepath of jwt secret") f.Duration(prefix+".rebuild-interval", DefaultBlockMetadataRebuilderConfig.RebuildInterval, "interval at which blockMetadata is synced regularly") - f.Int(prefix+".api-blocks-limit", DefaultBlockMetadataRebuilderConfig.APIBlocksLimit, "maximum number of blocks allowed to be queried for blockMetadata per arb_getRawBlockMetadata query.\n"+ + f.Uint64(prefix+".api-blocks-limit", DefaultBlockMetadataRebuilderConfig.APIBlocksLimit, "maximum number of blocks allowed to be queried for blockMetadata per arb_getRawBlockMetadata query.\n"+ "This should be set lesser than or equal to the value set on the api provider side") } @@ -124,6 +116,22 @@ func (b *BlockMetadataRebuilder) PushBlockMetadataToDB(query []uint64, result [] } func (b *BlockMetadataRebuilder) Update(ctx context.Context) time.Duration { + handleQuery := func(query []uint64) bool { + result, err := b.Fetch( + ctx, + b.exec.MessageIndexToBlockNumber(arbutil.MessageIndex(query[0])), + b.exec.MessageIndexToBlockNumber(arbutil.MessageIndex(query[len(query)-1])), + ) + if err != nil { + log.Error("Error getting result from bulk blockMetadata API", "err", err) + return false + } + if err = b.PushBlockMetadataToDB(query, result); err != nil { + log.Error("Error committing result from bulk blockMetadata API to ArbDB", "err", err) + return false + } + return true + } iter := b.db.NewIterator(missingBlockMetadataInputFeedPrefix, nil) defer iter.Release() var query []uint64 @@ -131,30 +139,21 @@ func (b *BlockMetadataRebuilder) Update(ctx context.Context) time.Duration { keyBytes := bytes.TrimPrefix(iter.Key(), missingBlockMetadataInputFeedPrefix) query = append(query, binary.BigEndian.Uint64(keyBytes)) end := len(query) - 1 - if query[end]-query[0] >= uint64(b.config.APIBlocksLimit) { - if query[end]-query[0] > uint64(b.config.APIBlocksLimit) { - if len(query) >= 2 { - end -= 1 - } else { - end = 0 - } - } - result, err := b.Fetch( - ctx, - b.exec.MessageIndexToBlockNumber(arbutil.MessageIndex(query[0])), - b.exec.MessageIndexToBlockNumber(arbutil.MessageIndex(query[end])), - ) - if err != nil { - log.Error("Error getting result from bulk blockMetadata API", "err", err) - return b.config.RebuildInterval // backoff + if query[end]-query[0]+1 >= uint64(b.config.APIBlocksLimit) { + if query[end]-query[0]+1 > uint64(b.config.APIBlocksLimit) && len(query) >= 2 { + end -= 1 } - if err = b.PushBlockMetadataToDB(query[:end+1], result); err != nil { - log.Error("Error committing result from bulk blockMetadata API to ArbDB", "err", err) - return b.config.RebuildInterval // backoff + if success := handleQuery(query[:end+1]); !success { + return b.config.RebuildInterval } query = query[end+1:] } } + if len(query) > 0 { + if success := handleQuery(query); !success { + return b.config.RebuildInterval + } + } return b.config.RebuildInterval } diff --git a/arbnode/node.go b/arbnode/node.go index 80603cd8d1..5e27e3141e 100644 --- a/arbnode/node.go +++ b/arbnode/node.go @@ -130,9 +130,6 @@ func (c *Config) Validate() error { if err := c.Staker.Validate(); err != nil { return err } - if err := c.BlockMetadataRebuilder.Validate(); err != nil { - return err - } if c.Sequencer && c.TransactionStreamer.TrackBlockMetadataFrom == 0 { return errors.New("when sequencer is enabled track-missing-block-metadata should be enabled as well") } From dfe0c51b6865de5f18db82eb733d868cf7dcf0f9 Mon Sep 17 00:00:00 2001 From: Ganesh Vanahalli Date: Tue, 12 Nov 2024 13:21:44 +0530 Subject: [PATCH 06/14] address PR comments --- arbnode/blockmetadata.go | 69 ++++++++++++++------------------- arbnode/node.go | 3 ++ arbnode/schema.go | 2 +- arbnode/transaction_streamer.go | 2 +- system_tests/timeboost_test.go | 14 ++++--- 5 files changed, 43 insertions(+), 47 deletions(-) diff --git a/arbnode/blockmetadata.go b/arbnode/blockmetadata.go index bbcbb3c045..d5d8565f06 100644 --- a/arbnode/blockmetadata.go +++ b/arbnode/blockmetadata.go @@ -4,71 +4,54 @@ import ( "bytes" "context" "encoding/binary" - "fmt" "time" "github.com/spf13/pflag" - "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/rpc" "github.com/offchainlabs/nitro/arbutil" "github.com/offchainlabs/nitro/execution" "github.com/offchainlabs/nitro/execution/gethexec" - "github.com/offchainlabs/nitro/util/signature" + "github.com/offchainlabs/nitro/util/rpcclient" "github.com/offchainlabs/nitro/util/stopwaiter" ) type BlockMetadataRebuilderConfig struct { - Enable bool `koanf:"enable"` - Url string `koanf:"url"` - JWTSecret string `koanf:"jwt-secret"` - RebuildInterval time.Duration `koanf:"rebuild-interval"` - APIBlocksLimit uint64 `koanf:"api-blocks-limit"` + Enable bool `koanf:"enable"` + Source rpcclient.ClientConfig `koanf:"source"` + SyncInterval time.Duration `koanf:"sync-interval"` + APIBlocksLimit uint64 `koanf:"api-blocks-limit"` } var DefaultBlockMetadataRebuilderConfig = BlockMetadataRebuilderConfig{ - Enable: false, - RebuildInterval: time.Minute * 5, - APIBlocksLimit: 100, + Enable: false, + Source: rpcclient.DefaultClientConfig, + SyncInterval: time.Minute * 5, + APIBlocksLimit: 100, } func BlockMetadataRebuilderConfigAddOptions(prefix string, f *pflag.FlagSet) { - f.Bool(prefix+".enable", DefaultBlockMetadataRebuilderConfig.Enable, "enable syncing blockMetadata using a bulk metadata api") - f.String(prefix+".url", DefaultBlockMetadataRebuilderConfig.Url, "url for bulk blockMetadata api") - f.String(prefix+".jwt-secret", DefaultBlockMetadataRebuilderConfig.JWTSecret, "filepath of jwt secret") - f.Duration(prefix+".rebuild-interval", DefaultBlockMetadataRebuilderConfig.RebuildInterval, "interval at which blockMetadata is synced regularly") + f.Bool(prefix+".enable", DefaultBlockMetadataRebuilderConfig.Enable, "enable syncing blockMetadata using a bulk blockMetadata api") + rpcclient.RPCClientAddOptions(prefix+".source", f, &DefaultBlockMetadataRebuilderConfig.Source) + f.Duration(prefix+".rebuild-interval", DefaultBlockMetadataRebuilderConfig.SyncInterval, "interval at which blockMetadata are synced regularly") f.Uint64(prefix+".api-blocks-limit", DefaultBlockMetadataRebuilderConfig.APIBlocksLimit, "maximum number of blocks allowed to be queried for blockMetadata per arb_getRawBlockMetadata query.\n"+ - "This should be set lesser than or equal to the value set on the api provider side") + "This should be set lesser than or equal to the limit on the api provider side") } type BlockMetadataRebuilder struct { stopwaiter.StopWaiter config BlockMetadataRebuilderConfig db ethdb.Database - client *rpc.Client + client *rpcclient.RpcClient exec execution.ExecutionClient } func NewBlockMetadataRebuilder(ctx context.Context, c BlockMetadataRebuilderConfig, db ethdb.Database, exec execution.ExecutionClient) (*BlockMetadataRebuilder, error) { - var err error - var jwt *common.Hash - if c.JWTSecret != "" { - jwt, err = signature.LoadSigningKey(c.JWTSecret) - if err != nil { - return nil, fmt.Errorf("BlockMetadataRebuilder: error loading jwt secret: %w", err) - } - } - var client *rpc.Client - if jwt == nil { - client, err = rpc.DialOptions(ctx, c.Url) - } else { - client, err = rpc.DialOptions(ctx, c.Url, rpc.WithHTTPAuth(node.NewJWTAuth([32]byte(*jwt)))) - } - if err != nil { - return nil, fmt.Errorf("BlockMetadataRebuilder: error connecting to bulk blockMetadata API: %w", err) + client := rpcclient.NewRpcClient(func() *rpcclient.ClientConfig { return &c.Source }, nil) + if err := client.Start(ctx); err != nil { + return nil, err } return &BlockMetadataRebuilder{ config: c, @@ -95,7 +78,7 @@ func ArrayToMap[T comparable](arr []T) map[T]struct{} { return ret } -func (b *BlockMetadataRebuilder) PushBlockMetadataToDB(query []uint64, result []gethexec.NumberAndBlockMetadata) error { +func (b *BlockMetadataRebuilder) PersistBlockMetadata(query []uint64, result []gethexec.NumberAndBlockMetadata) error { batch := b.db.NewBatch() queryMap := ArrayToMap(query) for _, elem := range result { @@ -110,6 +93,13 @@ func (b *BlockMetadataRebuilder) PushBlockMetadataToDB(query []uint64, result [] if err := batch.Delete(dbKey(missingBlockMetadataInputFeedPrefix, uint64(pos))); err != nil { return err } + // If we exceeded the ideal batch size, commit and reset + if batch.ValueSize() >= ethdb.IdealBatchSize { + if err := batch.Write(); err != nil { + return err + } + batch.Reset() + } } } return batch.Write() @@ -126,7 +116,7 @@ func (b *BlockMetadataRebuilder) Update(ctx context.Context) time.Duration { log.Error("Error getting result from bulk blockMetadata API", "err", err) return false } - if err = b.PushBlockMetadataToDB(query, result); err != nil { + if err = b.PersistBlockMetadata(query, result); err != nil { log.Error("Error committing result from bulk blockMetadata API to ArbDB", "err", err) return false } @@ -144,17 +134,17 @@ func (b *BlockMetadataRebuilder) Update(ctx context.Context) time.Duration { end -= 1 } if success := handleQuery(query[:end+1]); !success { - return b.config.RebuildInterval + return b.config.SyncInterval } query = query[end+1:] } } if len(query) > 0 { if success := handleQuery(query); !success { - return b.config.RebuildInterval + return b.config.SyncInterval } } - return b.config.RebuildInterval + return b.config.SyncInterval } func (b *BlockMetadataRebuilder) Start(ctx context.Context) { @@ -164,4 +154,5 @@ func (b *BlockMetadataRebuilder) Start(ctx context.Context) { func (b *BlockMetadataRebuilder) StopAndWait() { b.StopWaiter.StopAndWait() + b.client.Close() } diff --git a/arbnode/node.go b/arbnode/node.go index 5e27e3141e..af9cadf7f4 100644 --- a/arbnode/node.go +++ b/arbnode/node.go @@ -133,6 +133,9 @@ func (c *Config) Validate() error { if c.Sequencer && c.TransactionStreamer.TrackBlockMetadataFrom == 0 { return errors.New("when sequencer is enabled track-missing-block-metadata should be enabled as well") } + if c.TransactionStreamer.TrackBlockMetadataFrom != 0 && !c.BlockMetadataRebuilder.Enable { + log.Warn("track-missing-block-metadata is set but blockMetadata rebuilder is not enabled") + } return nil } diff --git a/arbnode/schema.go b/arbnode/schema.go index 09554d6161..88d3139f21 100644 --- a/arbnode/schema.go +++ b/arbnode/schema.go @@ -7,7 +7,7 @@ var ( messagePrefix []byte = []byte("m") // maps a message sequence number to a message blockHashInputFeedPrefix []byte = []byte("b") // maps a message sequence number to a block hash received through the input feed blockMetadataInputFeedPrefix []byte = []byte("t") // maps a message sequence number to a blockMetaData byte array received through the input feed - missingBlockMetadataInputFeedPrefix []byte = []byte("mt") // maps a message sequence number whose blockMetaData byte array is missing to nil + missingBlockMetadataInputFeedPrefix []byte = []byte("xt") // maps a message sequence number whose blockMetaData byte array is missing to nil. Leading "x" implies we are tracking the missing of such a data point messageResultPrefix []byte = []byte("r") // maps a message sequence number to a message result legacyDelayedMessagePrefix []byte = []byte("d") // maps a delayed sequence number to an accumulator and a message as serialized on L1 rlpDelayedMessagePrefix []byte = []byte("e") // maps a delayed sequence number to an accumulator and an RLP encoded message diff --git a/arbnode/transaction_streamer.go b/arbnode/transaction_streamer.go index 2d2192d4b9..d2e238ec7a 100644 --- a/arbnode/transaction_streamer.go +++ b/arbnode/transaction_streamer.go @@ -99,7 +99,7 @@ func TransactionStreamerConfigAddOptions(prefix string, f *flag.FlagSet) { f.Int(prefix+".max-broadcaster-queue-size", DefaultTransactionStreamerConfig.MaxBroadcasterQueueSize, "maximum cache of pending broadcaster messages") f.Int64(prefix+".max-reorg-resequence-depth", DefaultTransactionStreamerConfig.MaxReorgResequenceDepth, "maximum number of messages to attempt to resequence on reorg (0 = never resequence, -1 = always resequence)") f.Duration(prefix+".execute-message-loop-delay", DefaultTransactionStreamerConfig.ExecuteMessageLoopDelay, "delay when polling calls to execute messages") - f.Uint64(prefix+".track-block-metadata-from", DefaultTransactionStreamerConfig.TrackBlockMetadataFrom, "block number starting from which the missing of blockmetadata is being tracked in the local disk. Disabled by default") + f.Uint64(prefix+".track-block-metadata-from", DefaultTransactionStreamerConfig.TrackBlockMetadataFrom, "block number starting from which the missing of blockmetadata is being tracked in the local disk. Setting to zero (default value) disables this") } func NewTransactionStreamer( diff --git a/system_tests/timeboost_test.go b/system_tests/timeboost_test.go index 276ff4c08a..7288256d74 100644 --- a/system_tests/timeboost_test.go +++ b/system_tests/timeboost_test.go @@ -42,12 +42,13 @@ import ( "github.com/offchainlabs/nitro/util/arbmath" "github.com/offchainlabs/nitro/util/containers" "github.com/offchainlabs/nitro/util/redisutil" + "github.com/offchainlabs/nitro/util/rpcclient" "github.com/offchainlabs/nitro/util/stopwaiter" "github.com/offchainlabs/nitro/util/testhelpers" "github.com/stretchr/testify/require" ) -var blockMetadataInputFeedKey = func(pos uint64) []byte { +func blockMetadataInputFeedKey(pos uint64) []byte { var key []byte prefix := []byte("t") key = append(key, prefix...) @@ -94,7 +95,8 @@ func TestTimeboostBulkBlockMetadataRebuilder(t *testing.T) { } ndcfg := arbnode.ConfigDefaultL1NonSequencerTest() - ndcfg.TransactionStreamer.TrackBlockMetadataFrom = 1 + trackBlockMetadataFrom := uint64(5) + ndcfg.TransactionStreamer.TrackBlockMetadataFrom = trackBlockMetadataFrom newNode, cleanupNewNode := builder.Build2ndNode(t, &SecondNodeParams{ nodeConfig: ndcfg, stackConfig: testhelpers.CreateStackConfigForTest(t.TempDir()), @@ -112,7 +114,7 @@ func TestTimeboostBulkBlockMetadataRebuilder(t *testing.T) { } blockMetadataInputFeedPrefix := []byte("t") - missingBlockMetadataInputFeedPrefix := []byte("mt") + missingBlockMetadataInputFeedPrefix := []byte("xt") arbDb = newNode.ConsensusNode.ArbDB // Check if all block numbers with missingBlockMetadataInputFeedPrefix are present as keys in arbDB and that no keys with blockMetadataInputFeedPrefix @@ -123,7 +125,7 @@ func TestTimeboostBulkBlockMetadataRebuilder(t *testing.T) { } iter.Release() iter = arbDb.NewIterator(missingBlockMetadataInputFeedPrefix, nil) - pos := uint64(1) + pos := trackBlockMetadataFrom for iter.Next() { keyBytes := bytes.TrimPrefix(iter.Key(), missingBlockMetadataInputFeedPrefix) if pos != binary.BigEndian.Uint64(keyBytes) { @@ -137,13 +139,13 @@ func TestTimeboostBulkBlockMetadataRebuilder(t *testing.T) { iter.Release() // Rebuild blockMetadata and cleanup trackers from ArbDB - blockMetadataRebuilder, err := arbnode.NewBlockMetadataRebuilder(ctx, arbnode.BlockMetadataRebuilderConfig{Url: "http://127.0.0.1:8547"}, arbDb, newNode.ExecNode) + blockMetadataRebuilder, err := arbnode.NewBlockMetadataRebuilder(ctx, arbnode.BlockMetadataRebuilderConfig{Source: rpcclient.ClientConfig{URL: builder.L2.Stack.HTTPEndpoint()}}, arbDb, newNode.ExecNode) Require(t, err) blockMetadataRebuilder.Update(ctx) // Check if all blockMetadata was synced from bulk BlockMetadata API via the blockMetadataRebuilder and that trackers for missing blockMetadata were cleared iter = arbDb.NewIterator(blockMetadataInputFeedPrefix, nil) - pos = uint64(1) + pos = trackBlockMetadataFrom for iter.Next() { keyBytes := bytes.TrimPrefix(iter.Key(), blockMetadataInputFeedPrefix) if binary.BigEndian.Uint64(keyBytes) != pos { From bab01573d3985d716a139d75dbd3776b7308bc3d Mon Sep 17 00:00:00 2001 From: Ganesh Vanahalli Date: Wed, 13 Nov 2024 11:06:49 +0530 Subject: [PATCH 07/14] address PR comments --- arbnode/blockmetadata.go | 53 ++++++++----------- arbnode/node.go | 94 +++++++++++++++++----------------- arbnode/schema.go | 20 ++++---- system_tests/timeboost_test.go | 71 ++++++++++++++----------- util/common.go | 9 ++++ 5 files changed, 130 insertions(+), 117 deletions(-) create mode 100644 util/common.go diff --git a/arbnode/blockmetadata.go b/arbnode/blockmetadata.go index d5d8565f06..82928d9fa7 100644 --- a/arbnode/blockmetadata.go +++ b/arbnode/blockmetadata.go @@ -14,46 +14,47 @@ import ( "github.com/offchainlabs/nitro/arbutil" "github.com/offchainlabs/nitro/execution" "github.com/offchainlabs/nitro/execution/gethexec" + "github.com/offchainlabs/nitro/util" "github.com/offchainlabs/nitro/util/rpcclient" "github.com/offchainlabs/nitro/util/stopwaiter" ) -type BlockMetadataRebuilderConfig struct { +type BlockMetadataFetcherConfig struct { Enable bool `koanf:"enable"` Source rpcclient.ClientConfig `koanf:"source"` SyncInterval time.Duration `koanf:"sync-interval"` APIBlocksLimit uint64 `koanf:"api-blocks-limit"` } -var DefaultBlockMetadataRebuilderConfig = BlockMetadataRebuilderConfig{ +var DefaultBlockMetadataFetcherConfig = BlockMetadataFetcherConfig{ Enable: false, Source: rpcclient.DefaultClientConfig, SyncInterval: time.Minute * 5, APIBlocksLimit: 100, } -func BlockMetadataRebuilderConfigAddOptions(prefix string, f *pflag.FlagSet) { - f.Bool(prefix+".enable", DefaultBlockMetadataRebuilderConfig.Enable, "enable syncing blockMetadata using a bulk blockMetadata api") - rpcclient.RPCClientAddOptions(prefix+".source", f, &DefaultBlockMetadataRebuilderConfig.Source) - f.Duration(prefix+".rebuild-interval", DefaultBlockMetadataRebuilderConfig.SyncInterval, "interval at which blockMetadata are synced regularly") - f.Uint64(prefix+".api-blocks-limit", DefaultBlockMetadataRebuilderConfig.APIBlocksLimit, "maximum number of blocks allowed to be queried for blockMetadata per arb_getRawBlockMetadata query.\n"+ +func BlockMetadataFetcherConfigAddOptions(prefix string, f *pflag.FlagSet) { + f.Bool(prefix+".enable", DefaultBlockMetadataFetcherConfig.Enable, "enable syncing blockMetadata using a bulk blockMetadata api") + rpcclient.RPCClientAddOptions(prefix+".source", f, &DefaultBlockMetadataFetcherConfig.Source) + f.Duration(prefix+".sync-interval", DefaultBlockMetadataFetcherConfig.SyncInterval, "interval at which blockMetadata are synced regularly") + f.Uint64(prefix+".api-blocks-limit", DefaultBlockMetadataFetcherConfig.APIBlocksLimit, "maximum number of blocks allowed to be queried for blockMetadata per arb_getRawBlockMetadata query.\n"+ "This should be set lesser than or equal to the limit on the api provider side") } -type BlockMetadataRebuilder struct { +type BlockMetadataFetcher struct { stopwaiter.StopWaiter - config BlockMetadataRebuilderConfig + config BlockMetadataFetcherConfig db ethdb.Database client *rpcclient.RpcClient exec execution.ExecutionClient } -func NewBlockMetadataRebuilder(ctx context.Context, c BlockMetadataRebuilderConfig, db ethdb.Database, exec execution.ExecutionClient) (*BlockMetadataRebuilder, error) { +func NewBlockMetadataFetcher(ctx context.Context, c BlockMetadataFetcherConfig, db ethdb.Database, exec execution.ExecutionClient) (*BlockMetadataFetcher, error) { client := rpcclient.NewRpcClient(func() *rpcclient.ClientConfig { return &c.Source }, nil) if err := client.Start(ctx); err != nil { return nil, err } - return &BlockMetadataRebuilder{ + return &BlockMetadataFetcher{ config: c, db: db, client: client, @@ -61,7 +62,7 @@ func NewBlockMetadataRebuilder(ctx context.Context, c BlockMetadataRebuilderConf }, nil } -func (b *BlockMetadataRebuilder) Fetch(ctx context.Context, fromBlock, toBlock uint64) ([]gethexec.NumberAndBlockMetadata, error) { +func (b *BlockMetadataFetcher) fetch(ctx context.Context, fromBlock, toBlock uint64) ([]gethexec.NumberAndBlockMetadata, error) { var result []gethexec.NumberAndBlockMetadata err := b.client.CallContext(ctx, &result, "arb_getRawBlockMetadata", rpc.BlockNumber(fromBlock), rpc.BlockNumber(toBlock)) if err != nil { @@ -70,17 +71,9 @@ func (b *BlockMetadataRebuilder) Fetch(ctx context.Context, fromBlock, toBlock u return result, nil } -func ArrayToMap[T comparable](arr []T) map[T]struct{} { - ret := make(map[T]struct{}) - for _, elem := range arr { - ret[elem] = struct{}{} - } - return ret -} - -func (b *BlockMetadataRebuilder) PersistBlockMetadata(query []uint64, result []gethexec.NumberAndBlockMetadata) error { +func (b *BlockMetadataFetcher) persistBlockMetadata(query []uint64, result []gethexec.NumberAndBlockMetadata) error { batch := b.db.NewBatch() - queryMap := ArrayToMap(query) + queryMap := util.ArrayToMap(query) for _, elem := range result { pos, err := b.exec.BlockNumberToMessageIndex(elem.BlockNumber) if err != nil { @@ -93,7 +86,7 @@ func (b *BlockMetadataRebuilder) PersistBlockMetadata(query []uint64, result []g if err := batch.Delete(dbKey(missingBlockMetadataInputFeedPrefix, uint64(pos))); err != nil { return err } - // If we exceeded the ideal batch size, commit and reset + // If we reached the ideal batch size, commit and reset if batch.ValueSize() >= ethdb.IdealBatchSize { if err := batch.Write(); err != nil { return err @@ -105,9 +98,9 @@ func (b *BlockMetadataRebuilder) PersistBlockMetadata(query []uint64, result []g return batch.Write() } -func (b *BlockMetadataRebuilder) Update(ctx context.Context) time.Duration { +func (b *BlockMetadataFetcher) Update(ctx context.Context) time.Duration { handleQuery := func(query []uint64) bool { - result, err := b.Fetch( + result, err := b.fetch( ctx, b.exec.MessageIndexToBlockNumber(arbutil.MessageIndex(query[0])), b.exec.MessageIndexToBlockNumber(arbutil.MessageIndex(query[len(query)-1])), @@ -116,7 +109,7 @@ func (b *BlockMetadataRebuilder) Update(ctx context.Context) time.Duration { log.Error("Error getting result from bulk blockMetadata API", "err", err) return false } - if err = b.PersistBlockMetadata(query, result); err != nil { + if err = b.persistBlockMetadata(query, result); err != nil { log.Error("Error committing result from bulk blockMetadata API to ArbDB", "err", err) return false } @@ -140,19 +133,17 @@ func (b *BlockMetadataRebuilder) Update(ctx context.Context) time.Duration { } } if len(query) > 0 { - if success := handleQuery(query); !success { - return b.config.SyncInterval - } + _ = handleQuery(query) } return b.config.SyncInterval } -func (b *BlockMetadataRebuilder) Start(ctx context.Context) { +func (b *BlockMetadataFetcher) Start(ctx context.Context) { b.StopWaiter.Start(ctx, b) b.CallIteratively(b.Update) } -func (b *BlockMetadataRebuilder) StopAndWait() { +func (b *BlockMetadataFetcher) StopAndWait() { b.StopWaiter.StopAndWait() b.client.Close() } diff --git a/arbnode/node.go b/arbnode/node.go index af9cadf7f4..729775a772 100644 --- a/arbnode/node.go +++ b/arbnode/node.go @@ -77,23 +77,23 @@ func GenerateRollupConfig(prod bool, wasmModuleRoot common.Hash, rollupOwner com } type Config struct { - Sequencer bool `koanf:"sequencer"` - ParentChainReader headerreader.Config `koanf:"parent-chain-reader" reload:"hot"` - InboxReader InboxReaderConfig `koanf:"inbox-reader" reload:"hot"` - DelayedSequencer DelayedSequencerConfig `koanf:"delayed-sequencer" reload:"hot"` - BatchPoster BatchPosterConfig `koanf:"batch-poster" reload:"hot"` - MessagePruner MessagePrunerConfig `koanf:"message-pruner" reload:"hot"` - BlockValidator staker.BlockValidatorConfig `koanf:"block-validator" reload:"hot"` - Feed broadcastclient.FeedConfig `koanf:"feed" reload:"hot"` - Staker staker.L1ValidatorConfig `koanf:"staker" reload:"hot"` - SeqCoordinator SeqCoordinatorConfig `koanf:"seq-coordinator"` - DataAvailability das.DataAvailabilityConfig `koanf:"data-availability"` - SyncMonitor SyncMonitorConfig `koanf:"sync-monitor"` - Dangerous DangerousConfig `koanf:"dangerous"` - TransactionStreamer TransactionStreamerConfig `koanf:"transaction-streamer" reload:"hot"` - Maintenance MaintenanceConfig `koanf:"maintenance" reload:"hot"` - ResourceMgmt resourcemanager.Config `koanf:"resource-mgmt" reload:"hot"` - BlockMetadataRebuilder BlockMetadataRebuilderConfig `koanf:"block-metadata-rebuilder" reload:"hot"` + Sequencer bool `koanf:"sequencer"` + ParentChainReader headerreader.Config `koanf:"parent-chain-reader" reload:"hot"` + InboxReader InboxReaderConfig `koanf:"inbox-reader" reload:"hot"` + DelayedSequencer DelayedSequencerConfig `koanf:"delayed-sequencer" reload:"hot"` + BatchPoster BatchPosterConfig `koanf:"batch-poster" reload:"hot"` + MessagePruner MessagePrunerConfig `koanf:"message-pruner" reload:"hot"` + BlockValidator staker.BlockValidatorConfig `koanf:"block-validator" reload:"hot"` + Feed broadcastclient.FeedConfig `koanf:"feed" reload:"hot"` + Staker staker.L1ValidatorConfig `koanf:"staker" reload:"hot"` + SeqCoordinator SeqCoordinatorConfig `koanf:"seq-coordinator"` + DataAvailability das.DataAvailabilityConfig `koanf:"data-availability"` + SyncMonitor SyncMonitorConfig `koanf:"sync-monitor"` + Dangerous DangerousConfig `koanf:"dangerous"` + TransactionStreamer TransactionStreamerConfig `koanf:"transaction-streamer" reload:"hot"` + Maintenance MaintenanceConfig `koanf:"maintenance" reload:"hot"` + ResourceMgmt resourcemanager.Config `koanf:"resource-mgmt" reload:"hot"` + BlockMetadataFetcher BlockMetadataFetcherConfig `koanf:"block-metadata-fetcher" reload:"hot"` // SnapSyncConfig is only used for testing purposes, these should not be configured in production. SnapSyncTest SnapSyncConfig } @@ -131,10 +131,10 @@ func (c *Config) Validate() error { return err } if c.Sequencer && c.TransactionStreamer.TrackBlockMetadataFrom == 0 { - return errors.New("when sequencer is enabled track-missing-block-metadata should be enabled as well") + return errors.New("when sequencer is enabled track-missing-block-metadata-from should be set as well") } - if c.TransactionStreamer.TrackBlockMetadataFrom != 0 && !c.BlockMetadataRebuilder.Enable { - log.Warn("track-missing-block-metadata is set but blockMetadata rebuilder is not enabled") + if c.TransactionStreamer.TrackBlockMetadataFrom != 0 && !c.BlockMetadataFetcher.Enable { + log.Warn("track-missing-block-metadata-from is set but blockMetadata fetcher is not enabled") } return nil } @@ -165,28 +165,28 @@ func ConfigAddOptions(prefix string, f *flag.FlagSet, feedInputEnable bool, feed DangerousConfigAddOptions(prefix+".dangerous", f) TransactionStreamerConfigAddOptions(prefix+".transaction-streamer", f) MaintenanceConfigAddOptions(prefix+".maintenance", f) - BlockMetadataRebuilderConfigAddOptions(prefix+"block-metadata-rebuilder", f) + BlockMetadataFetcherConfigAddOptions(prefix+"block-metadata-fetcher", f) } var ConfigDefault = Config{ - Sequencer: false, - ParentChainReader: headerreader.DefaultConfig, - InboxReader: DefaultInboxReaderConfig, - DelayedSequencer: DefaultDelayedSequencerConfig, - BatchPoster: DefaultBatchPosterConfig, - MessagePruner: DefaultMessagePrunerConfig, - BlockValidator: staker.DefaultBlockValidatorConfig, - Feed: broadcastclient.FeedConfigDefault, - Staker: staker.DefaultL1ValidatorConfig, - SeqCoordinator: DefaultSeqCoordinatorConfig, - DataAvailability: das.DefaultDataAvailabilityConfig, - SyncMonitor: DefaultSyncMonitorConfig, - Dangerous: DefaultDangerousConfig, - TransactionStreamer: DefaultTransactionStreamerConfig, - ResourceMgmt: resourcemanager.DefaultConfig, - Maintenance: DefaultMaintenanceConfig, - BlockMetadataRebuilder: DefaultBlockMetadataRebuilderConfig, - SnapSyncTest: DefaultSnapSyncConfig, + Sequencer: false, + ParentChainReader: headerreader.DefaultConfig, + InboxReader: DefaultInboxReaderConfig, + DelayedSequencer: DefaultDelayedSequencerConfig, + BatchPoster: DefaultBatchPosterConfig, + MessagePruner: DefaultMessagePrunerConfig, + BlockValidator: staker.DefaultBlockValidatorConfig, + Feed: broadcastclient.FeedConfigDefault, + Staker: staker.DefaultL1ValidatorConfig, + SeqCoordinator: DefaultSeqCoordinatorConfig, + DataAvailability: das.DefaultDataAvailabilityConfig, + SyncMonitor: DefaultSyncMonitorConfig, + Dangerous: DefaultDangerousConfig, + TransactionStreamer: DefaultTransactionStreamerConfig, + ResourceMgmt: resourcemanager.DefaultConfig, + Maintenance: DefaultMaintenanceConfig, + BlockMetadataFetcher: DefaultBlockMetadataFetcherConfig, + SnapSyncTest: DefaultSnapSyncConfig, } func ConfigDefaultL1Test() *Config { @@ -281,7 +281,7 @@ type Node struct { MaintenanceRunner *MaintenanceRunner DASLifecycleManager *das.LifecycleManager SyncMonitor *SyncMonitor - blockMetadataRebuilder *BlockMetadataRebuilder + blockMetadataFetcher *BlockMetadataFetcher configFetcher ConfigFetcher ctx context.Context } @@ -490,9 +490,9 @@ func createNodeImpl( } } - var blockMetadataRebuilder *BlockMetadataRebuilder - if config.BlockMetadataRebuilder.Enable { - blockMetadataRebuilder, err = NewBlockMetadataRebuilder(ctx, config.BlockMetadataRebuilder, arbDb, exec) + var blockMetadataFetcher *BlockMetadataFetcher + if config.BlockMetadataFetcher.Enable { + blockMetadataFetcher, err = NewBlockMetadataFetcher(ctx, config.BlockMetadataFetcher, arbDb, exec) if err != nil { return nil, err } @@ -521,7 +521,7 @@ func createNodeImpl( MaintenanceRunner: maintenanceRunner, DASLifecycleManager: nil, SyncMonitor: syncMonitor, - blockMetadataRebuilder: blockMetadataRebuilder, + blockMetadataFetcher: blockMetadataFetcher, configFetcher: configFetcher, ctx: ctx, }, nil @@ -758,7 +758,7 @@ func createNodeImpl( MaintenanceRunner: maintenanceRunner, DASLifecycleManager: dasLifecycleManager, SyncMonitor: syncMonitor, - blockMetadataRebuilder: blockMetadataRebuilder, + blockMetadataFetcher: blockMetadataFetcher, configFetcher: configFetcher, ctx: ctx, }, nil @@ -942,8 +942,8 @@ func (n *Node) Start(ctx context.Context) error { n.BroadcastClients.Start(ctx) }() } - if n.blockMetadataRebuilder != nil { - n.blockMetadataRebuilder.Start(ctx) + if n.blockMetadataFetcher != nil { + n.blockMetadataFetcher.Start(ctx) } if n.configFetcher != nil { n.configFetcher.Start(ctx) diff --git a/arbnode/schema.go b/arbnode/schema.go index 88d3139f21..acf54c9203 100644 --- a/arbnode/schema.go +++ b/arbnode/schema.go @@ -4,16 +4,16 @@ package arbnode var ( - messagePrefix []byte = []byte("m") // maps a message sequence number to a message - blockHashInputFeedPrefix []byte = []byte("b") // maps a message sequence number to a block hash received through the input feed - blockMetadataInputFeedPrefix []byte = []byte("t") // maps a message sequence number to a blockMetaData byte array received through the input feed - missingBlockMetadataInputFeedPrefix []byte = []byte("xt") // maps a message sequence number whose blockMetaData byte array is missing to nil. Leading "x" implies we are tracking the missing of such a data point - messageResultPrefix []byte = []byte("r") // maps a message sequence number to a message result - legacyDelayedMessagePrefix []byte = []byte("d") // maps a delayed sequence number to an accumulator and a message as serialized on L1 - rlpDelayedMessagePrefix []byte = []byte("e") // maps a delayed sequence number to an accumulator and an RLP encoded message - parentChainBlockNumberPrefix []byte = []byte("p") // maps a delayed sequence number to a parent chain block number - sequencerBatchMetaPrefix []byte = []byte("s") // maps a batch sequence number to BatchMetadata - delayedSequencedPrefix []byte = []byte("a") // maps a delayed message count to the first sequencer batch sequence number with this delayed count + messagePrefix []byte = []byte("m") // maps a message sequence number to a message + blockHashInputFeedPrefix []byte = []byte("b") // maps a message sequence number to a block hash received through the input feed + blockMetadataInputFeedPrefix []byte = []byte("t") // maps a message sequence number to a blockMetaData byte array received through the input feed + missingBlockMetadataInputFeedPrefix []byte = []byte("x") // maps a message sequence number whose blockMetaData byte array is missing to nil + messageResultPrefix []byte = []byte("r") // maps a message sequence number to a message result + legacyDelayedMessagePrefix []byte = []byte("d") // maps a delayed sequence number to an accumulator and a message as serialized on L1 + rlpDelayedMessagePrefix []byte = []byte("e") // maps a delayed sequence number to an accumulator and an RLP encoded message + parentChainBlockNumberPrefix []byte = []byte("p") // maps a delayed sequence number to a parent chain block number + sequencerBatchMetaPrefix []byte = []byte("s") // maps a batch sequence number to BatchMetadata + delayedSequencedPrefix []byte = []byte("a") // maps a delayed message count to the first sequencer batch sequence number with this delayed count messageCountKey []byte = []byte("_messageCount") // contains the current message count delayedMessageCountKey []byte = []byte("_delayedMessageCount") // contains the current delayed message count diff --git a/system_tests/timeboost_test.go b/system_tests/timeboost_test.go index fcf12f1e34..e63a6187da 100644 --- a/system_tests/timeboost_test.go +++ b/system_tests/timeboost_test.go @@ -48,9 +48,8 @@ import ( "github.com/stretchr/testify/require" ) -func blockMetadataInputFeedKey(pos uint64) []byte { +func dbKey(prefix []byte, pos uint64) []byte { var key []byte - prefix := []byte("t") key = append(key, prefix...) data := make([]byte, 8) binary.BigEndian.PutUint64(data, pos) @@ -58,7 +57,9 @@ func blockMetadataInputFeedKey(pos uint64) []byte { return key } -func TestTimeboostBulkBlockMetadataRebuilder(t *testing.T) { +func TestTimeboostBulkBlockMetadataFetcher(t *testing.T) { + t.Parallel() + ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -77,12 +78,13 @@ func TestTimeboostBulkBlockMetadataRebuilder(t *testing.T) { user := builder.L2Info.GetDefaultTransactOpts("User", ctx) var latestL2 uint64 var err error + var lastTx *types.Transaction for i := 0; ; i++ { - builder.L2.TransferBalanceTo(t, "Owner", util.RemapL1Address(user.From), big.NewInt(1e18), builder.L2Info) + lastTx, _ = builder.L2.TransferBalanceTo(t, "Owner", util.RemapL1Address(user.From), big.NewInt(1e18), builder.L2Info) latestL2, err = builder.L2.Client.BlockNumber(ctx) Require(t, err) // Clean BlockMetadata from arbDB so that we can modify it at will - Require(t, arbDb.Delete(blockMetadataInputFeedKey(latestL2))) + Require(t, arbDb.Delete(dbKey([]byte("t"), latestL2))) if latestL2 > uint64(20) { break } @@ -91,44 +93,55 @@ func TestTimeboostBulkBlockMetadataRebuilder(t *testing.T) { for i := 1; i <= int(latestL2); i++ { blockMetadata := []byte{0, uint8(i)} sampleBulkData = append(sampleBulkData, blockMetadata) - arbDb.Put(blockMetadataInputFeedKey(uint64(i)), blockMetadata) + Require(t, arbDb.Put(dbKey([]byte("t"), uint64(i)), blockMetadata)) } - ndcfg := arbnode.ConfigDefaultL1NonSequencerTest() + nodecfg := arbnode.ConfigDefaultL1NonSequencerTest() trackBlockMetadataFrom := uint64(5) - ndcfg.TransactionStreamer.TrackBlockMetadataFrom = trackBlockMetadataFrom + nodecfg.TransactionStreamer.TrackBlockMetadataFrom = trackBlockMetadataFrom newNode, cleanupNewNode := builder.Build2ndNode(t, &SecondNodeParams{ - nodeConfig: ndcfg, + nodeConfig: nodecfg, stackConfig: testhelpers.CreateStackConfigForTest(t.TempDir()), }) defer cleanupNewNode() // Wait for second node to catchup via L1, since L1 doesn't have the blockMetadata, we ensure that messages are tracked with missingBlockMetadataInputFeedPrefix prefix - for { - current, err := newNode.Client.BlockNumber(ctx) - Require(t, err) - if current == latestL2 { - break - } - time.Sleep(time.Second) - } + _, err = WaitForTx(ctx, newNode.Client, lastTx.Hash(), time.Second*5) + Require(t, err) blockMetadataInputFeedPrefix := []byte("t") - missingBlockMetadataInputFeedPrefix := []byte("xt") + missingBlockMetadataInputFeedPrefix := []byte("x") arbDb = newNode.ConsensusNode.ArbDB + // Introduce fragmentation + blocksWithBlockMetadata := []uint64{8, 9, 10, 14, 16} + for _, key := range blocksWithBlockMetadata { + Require(t, arbDb.Put(dbKey([]byte("t"), key), sampleBulkData[key-1])) + Require(t, arbDb.Delete(dbKey([]byte("x"), key))) + } + // Check if all block numbers with missingBlockMetadataInputFeedPrefix are present as keys in arbDB and that no keys with blockMetadataInputFeedPrefix iter := arbDb.NewIterator(blockMetadataInputFeedPrefix, nil) + pos := uint64(0) for iter.Next() { keyBytes := bytes.TrimPrefix(iter.Key(), blockMetadataInputFeedPrefix) - t.Fatalf("unexpected presence of blockMetadata when blocks are synced via L1. msgSeqNum: %d", binary.BigEndian.Uint64(keyBytes)) + if binary.BigEndian.Uint64(keyBytes) != blocksWithBlockMetadata[pos] { + t.Fatalf("unexpected presence of blockMetadata, when blocks are synced via L1. msgSeqNum: %d, expectedMsgSeqNum: %d", binary.BigEndian.Uint64(keyBytes), blocksWithBlockMetadata[pos]) + } + pos++ } iter.Release() iter = arbDb.NewIterator(missingBlockMetadataInputFeedPrefix, nil) - pos := trackBlockMetadataFrom + pos = trackBlockMetadataFrom + i := 0 for iter.Next() { + // Blocks with blockMetadata present shouldn't have the missingBlockMetadataInputFeedPrefix keys present in arbDB + for i < len(blocksWithBlockMetadata) && blocksWithBlockMetadata[i] == pos { + i++ + pos++ + } keyBytes := bytes.TrimPrefix(iter.Key(), missingBlockMetadataInputFeedPrefix) - if pos != binary.BigEndian.Uint64(keyBytes) { + if binary.BigEndian.Uint64(keyBytes) != pos { t.Fatalf("unexpected msgSeqNum with missingBlockMetadataInputFeedPrefix for blockMetadata. Want: %d, Got: %d", pos, binary.BigEndian.Uint64(keyBytes)) } pos++ @@ -139,11 +152,11 @@ func TestTimeboostBulkBlockMetadataRebuilder(t *testing.T) { iter.Release() // Rebuild blockMetadata and cleanup trackers from ArbDB - blockMetadataRebuilder, err := arbnode.NewBlockMetadataRebuilder(ctx, arbnode.BlockMetadataRebuilderConfig{Source: rpcclient.ClientConfig{URL: builder.L2.Stack.HTTPEndpoint()}}, arbDb, newNode.ExecNode) + blockMetadataFetcher, err := arbnode.NewBlockMetadataFetcher(ctx, arbnode.BlockMetadataFetcherConfig{Source: rpcclient.ClientConfig{URL: builder.L2.Stack.HTTPEndpoint()}}, arbDb, newNode.ExecNode) Require(t, err) - blockMetadataRebuilder.Update(ctx) + blockMetadataFetcher.Update(ctx) - // Check if all blockMetadata was synced from bulk BlockMetadata API via the blockMetadataRebuilder and that trackers for missing blockMetadata were cleared + // Check if all blockMetadata was synced from bulk BlockMetadata API via the blockMetadataFetcher and that trackers for missing blockMetadata were cleared iter = arbDb.NewIterator(blockMetadataInputFeedPrefix, nil) pos = trackBlockMetadataFrom for iter.Next() { @@ -152,7 +165,7 @@ func TestTimeboostBulkBlockMetadataRebuilder(t *testing.T) { t.Fatalf("unexpected msgSeqNum with blockMetadataInputFeedPrefix for blockMetadata. Want: %d, Got: %d", pos, binary.BigEndian.Uint64(keyBytes)) } if !bytes.Equal(sampleBulkData[pos-1], iter.Value()) { - t.Fatalf("blockMetadata mismatch. Want: %v, Got: %v", sampleBulkData[pos-1], iter.Value()) + t.Fatalf("blockMetadata mismatch for blockNumber: %d. Want: %v, Got: %v", pos, sampleBulkData[pos-1], iter.Value()) } pos++ } @@ -189,7 +202,7 @@ func TestTimeboostBulkBlockMetadataAPI(t *testing.T) { latestL2, err := builder.L2.Client.BlockNumber(ctx) Require(t, err) // Clean BlockMetadata from arbDB so that we can modify it at will - Require(t, arbDb.Delete(blockMetadataInputFeedKey(latestL2))) + Require(t, arbDb.Delete(dbKey([]byte("t"), latestL2))) if latestL2 > uint64(end)+10 { break } @@ -201,7 +214,7 @@ func TestTimeboostBulkBlockMetadataAPI(t *testing.T) { RawMetadata: []byte{0, uint8(i)}, } sampleBulkData = append(sampleBulkData, sampleData) - arbDb.Put(blockMetadataInputFeedKey(sampleData.BlockNumber), sampleData.RawMetadata) + Require(t, arbDb.Put(dbKey([]byte("t"), sampleData.BlockNumber), sampleData.RawMetadata)) } l2rpc := builder.L2.Stack.Attach() @@ -223,7 +236,7 @@ func TestTimeboostBulkBlockMetadataAPI(t *testing.T) { // Test that without cache the result returned is always in sync with ArbDB sampleBulkData[0].RawMetadata = []byte{1, 11} - arbDb.Put(blockMetadataInputFeedKey(1), sampleBulkData[0].RawMetadata) + Require(t, arbDb.Put(dbKey([]byte("t"), 1), sampleBulkData[0].RawMetadata)) err = l2rpc.CallContext(ctx, &result, "arb_getRawBlockMetadata", rpc.BlockNumber(1), rpc.BlockNumber(1)) Require(t, err) @@ -244,7 +257,7 @@ func TestTimeboostBulkBlockMetadataAPI(t *testing.T) { arbDb = builder.L2.ConsensusNode.ArbDB updatedBlockMetadata := []byte{2, 12} - arbDb.Put(blockMetadataInputFeedKey(1), updatedBlockMetadata) + Require(t, arbDb.Put(dbKey([]byte("t"), 1), updatedBlockMetadata)) err = l2rpc.CallContext(ctx, &result, "arb_getRawBlockMetadata", rpc.BlockNumber(1), rpc.BlockNumber(1)) Require(t, err) diff --git a/util/common.go b/util/common.go new file mode 100644 index 0000000000..ad6f6a875b --- /dev/null +++ b/util/common.go @@ -0,0 +1,9 @@ +package util + +func ArrayToMap[T comparable](arr []T) map[T]struct{} { + ret := make(map[T]struct{}) + for _, elem := range arr { + ret[elem] = struct{}{} + } + return ret +} From a5f248db7ae67a7d16637aa379380966d3d02792 Mon Sep 17 00:00:00 2001 From: Ganesh Vanahalli Date: Wed, 13 Nov 2024 11:11:19 +0530 Subject: [PATCH 08/14] make flag description clearer --- arbnode/transaction_streamer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arbnode/transaction_streamer.go b/arbnode/transaction_streamer.go index d2e238ec7a..4a06223d7a 100644 --- a/arbnode/transaction_streamer.go +++ b/arbnode/transaction_streamer.go @@ -99,7 +99,7 @@ func TransactionStreamerConfigAddOptions(prefix string, f *flag.FlagSet) { f.Int(prefix+".max-broadcaster-queue-size", DefaultTransactionStreamerConfig.MaxBroadcasterQueueSize, "maximum cache of pending broadcaster messages") f.Int64(prefix+".max-reorg-resequence-depth", DefaultTransactionStreamerConfig.MaxReorgResequenceDepth, "maximum number of messages to attempt to resequence on reorg (0 = never resequence, -1 = always resequence)") f.Duration(prefix+".execute-message-loop-delay", DefaultTransactionStreamerConfig.ExecuteMessageLoopDelay, "delay when polling calls to execute messages") - f.Uint64(prefix+".track-block-metadata-from", DefaultTransactionStreamerConfig.TrackBlockMetadataFrom, "block number starting from which the missing of blockmetadata is being tracked in the local disk. Setting to zero (default value) disables this") + f.Uint64(prefix+".track-block-metadata-from", DefaultTransactionStreamerConfig.TrackBlockMetadataFrom, "this is the block number starting from which missing of blockmetadata is being tracked in the local disk. Setting to zero (default value) disables this") } func NewTransactionStreamer( From 71f9c5abaa09a489899e6252861c4f5c8f3fd889 Mon Sep 17 00:00:00 2001 From: Ganesh Vanahalli Date: Mon, 18 Nov 2024 12:50:50 +0530 Subject: [PATCH 09/14] address PR comments --- arbnode/blockmetadata.go | 4 ++-- util/common.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/arbnode/blockmetadata.go b/arbnode/blockmetadata.go index 82928d9fa7..bd2bd1ad4b 100644 --- a/arbnode/blockmetadata.go +++ b/arbnode/blockmetadata.go @@ -34,7 +34,7 @@ var DefaultBlockMetadataFetcherConfig = BlockMetadataFetcherConfig{ } func BlockMetadataFetcherConfigAddOptions(prefix string, f *pflag.FlagSet) { - f.Bool(prefix+".enable", DefaultBlockMetadataFetcherConfig.Enable, "enable syncing blockMetadata using a bulk blockMetadata api") + f.Bool(prefix+".enable", DefaultBlockMetadataFetcherConfig.Enable, "enable syncing blockMetadata using a bulk blockMetadata api. If the source doesn't have the missing blockMetadata, we keep retyring in every sync-interval (default=5mins) duration") rpcclient.RPCClientAddOptions(prefix+".source", f, &DefaultBlockMetadataFetcherConfig.Source) f.Duration(prefix+".sync-interval", DefaultBlockMetadataFetcherConfig.SyncInterval, "interval at which blockMetadata are synced regularly") f.Uint64(prefix+".api-blocks-limit", DefaultBlockMetadataFetcherConfig.APIBlocksLimit, "maximum number of blocks allowed to be queried for blockMetadata per arb_getRawBlockMetadata query.\n"+ @@ -73,7 +73,7 @@ func (b *BlockMetadataFetcher) fetch(ctx context.Context, fromBlock, toBlock uin func (b *BlockMetadataFetcher) persistBlockMetadata(query []uint64, result []gethexec.NumberAndBlockMetadata) error { batch := b.db.NewBatch() - queryMap := util.ArrayToMap(query) + queryMap := util.ArrayToSet(query) for _, elem := range result { pos, err := b.exec.BlockNumberToMessageIndex(elem.BlockNumber) if err != nil { diff --git a/util/common.go b/util/common.go index ad6f6a875b..fc71e4a704 100644 --- a/util/common.go +++ b/util/common.go @@ -1,6 +1,6 @@ package util -func ArrayToMap[T comparable](arr []T) map[T]struct{} { +func ArrayToSet[T comparable](arr []T) map[T]struct{} { ret := make(map[T]struct{}) for _, elem := range arr { ret[elem] = struct{}{} From 0178eb7c7291ac87704395c8a1e3f836a7b565d3 Mon Sep 17 00:00:00 2001 From: Ganesh Vanahalli Date: Thu, 21 Nov 2024 11:00:10 +0530 Subject: [PATCH 10/14] clear blockMetadata if there's a mismatch between feed's blockhash and locally computed hash --- arbnode/transaction_streamer.go | 28 ++++++++++++++++++++++------ system_tests/timeboost_test.go | 1 + 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/arbnode/transaction_streamer.go b/arbnode/transaction_streamer.go index 4a06223d7a..53f1910e92 100644 --- a/arbnode/transaction_streamer.go +++ b/arbnode/transaction_streamer.go @@ -1164,17 +1164,33 @@ func (s *TransactionStreamer) ResultAtCount(count arbutil.MessageIndex) (*execut return msgResult, nil } -func (s *TransactionStreamer) checkResult(msgResult *execution.MessageResult, expectedBlockHash *common.Hash) { - if expectedBlockHash == nil { +func (s *TransactionStreamer) checkResult(pos arbutil.MessageIndex, msgResult *execution.MessageResult, msgAndBlockInfo *arbostypes.MessageWithMetadataAndBlockInfo) { + if msgAndBlockInfo.BlockHash == nil { return } - if msgResult.BlockHash != *expectedBlockHash { + if msgResult.BlockHash != *msgAndBlockInfo.BlockHash { log.Error( BlockHashMismatchLogMsg, - "expected", expectedBlockHash, + "expected", msgAndBlockInfo.BlockHash, "actual", msgResult.BlockHash, ) - return + // Try deleting the existing blockMetadata for this block in arbDB and set it as missing + if msgAndBlockInfo.BlockMetadata != nil { + batch := s.db.NewBatch() + if err := batch.Delete(dbKey(blockMetadataInputFeedPrefix, uint64(pos))); err != nil { + log.Error("error deleting blockMetadata of block whose BlockHash from feed doesn't match locally computed hash", "msgSeqNum", pos, "err", err) + return + } + if s.trackBlockMetadataFrom != 0 && pos >= s.trackBlockMetadataFrom { + if err := batch.Put(dbKey(missingBlockMetadataInputFeedPrefix, uint64(pos)), nil); err != nil { + log.Error("error marking deleted blockMetadata as missing in arbDB for a block whose BlockHash from feed doesn't match locally computed hash", "msgSeqNum", pos, "err", err) + return + } + } + if err := batch.Write(); err != nil { + log.Error("error writing batch that deletes blockMetadata of the block whose BlockHash from feed doesn't match locally computed hash", "msgSeqNum", pos, "err", err) + } + } } } @@ -1242,7 +1258,7 @@ func (s *TransactionStreamer) ExecuteNextMsg(ctx context.Context, exec execution } // we just log the error but not update the value in db itself with msgResult.BlockHash? and instead forward the new block hash - s.checkResult(msgResult, msgAndBlockInfo.BlockHash) + s.checkResult(pos, msgResult, msgAndBlockInfo) batch := s.db.NewBatch() err = s.storeResult(pos, *msgResult, batch) diff --git a/system_tests/timeboost_test.go b/system_tests/timeboost_test.go index e63a6187da..a3bec57b6a 100644 --- a/system_tests/timeboost_test.go +++ b/system_tests/timeboost_test.go @@ -563,6 +563,7 @@ func setupExpressLaneAuction( ExpressLaneAdvantage: time.Second * 5, SequencerHTTPEndpoint: fmt.Sprintf("http://localhost:%d", seqPort), } + builderSeq.nodeConfig.TransactionStreamer.TrackBlockMetadataFrom = 1 cleanupSeq := builderSeq.Build(t) seqInfo, seqNode, seqClient := builderSeq.L2Info, builderSeq.L2.ConsensusNode, builderSeq.L2.Client From 411b1cd87d3ac35882d3a297c42352efbe18f59e Mon Sep 17 00:00:00 2001 From: Ganesh Vanahalli Date: Fri, 27 Dec 2024 12:38:11 -0600 Subject: [PATCH 11/14] fix lint errors --- arbnode/blockmetadata.go | 2 ++ system_tests/timeboost_test.go | 2 ++ 2 files changed, 4 insertions(+) diff --git a/arbnode/blockmetadata.go b/arbnode/blockmetadata.go index bd2bd1ad4b..a800f6a8e6 100644 --- a/arbnode/blockmetadata.go +++ b/arbnode/blockmetadata.go @@ -11,6 +11,7 @@ import ( "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rpc" + "github.com/offchainlabs/nitro/arbutil" "github.com/offchainlabs/nitro/execution" "github.com/offchainlabs/nitro/execution/gethexec" @@ -64,6 +65,7 @@ func NewBlockMetadataFetcher(ctx context.Context, c BlockMetadataFetcherConfig, func (b *BlockMetadataFetcher) fetch(ctx context.Context, fromBlock, toBlock uint64) ([]gethexec.NumberAndBlockMetadata, error) { var result []gethexec.NumberAndBlockMetadata + // #nosec G115 err := b.client.CallContext(ctx, &result, "arb_getRawBlockMetadata", rpc.BlockNumber(fromBlock), rpc.BlockNumber(toBlock)) if err != nil { return nil, err diff --git a/system_tests/timeboost_test.go b/system_tests/timeboost_test.go index f9c077cb1e..c1060f45b4 100644 --- a/system_tests/timeboost_test.go +++ b/system_tests/timeboost_test.go @@ -93,8 +93,10 @@ func TestTimeboostBulkBlockMetadataFetcher(t *testing.T) { } var sampleBulkData []arbostypes.BlockMetadata for i := 1; i <= int(latestL2); i++ { + // #nosec G115 blockMetadata := []byte{0, uint8(i)} sampleBulkData = append(sampleBulkData, blockMetadata) + // #nosec G115 Require(t, arbDb.Put(dbKey([]byte("t"), uint64(i)), blockMetadata)) } From 067b687b83907112df9202a7bf42f3f79e84da4b Mon Sep 17 00:00:00 2001 From: Ganesh Vanahalli Date: Fri, 27 Dec 2024 13:29:51 -0600 Subject: [PATCH 12/14] fix minor issues with config --- arbnode/blockmetadata.go | 2 +- arbnode/node.go | 7 ++++--- cmd/nitro/config_test.go | 8 ++++---- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/arbnode/blockmetadata.go b/arbnode/blockmetadata.go index a800f6a8e6..96e02e07b8 100644 --- a/arbnode/blockmetadata.go +++ b/arbnode/blockmetadata.go @@ -22,7 +22,7 @@ import ( type BlockMetadataFetcherConfig struct { Enable bool `koanf:"enable"` - Source rpcclient.ClientConfig `koanf:"source"` + Source rpcclient.ClientConfig `koanf:"source" reload:"hot"` SyncInterval time.Duration `koanf:"sync-interval"` APIBlocksLimit uint64 `koanf:"api-blocks-limit"` } diff --git a/arbnode/node.go b/arbnode/node.go index 2baf0237eb..146b37d30e 100644 --- a/arbnode/node.go +++ b/arbnode/node.go @@ -137,10 +137,10 @@ func (c *Config) Validate() error { return err } if c.Sequencer && c.TransactionStreamer.TrackBlockMetadataFrom == 0 { - return errors.New("when sequencer is enabled track-missing-block-metadata-from should be set as well") + return errors.New("when sequencer is enabled track-block-metadata-from should be set as well") } if c.TransactionStreamer.TrackBlockMetadataFrom != 0 && !c.BlockMetadataFetcher.Enable { - log.Warn("track-missing-block-metadata-from is set but blockMetadata fetcher is not enabled") + log.Warn("track-block-metadata-from is set but blockMetadata fetcher is not enabled") } return nil } @@ -172,7 +172,7 @@ func ConfigAddOptions(prefix string, f *flag.FlagSet, feedInputEnable bool, feed DangerousConfigAddOptions(prefix+".dangerous", f) TransactionStreamerConfigAddOptions(prefix+".transaction-streamer", f) MaintenanceConfigAddOptions(prefix+".maintenance", f) - BlockMetadataFetcherConfigAddOptions(prefix+"block-metadata-fetcher", f) + BlockMetadataFetcherConfigAddOptions(prefix+".block-metadata-fetcher", f) } var ConfigDefault = Config{ @@ -204,6 +204,7 @@ func ConfigDefaultL1Test() *Config { config.SeqCoordinator = TestSeqCoordinatorConfig config.Sequencer = true config.Dangerous.NoSequencerCoordinator = true + config.TransactionStreamer.TrackBlockMetadataFrom = 1 return config } diff --git a/cmd/nitro/config_test.go b/cmd/nitro/config_test.go index ef41d704f1..9e7cb87524 100644 --- a/cmd/nitro/config_test.go +++ b/cmd/nitro/config_test.go @@ -42,7 +42,7 @@ func TestEmptyCliConfig(t *testing.T) { } func TestSeqConfig(t *testing.T) { - args := strings.Split("--persistent.chain /tmp/data --init.dev-init --node.parent-chain-reader.enable=false --parent-chain.id 5 --chain.id 421613 --node.batch-poster.parent-chain-wallet.pathname /l1keystore --node.batch-poster.parent-chain-wallet.password passphrase --http.addr 0.0.0.0 --ws.addr 0.0.0.0 --node.sequencer --execution.sequencer.enable --node.feed.output.enable --node.feed.output.port 9642", " ") + args := strings.Split("--persistent.chain /tmp/data --init.dev-init --node.parent-chain-reader.enable=false --parent-chain.id 5 --chain.id 421613 --node.batch-poster.parent-chain-wallet.pathname /l1keystore --node.batch-poster.parent-chain-wallet.password passphrase --http.addr 0.0.0.0 --ws.addr 0.0.0.0 --node.sequencer --execution.sequencer.enable --node.feed.output.enable --node.feed.output.port 9642 --node.transaction-streamer.track-block-metadata-from=10", " ") _, _, err := ParseNode(context.Background(), args) Require(t, err) } @@ -79,7 +79,7 @@ func TestInvalidArchiveConfig(t *testing.T) { } func TestAggregatorConfig(t *testing.T) { - args := strings.Split("--persistent.chain /tmp/data --init.dev-init --node.parent-chain-reader.enable=false --parent-chain.id 5 --chain.id 421613 --node.batch-poster.parent-chain-wallet.pathname /l1keystore --node.batch-poster.parent-chain-wallet.password passphrase --http.addr 0.0.0.0 --ws.addr 0.0.0.0 --node.sequencer --execution.sequencer.enable --node.feed.output.enable --node.feed.output.port 9642 --node.data-availability.enable --node.data-availability.rpc-aggregator.backends [{\"url\":\"http://localhost:8547\",\"pubkey\":\"abc==\"}]", " ") + args := strings.Split("--persistent.chain /tmp/data --init.dev-init --node.parent-chain-reader.enable=false --parent-chain.id 5 --chain.id 421613 --node.batch-poster.parent-chain-wallet.pathname /l1keystore --node.batch-poster.parent-chain-wallet.password passphrase --http.addr 0.0.0.0 --ws.addr 0.0.0.0 --node.sequencer --execution.sequencer.enable --node.feed.output.enable --node.feed.output.port 9642 --node.data-availability.enable --node.data-availability.rpc-aggregator.backends [{\"url\":\"http://localhost:8547\",\"pubkey\":\"abc==\"}] --node.transaction-streamer.track-block-metadata-from=10", " ") _, _, err := ParseNode(context.Background(), args) Require(t, err) } @@ -142,7 +142,7 @@ func TestLiveNodeConfig(t *testing.T) { jsonConfig := "{\"chain\":{\"id\":421613}}" Require(t, WriteToConfigFile(configFile, jsonConfig)) - args := strings.Split("--file-logging.enable=false --persistent.chain /tmp/data --init.dev-init --node.parent-chain-reader.enable=false --parent-chain.id 5 --node.batch-poster.parent-chain-wallet.pathname /l1keystore --node.batch-poster.parent-chain-wallet.password passphrase --http.addr 0.0.0.0 --ws.addr 0.0.0.0 --node.sequencer --execution.sequencer.enable --node.feed.output.enable --node.feed.output.port 9642", " ") + args := strings.Split("--file-logging.enable=false --persistent.chain /tmp/data --init.dev-init --node.parent-chain-reader.enable=false --parent-chain.id 5 --node.batch-poster.parent-chain-wallet.pathname /l1keystore --node.batch-poster.parent-chain-wallet.password passphrase --http.addr 0.0.0.0 --ws.addr 0.0.0.0 --node.sequencer --execution.sequencer.enable --node.feed.output.enable --node.feed.output.port 9642 --node.transaction-streamer.track-block-metadata-from=10", " ") args = append(args, []string{"--conf.file", configFile}...) config, _, err := ParseNode(context.Background(), args) Require(t, err) @@ -223,7 +223,7 @@ func TestPeriodicReloadOfLiveNodeConfig(t *testing.T) { jsonConfig := "{\"conf\":{\"reload-interval\":\"20ms\"}}" Require(t, WriteToConfigFile(configFile, jsonConfig)) - args := strings.Split("--persistent.chain /tmp/data --init.dev-init --node.parent-chain-reader.enable=false --parent-chain.id 5 --chain.id 421613 --node.batch-poster.parent-chain-wallet.pathname /l1keystore --node.batch-poster.parent-chain-wallet.password passphrase --http.addr 0.0.0.0 --ws.addr 0.0.0.0 --node.sequencer --execution.sequencer.enable --node.feed.output.enable --node.feed.output.port 9642", " ") + args := strings.Split("--persistent.chain /tmp/data --init.dev-init --node.parent-chain-reader.enable=false --parent-chain.id 5 --chain.id 421613 --node.batch-poster.parent-chain-wallet.pathname /l1keystore --node.batch-poster.parent-chain-wallet.password passphrase --http.addr 0.0.0.0 --ws.addr 0.0.0.0 --node.sequencer --execution.sequencer.enable --node.feed.output.enable --node.feed.output.port 9642 --node.transaction-streamer.track-block-metadata-from=10", " ") args = append(args, []string{"--conf.file", configFile}...) config, _, err := ParseNode(context.Background(), args) Require(t, err) From 8d759a9717507e5dbff83d58fb0fe13cb6ccf192 Mon Sep 17 00:00:00 2001 From: Ganesh Vanahalli Date: Fri, 27 Dec 2024 15:23:22 -0600 Subject: [PATCH 13/14] add required default flag to --dev and nitro testnode --- cmd/util/confighelpers/configuration.go | 1 + nitro-testnode | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/cmd/util/confighelpers/configuration.go b/cmd/util/confighelpers/configuration.go index 8c4ef2a70b..6a139e4851 100644 --- a/cmd/util/confighelpers/configuration.go +++ b/cmd/util/confighelpers/configuration.go @@ -210,6 +210,7 @@ func devFlagArgs() []string { "--http.port", "8547", "--http.addr", "127.0.0.1", "--http.api=net,web3,eth,arb,arbdebug,debug", + "--node.transaction-streamer.track-block-metadata-from=1", } return args } diff --git a/nitro-testnode b/nitro-testnode index c177f28234..15a2bfea70 160000 --- a/nitro-testnode +++ b/nitro-testnode @@ -1 +1 @@ -Subproject commit c177f282340285bcdae2d6a784547e2bb8b97498 +Subproject commit 15a2bfea7030377771c5d2749f24afc6b48c5deb From 1d7f045983ee8b305cb97f572879dc9d37bd30f4 Mon Sep 17 00:00:00 2001 From: Ganesh Vanahalli Date: Mon, 30 Dec 2024 10:21:37 -0600 Subject: [PATCH 14/14] merge upstream --- arbnode/node.go | 4 +- arbnode/seq_coordinator.go | 11 +- arbnode/seq_coordinator_test.go | 4 +- arbnode/transaction_streamer.go | 8 +- arbos/arbostypes/messagewithmeta.go | 20 +-- broadcaster/broadcaster.go | 4 +- broadcaster/message/message.go | 2 +- .../message/message_blockmetadata_test.go | 4 +- execution/gethexec/blockmetadata.go | 12 +- execution/gethexec/executionengine.go | 6 +- execution/gethexec/sync_monitor.go | 15 +++ execution/interface.go | 4 +- go-ethereum | 2 +- system_tests/timeboost_test.go | 123 +++++++++++++++++- 14 files changed, 164 insertions(+), 55 deletions(-) diff --git a/arbnode/node.go b/arbnode/node.go index 146b37d30e..8bd63bec11 100644 --- a/arbnode/node.go +++ b/arbnode/node.go @@ -1072,7 +1072,7 @@ func (n *Node) GetFinalizedMsgCount(ctx context.Context) (arbutil.MessageIndex, return n.InboxReader.GetFinalizedMsgCount(ctx) } -func (n *Node) WriteMessageFromSequencer(pos arbutil.MessageIndex, msgWithMeta arbostypes.MessageWithMetadata, msgResult execution.MessageResult, blockMetadata arbostypes.BlockMetadata) error { +func (n *Node) WriteMessageFromSequencer(pos arbutil.MessageIndex, msgWithMeta arbostypes.MessageWithMetadata, msgResult execution.MessageResult, blockMetadata common.BlockMetadata) error { return n.TxStreamer.WriteMessageFromSequencer(pos, msgWithMeta, msgResult, blockMetadata) } @@ -1087,6 +1087,6 @@ func (n *Node) ValidatedMessageCount() (arbutil.MessageIndex, error) { return n.BlockValidator.GetValidated(), nil } -func (n *Node) BlockMetadataAtCount(count arbutil.MessageIndex) (arbostypes.BlockMetadata, error) { +func (n *Node) BlockMetadataAtCount(count arbutil.MessageIndex) (common.BlockMetadata, error) { return n.TxStreamer.BlockMetadataAtCount(count) } diff --git a/arbnode/seq_coordinator.go b/arbnode/seq_coordinator.go index d898ac25dd..c0065939ed 100644 --- a/arbnode/seq_coordinator.go +++ b/arbnode/seq_coordinator.go @@ -17,6 +17,7 @@ import ( "github.com/redis/go-redis/v9" flag "github.com/spf13/pflag" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" @@ -270,7 +271,7 @@ func (c *SeqCoordinator) signedBytesToMsgCount(ctx context.Context, data []byte) } // Acquires or refreshes the chosen one lockout and optionally writes a message into redis atomically. -func (c *SeqCoordinator) acquireLockoutAndWriteMessage(ctx context.Context, msgCountExpected, msgCountToWrite arbutil.MessageIndex, lastmsg *arbostypes.MessageWithMetadata, blockMetadata arbostypes.BlockMetadata) error { +func (c *SeqCoordinator) acquireLockoutAndWriteMessage(ctx context.Context, msgCountExpected, msgCountToWrite arbutil.MessageIndex, lastmsg *arbostypes.MessageWithMetadata, blockMetadata common.BlockMetadata) error { var messageData *string var messageSigData *string if lastmsg != nil { @@ -600,7 +601,7 @@ func (c *SeqCoordinator) deleteFinalizedMsgsFromRedis(ctx context.Context, final return nil } -func (c *SeqCoordinator) blockMetadataAt(ctx context.Context, pos arbutil.MessageIndex) (arbostypes.BlockMetadata, error) { +func (c *SeqCoordinator) blockMetadataAt(ctx context.Context, pos arbutil.MessageIndex) (common.BlockMetadata, error) { blockMetadataStr, err := c.RedisCoordinator().Client.Get(ctx, redisutil.BlockMetadataKeyFor(pos)).Result() if err != nil { if errors.Is(err, redis.Nil) { @@ -608,7 +609,7 @@ func (c *SeqCoordinator) blockMetadataAt(ctx context.Context, pos arbutil.Messag } return nil, err } - return arbostypes.BlockMetadata(blockMetadataStr), nil + return common.BlockMetadata(blockMetadataStr), nil } func (c *SeqCoordinator) update(ctx context.Context) time.Duration { @@ -675,7 +676,7 @@ func (c *SeqCoordinator) update(ctx context.Context) time.Duration { log.Info("coordinator caught up to prev redis coordinator", "msgcount", localMsgCount, "prevMsgCount", c.prevRedisMessageCount) } var messages []arbostypes.MessageWithMetadata - var blockMetadataArr []arbostypes.BlockMetadata + var blockMetadataArr []common.BlockMetadata msgToRead := localMsgCount var msgReadErr error for msgToRead < readUntil && localMsgCount >= remoteFinalizedMsgCount { @@ -991,7 +992,7 @@ func (c *SeqCoordinator) CurrentlyChosen() bool { return time.Now().Before(atomicTimeRead(&c.lockoutUntil)) } -func (c *SeqCoordinator) SequencingMessage(pos arbutil.MessageIndex, msg *arbostypes.MessageWithMetadata, blockMetadata arbostypes.BlockMetadata) error { +func (c *SeqCoordinator) SequencingMessage(pos arbutil.MessageIndex, msg *arbostypes.MessageWithMetadata, blockMetadata common.BlockMetadata) error { if !c.CurrentlyChosen() { return fmt.Errorf("%w: not main sequencer", execution.ErrRetrySequencer) } diff --git a/arbnode/seq_coordinator_test.go b/arbnode/seq_coordinator_test.go index d459f59761..e308247d34 100644 --- a/arbnode/seq_coordinator_test.go +++ b/arbnode/seq_coordinator_test.go @@ -13,6 +13,8 @@ import ( "testing" "time" + "github.com/ethereum/go-ethereum/common" + "github.com/offchainlabs/nitro/arbos/arbostypes" "github.com/offchainlabs/nitro/arbutil" "github.com/offchainlabs/nitro/util/redisutil" @@ -279,7 +281,7 @@ func TestSeqCoordinatorAddsBlockMetadata(t *testing.T) { } pos := arbutil.MessageIndex(1) - blockMetadataWant := arbostypes.BlockMetadata{0, 4} + blockMetadataWant := common.BlockMetadata{0, 4} Require(t, coordinator.acquireLockoutAndWriteMessage(ctx, pos, pos+1, &arbostypes.EmptyTestMessageWithMetadata, blockMetadataWant)) blockMetadataGot, err := coordinator.blockMetadataAt(ctx, pos) Require(t, err) diff --git a/arbnode/transaction_streamer.go b/arbnode/transaction_streamer.go index e8e93ea4fd..bfc5d952fe 100644 --- a/arbnode/transaction_streamer.go +++ b/arbnode/transaction_streamer.go @@ -538,7 +538,7 @@ func (s *TransactionStreamer) GetProcessedMessageCount() (arbutil.MessageIndex, return msgCount, nil } -func (s *TransactionStreamer) AddMessages(pos arbutil.MessageIndex, messagesAreConfirmed bool, messages []arbostypes.MessageWithMetadata, blockMetadataArr []arbostypes.BlockMetadata) error { +func (s *TransactionStreamer) AddMessages(pos arbutil.MessageIndex, messagesAreConfirmed bool, messages []arbostypes.MessageWithMetadata, blockMetadataArr []common.BlockMetadata) error { return s.AddMessagesAndEndBatch(pos, messagesAreConfirmed, messages, blockMetadataArr, nil) } @@ -696,7 +696,7 @@ func endBatch(batch ethdb.Batch) error { return batch.Write() } -func (s *TransactionStreamer) AddMessagesAndEndBatch(pos arbutil.MessageIndex, messagesAreConfirmed bool, messages []arbostypes.MessageWithMetadata, blockMetadataArr []arbostypes.BlockMetadata, batch ethdb.Batch) error { +func (s *TransactionStreamer) AddMessagesAndEndBatch(pos arbutil.MessageIndex, messagesAreConfirmed bool, messages []arbostypes.MessageWithMetadata, blockMetadataArr []common.BlockMetadata, batch ethdb.Batch) error { messagesWithBlockInfo := make([]arbostypes.MessageWithMetadataAndBlockInfo, 0, len(messages)) for _, message := range messages { messagesWithBlockInfo = append(messagesWithBlockInfo, arbostypes.MessageWithMetadataAndBlockInfo{ @@ -992,7 +992,7 @@ func (s *TransactionStreamer) WriteMessageFromSequencer( pos arbutil.MessageIndex, msgWithMeta arbostypes.MessageWithMetadata, msgResult execution.MessageResult, - blockMetadata arbostypes.BlockMetadata, + blockMetadata common.BlockMetadata, ) error { if err := s.ExpectChosenSequencer(); err != nil { return err @@ -1127,7 +1127,7 @@ func (s *TransactionStreamer) writeMessages(pos arbutil.MessageIndex, messages [ return nil } -func (s *TransactionStreamer) BlockMetadataAtCount(count arbutil.MessageIndex) (arbostypes.BlockMetadata, error) { +func (s *TransactionStreamer) BlockMetadataAtCount(count arbutil.MessageIndex) (common.BlockMetadata, error) { if count == 0 { return nil, nil } diff --git a/arbos/arbostypes/messagewithmeta.go b/arbos/arbostypes/messagewithmeta.go index d42db9a643..c32c3cd795 100644 --- a/arbos/arbostypes/messagewithmeta.go +++ b/arbos/arbostypes/messagewithmeta.go @@ -3,7 +3,6 @@ package arbostypes import ( "context" "encoding/binary" - "errors" "fmt" "github.com/ethereum/go-ethereum/common" @@ -20,12 +19,10 @@ type MessageWithMetadata struct { DelayedMessagesRead uint64 `json:"delayedMessagesRead"` } -type BlockMetadata []byte - type MessageWithMetadataAndBlockInfo struct { MessageWithMeta MessageWithMetadata BlockHash *common.Hash - BlockMetadata BlockMetadata + BlockMetadata common.BlockMetadata } var EmptyTestMessageWithMetadata = MessageWithMetadata{ @@ -37,21 +34,6 @@ var TestMessageWithMetadataAndRequestId = MessageWithMetadata{ Message: &TestIncomingMessageWithRequestId, } -// IsTxTimeboosted given a tx's index in the block returns whether the tx was timeboosted or not -func (b BlockMetadata) IsTxTimeboosted(txIndex int) (bool, error) { - if len(b) == 0 { - return false, errors.New("blockMetadata is not set") - } - if txIndex < 0 { - return false, fmt.Errorf("invalid transaction index- %d, should be positive", txIndex) - } - maxTxCount := (len(b) - 1) * 8 - if txIndex >= maxTxCount { - return false, nil - } - return b[1+(txIndex/8)]&(1<<(txIndex%8)) != 0, nil -} - func (m *MessageWithMetadata) Hash(sequenceNumber arbutil.MessageIndex, chainId uint64) (common.Hash, error) { serializedExtraData := make([]byte, 24) binary.BigEndian.PutUint64(serializedExtraData[:8], uint64(sequenceNumber)) diff --git a/broadcaster/broadcaster.go b/broadcaster/broadcaster.go index 76e37b879c..2c4ffd96ec 100644 --- a/broadcaster/broadcaster.go +++ b/broadcaster/broadcaster.go @@ -43,7 +43,7 @@ func (b *Broadcaster) NewBroadcastFeedMessage( message arbostypes.MessageWithMetadata, sequenceNumber arbutil.MessageIndex, blockHash *common.Hash, - blockMetadata arbostypes.BlockMetadata, + blockMetadata common.BlockMetadata, ) (*m.BroadcastFeedMessage, error) { var messageSignature []byte if b.dataSigner != nil { @@ -70,7 +70,7 @@ func (b *Broadcaster) BroadcastSingle( msg arbostypes.MessageWithMetadata, seq arbutil.MessageIndex, blockHash *common.Hash, - blockMetadata arbostypes.BlockMetadata, + blockMetadata common.BlockMetadata, ) (err error) { defer func() { if r := recover(); r != nil { diff --git a/broadcaster/message/message.go b/broadcaster/message/message.go index b0f439e614..3790fe8dae 100644 --- a/broadcaster/message/message.go +++ b/broadcaster/message/message.go @@ -38,7 +38,7 @@ type BroadcastFeedMessage struct { Message arbostypes.MessageWithMetadata `json:"message"` BlockHash *common.Hash `json:"blockHash,omitempty"` Signature []byte `json:"signature"` - BlockMetadata arbostypes.BlockMetadata `json:"blockMetadata,omitempty"` + BlockMetadata common.BlockMetadata `json:"blockMetadata,omitempty"` CumulativeSumMsgSize uint64 `json:"-"` } diff --git a/broadcaster/message/message_blockmetadata_test.go b/broadcaster/message/message_blockmetadata_test.go index 2f99c2d5b6..5e72e6e203 100644 --- a/broadcaster/message/message_blockmetadata_test.go +++ b/broadcaster/message/message_blockmetadata_test.go @@ -3,14 +3,14 @@ package message import ( "testing" - "github.com/offchainlabs/nitro/arbos/arbostypes" + "github.com/ethereum/go-ethereum/common" ) func TestTimeboostedInDifferentScenarios(t *testing.T) { t.Parallel() for _, tc := range []struct { name string - blockMetadata arbostypes.BlockMetadata + blockMetadata common.BlockMetadata txs []bool // Array representing whether the tx is timeboosted or not. First tx is always false as its an arbitrum internal tx }{ { diff --git a/execution/gethexec/blockmetadata.go b/execution/gethexec/blockmetadata.go index f7eb540275..26b1ae2526 100644 --- a/execution/gethexec/blockmetadata.go +++ b/execution/gethexec/blockmetadata.go @@ -5,12 +5,12 @@ import ( "errors" "fmt" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/common/lru" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/rpc" - "github.com/offchainlabs/nitro/arbos/arbostypes" "github.com/offchainlabs/nitro/arbutil" "github.com/offchainlabs/nitro/util/stopwaiter" ) @@ -18,7 +18,7 @@ import ( var ErrBlockMetadataApiBlocksLimitExceeded = errors.New("number of blocks requested for blockMetadata exceeded") type BlockMetadataFetcher interface { - BlockMetadataAtCount(count arbutil.MessageIndex) (arbostypes.BlockMetadata, error) + BlockMetadataAtCount(count arbutil.MessageIndex) (common.BlockMetadata, error) BlockNumberToMessageIndex(blockNum uint64) (arbutil.MessageIndex, error) MessageIndexToBlockNumber(messageNum arbutil.MessageIndex) uint64 SetReorgEventsNotifier(reorgEventsNotifier chan struct{}) @@ -30,14 +30,14 @@ type BulkBlockMetadataFetcher struct { fetcher BlockMetadataFetcher reorgDetector chan struct{} blocksLimit uint64 - cache *lru.SizeConstrainedCache[arbutil.MessageIndex, arbostypes.BlockMetadata] + cache *lru.SizeConstrainedCache[arbutil.MessageIndex, common.BlockMetadata] } func NewBulkBlockMetadataFetcher(bc *core.BlockChain, fetcher BlockMetadataFetcher, cacheSize, blocksLimit uint64) *BulkBlockMetadataFetcher { - var cache *lru.SizeConstrainedCache[arbutil.MessageIndex, arbostypes.BlockMetadata] + var cache *lru.SizeConstrainedCache[arbutil.MessageIndex, common.BlockMetadata] var reorgDetector chan struct{} if cacheSize != 0 { - cache = lru.NewSizeConstrainedCache[arbutil.MessageIndex, arbostypes.BlockMetadata](cacheSize) + cache = lru.NewSizeConstrainedCache[arbutil.MessageIndex, common.BlockMetadata](cacheSize) reorgDetector = make(chan struct{}) fetcher.SetReorgEventsNotifier(reorgDetector) } @@ -71,7 +71,7 @@ func (b *BulkBlockMetadataFetcher) Fetch(fromBlock, toBlock rpc.BlockNumber) ([] } var result []NumberAndBlockMetadata for i := start; i <= end; i++ { - var data arbostypes.BlockMetadata + var data common.BlockMetadata var found bool if b.cache != nil { data, found = b.cache.Get(i) diff --git a/execution/gethexec/executionengine.go b/execution/gethexec/executionengine.go index 6756a6c693..5966b2b270 100644 --- a/execution/gethexec/executionengine.go +++ b/execution/gethexec/executionengine.go @@ -259,7 +259,7 @@ func (s *ExecutionEngine) SetConsensus(consensus execution.FullConsensusClient) s.consensus = consensus } -func (s *ExecutionEngine) BlockMetadataAtCount(count arbutil.MessageIndex) (arbostypes.BlockMetadata, error) { +func (s *ExecutionEngine) BlockMetadataAtCount(count arbutil.MessageIndex) (common.BlockMetadata, error) { if s.consensus != nil { return s.consensus.BlockMetadataAtCount(count) } @@ -612,8 +612,8 @@ func (s *ExecutionEngine) sequenceTransactionsWithBlockMutex(header *arbostypes. // starting from the second byte, (N)th bit would represent if (N)th tx is timeboosted or not, 1 means yes and 0 means no // blockMetadata[index / 8 + 1] & (1 << (index % 8)) != 0; where index = (N - 1), implies whether (N)th tx in a block is timeboosted // note that number of txs in a block will always lag behind (len(blockMetadata) - 1) * 8 but it wont lag more than a value of 7 -func (s *ExecutionEngine) blockMetadataFromBlock(block *types.Block, timeboostedTxs map[common.Hash]struct{}) arbostypes.BlockMetadata { - bits := make(arbostypes.BlockMetadata, 1+arbmath.DivCeil(uint64(len(block.Transactions())), 8)) +func (s *ExecutionEngine) blockMetadataFromBlock(block *types.Block, timeboostedTxs map[common.Hash]struct{}) common.BlockMetadata { + bits := make(common.BlockMetadata, 1+arbmath.DivCeil(uint64(len(block.Transactions())), 8)) if len(timeboostedTxs) == 0 { return bits } diff --git a/execution/gethexec/sync_monitor.go b/execution/gethexec/sync_monitor.go index 7f04b2ee4a..07c05351d1 100644 --- a/execution/gethexec/sync_monitor.go +++ b/execution/gethexec/sync_monitor.go @@ -6,6 +6,9 @@ import ( "github.com/pkg/errors" flag "github.com/spf13/pflag" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/log" + "github.com/offchainlabs/nitro/execution" ) @@ -122,3 +125,15 @@ func (s *SyncMonitor) Synced() bool { func (s *SyncMonitor) SetConsensusInfo(consensus execution.ConsensusInfo) { s.consensus = consensus } + +func (s *SyncMonitor) BlockMetadataByNumber(blockNum uint64) (common.BlockMetadata, error) { + count, err := s.exec.BlockNumberToMessageIndex(blockNum) + if err != nil { + return nil, err + } + if s.consensus != nil { + return s.consensus.BlockMetadataAtCount(count + 1) + } + log.Debug("FullConsensusClient is not accessible to execution, BlockMetadataByNumber will return nil") + return nil, nil +} diff --git a/execution/interface.go b/execution/interface.go index d9e0e9f85d..ca067240d0 100644 --- a/execution/interface.go +++ b/execution/interface.go @@ -86,7 +86,7 @@ type ConsensusInfo interface { Synced() bool FullSyncProgressMap() map[string]interface{} SyncTargetMessageCount() arbutil.MessageIndex - BlockMetadataAtCount(count arbutil.MessageIndex) (arbostypes.BlockMetadata, error) + BlockMetadataAtCount(count arbutil.MessageIndex) (common.BlockMetadata, error) // TODO: switch from pulling to pushing safe/finalized GetSafeMsgCount(ctx context.Context) (arbutil.MessageIndex, error) @@ -95,7 +95,7 @@ type ConsensusInfo interface { } type ConsensusSequencer interface { - WriteMessageFromSequencer(pos arbutil.MessageIndex, msgWithMeta arbostypes.MessageWithMetadata, msgResult MessageResult, blockMetadata arbostypes.BlockMetadata) error + WriteMessageFromSequencer(pos arbutil.MessageIndex, msgWithMeta arbostypes.MessageWithMetadata, msgResult MessageResult, blockMetadata common.BlockMetadata) error ExpectChosenSequencer() error } diff --git a/go-ethereum b/go-ethereum index 2b9bafa37d..17286562eb 160000 --- a/go-ethereum +++ b/go-ethereum @@ -1 +1 @@ -Subproject commit 2b9bafa37da7b4b7a47833573d5b6969c0b216e4 +Subproject commit 17286562eb2f0dcd02dafdaf5da0fce5ff1f6096 diff --git a/system_tests/timeboost_test.go b/system_tests/timeboost_test.go index c1060f45b4..8bbeab61c9 100644 --- a/system_tests/timeboost_test.go +++ b/system_tests/timeboost_test.go @@ -5,6 +5,7 @@ import ( "context" "crypto/ecdsa" "encoding/binary" + "encoding/json" "fmt" "math/big" "net" @@ -25,12 +26,12 @@ import ( "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/crypto/secp256k1" "github.com/ethereum/go-ethereum/ethclient" + "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/rpc" "github.com/offchainlabs/nitro/arbnode" - "github.com/offchainlabs/nitro/arbos/arbostypes" "github.com/offchainlabs/nitro/arbos/util" "github.com/offchainlabs/nitro/arbutil" "github.com/offchainlabs/nitro/broadcastclient" @@ -43,6 +44,7 @@ import ( "github.com/offchainlabs/nitro/timeboost" "github.com/offchainlabs/nitro/timeboost/bindings" "github.com/offchainlabs/nitro/util/arbmath" + "github.com/offchainlabs/nitro/util/colors" "github.com/offchainlabs/nitro/util/containers" "github.com/offchainlabs/nitro/util/redisutil" "github.com/offchainlabs/nitro/util/rpcclient" @@ -91,7 +93,7 @@ func TestTimeboostBulkBlockMetadataFetcher(t *testing.T) { break } } - var sampleBulkData []arbostypes.BlockMetadata + var sampleBulkData []common.BlockMetadata for i := 1; i <= int(latestL2); i++ { // #nosec G115 blockMetadata := []byte{0, uint8(i)} @@ -185,6 +187,114 @@ func TestTimeboostBulkBlockMetadataFetcher(t *testing.T) { iter.Release() } +func TestTimeboostedFieldInReceiptsObject(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + builder := NewNodeBuilder(ctx).DefaultConfig(t, false) + builder.execConfig.BlockMetadataApiCacheSize = 0 // Caching is disabled + cleanup := builder.Build(t) + defer cleanup() + + // Generate blocks until current block is totalBlocks + arbDb := builder.L2.ConsensusNode.ArbDB + blockNum := big.NewInt(2) + builder.L2Info.GenerateAccount("User") + user := builder.L2Info.GetDefaultTransactOpts("User", ctx) + var latestL2 uint64 + var err error + for i := 0; ; i++ { + builder.L2.TransferBalanceTo(t, "Owner", util.RemapL1Address(user.From), big.NewInt(1e18), builder.L2Info) + latestL2, err = builder.L2.Client.BlockNumber(ctx) + Require(t, err) + if latestL2 >= blockNum.Uint64() { + break + } + } + + for i := uint64(1); i < latestL2; i++ { + // Clean BlockMetadata from arbDB so that we can modify it at will + Require(t, arbDb.Delete(dbKey([]byte("t"), i))) + } + + block, err := builder.L2.Client.BlockByNumber(ctx, blockNum) + Require(t, err) + if len(block.Transactions()) != 2 { + t.Fatalf("expecting two txs in the second block, but found: %d txs", len(block.Transactions())) + } + + // Set first tx (internal tx anyway) to not timeboosted and Second one to timeboosted- BlockMetadata (in bits)-> 00000000 00000010 + Require(t, arbDb.Put(dbKey([]byte("t"), blockNum.Uint64()), []byte{0, 2})) + l2rpc := builder.L2.Stack.Attach() + // Extra timeboosted field in pointer form to check for its existence + type timeboostedFromReceipt struct { + Timeboosted *bool `json:"timeboosted"` + } + var receiptResult []timeboostedFromReceipt + err = l2rpc.CallContext(ctx, &receiptResult, "eth_getBlockReceipts", rpc.BlockNumber(blockNum.Int64())) + Require(t, err) + if receiptResult[0].Timeboosted == nil || receiptResult[1].Timeboosted == nil { + t.Fatal("timeboosted field should exist in the receipt object of both- first and second txs") + } + if *receiptResult[0].Timeboosted != false { + t.Fatal("first tx was not timeboosted, but the field indicates otherwise") + } + if *receiptResult[1].Timeboosted != true { + t.Fatal("second tx was timeboosted, but the field indicates otherwise") + } + + // Check that timeboosted is accurate for eth_getTransactionReceipt as well + var txReceipt timeboostedFromReceipt + err = l2rpc.CallContext(ctx, &txReceipt, "eth_getTransactionReceipt", block.Transactions()[0].Hash()) + Require(t, err) + if txReceipt.Timeboosted == nil { + t.Fatal("timeboosted field should exist in the receipt object of first tx") + } + if *txReceipt.Timeboosted != false { + t.Fatal("first tx was not timeboosted, but the field indicates otherwise") + } + err = l2rpc.CallContext(ctx, &txReceipt, "eth_getTransactionReceipt", block.Transactions()[1].Hash()) + Require(t, err) + if txReceipt.Timeboosted == nil { + t.Fatal("timeboosted field should exist in the receipt object of second tx") + } + if *txReceipt.Timeboosted != true { + t.Fatal("second tx was timeboosted, but the field indicates otherwise") + } + + // Check that timeboosted field shouldn't exist for any txs of block=1, as this block doesn't have blockMetadata + block, err = builder.L2.Client.BlockByNumber(ctx, common.Big1) + Require(t, err) + if len(block.Transactions()) != 2 { + t.Fatalf("expecting two txs in the first block, but found: %d txs", len(block.Transactions())) + } + var receiptResult2 []timeboostedFromReceipt + err = l2rpc.CallContext(ctx, &receiptResult2, "eth_getBlockReceipts", rpc.BlockNumber(1)) + Require(t, err) + if receiptResult2[0].Timeboosted != nil || receiptResult2[1].Timeboosted != nil { + t.Fatal("timeboosted field shouldn't exist in the receipt object of all the txs") + } + var txReceipt2 timeboostedFromReceipt + err = l2rpc.CallContext(ctx, &txReceipt2, "eth_getTransactionReceipt", block.Transactions()[0].Hash()) + Require(t, err) + if txReceipt2.Timeboosted != nil { + t.Fatal("timeboosted field shouldn't exist in the receipt object of all the txs") + } + var txReceipt3 timeboostedFromReceipt + err = l2rpc.CallContext(ctx, &txReceipt3, "eth_getTransactionReceipt", block.Transactions()[1].Hash()) + Require(t, err) + if txReceipt3.Timeboosted != nil { + t.Fatal("timeboosted field shouldn't exist in the receipt object of all the txs") + } + + // Print the receipt object for reference + var receiptResultRaw json.RawMessage + err = l2rpc.CallContext(ctx, &receiptResultRaw, "eth_getBlockReceipts", rpc.BlockNumber(blockNum.Int64())) + Require(t, err) + colors.PrintGrey("receipt object- ", string(receiptResultRaw)) + +} + func TestTimeboostBulkBlockMetadataAPI(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -442,7 +552,7 @@ func TestSequencerFeed_ExpressLaneAuction_ExpressLaneTxsHaveAdvantage(t *testing func TestSequencerFeed_ExpressLaneAuction_InnerPayloadNoncesAreRespected_TimeboostedFieldIsCorrect(t *testing.T) { t.Parallel() - // logHandler := testhelpers.InitTestLog(t, log.LevelInfo) + logHandler := testhelpers.InitTestLog(t, log.LevelInfo) ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -570,10 +680,9 @@ func TestSequencerFeed_ExpressLaneAuction_InnerPayloadNoncesAreRespected_Timeboo verifyTimeboostedCorrectness(t, ctx, "Alice", feedListener.ConsensusNode, feedListener.Client, false, aliceTx, aliceBlock) verifyTimeboostedCorrectness(t, ctx, "Charlie", feedListener.ConsensusNode, feedListener.Client, true, charlie0, charlieBlock) - // arbnode.BlockHashMismatchLogMsg has been randomly appearing and disappearing when running this test, not sure why that might be happening - // if logHandler.WasLogged(arbnode.BlockHashMismatchLogMsg) { - // t.Fatal("BlockHashMismatchLogMsg was logged unexpectedly") - // } + if logHandler.WasLogged(arbnode.BlockHashMismatchLogMsg) { + t.Fatal("BlockHashMismatchLogMsg was logged unexpectedly") + } } // verifyTimeboostedCorrectness is used to check if the timeboosted byte array in both the sequencer's tx streamer and the client node's tx streamer (which is connected