From 49762e7f0c79e21e76c0f0443c6e0aee838c7200 Mon Sep 17 00:00:00 2001 From: Ethen Pociask Date: Wed, 11 Oct 2023 12:09:26 -0700 Subject: [PATCH 01/23] [epociask/debug] Debugging bridge processor --- .vscode/launch.json | 22 ++++++++++++++++++++++ indexer/database/blocks.go | 4 ++++ 2 files changed, 26 insertions(+) create mode 100644 .vscode/launch.json diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 000000000000..27d5932f59b1 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,22 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + + "type": "go", + "request": "launch", + "name": "Launch Program", + "program": "${workspaceFolder}/indexer/cmd/indexer", + "mode": "debug", + "env": { + "INDEXER_CONFIG": "/Users/ethen.pociask@coinbase.com/coinbase/indexer.toml" + }, + "args": [ + "index" + ] + } + ] +} diff --git a/indexer/database/blocks.go b/indexer/database/blocks.go index c28797ab69d9..0e24b2edf629 100644 --- a/indexer/database/blocks.go +++ b/indexer/database/blocks.go @@ -8,6 +8,7 @@ import ( "github.com/ethereum-optimism/optimism/indexer/bigint" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/log" "gorm.io/gorm" ) @@ -200,6 +201,7 @@ func (db *blocksDB) LatestObservedEpoch(fromL1Height *big.Int, maxL1Range uint64 result := db.gorm.Where(l1QueryFilter).Order("timestamp DESC").Take(&l1Header) if result.Error != nil { if errors.Is(result.Error, gorm.ErrRecordNotFound) { + log.Warn("Could not fetch latest L1 block header in bridge processor") return nil, nil } return nil, result.Error @@ -211,6 +213,7 @@ func (db *blocksDB) LatestObservedEpoch(fromL1Height *big.Int, maxL1Range uint64 result = db.gorm.Where("timestamp <= ?", toTimestamp).Order("timestamp DESC").Take(&l2Header) if result.Error != nil { if errors.Is(result.Error, gorm.ErrRecordNotFound) { + log.Warn("Could not fetch latest L2 block header in bridge processor") return nil, nil } return nil, result.Error @@ -233,6 +236,7 @@ func (db *blocksDB) LatestObservedEpoch(fromL1Height *big.Int, maxL1Range uint64 result := query.Take(&epoch) if result.Error != nil { if errors.Is(result.Error, gorm.ErrRecordNotFound) { + log.Warn("Could not fetch latest observed epoch in bridge processor") return nil, nil } return nil, result.Error From 67320df9646bf3f257c5775968ca12f0b72bc6e7 Mon Sep 17 00:00:00 2001 From: Ethen Pociask Date: Wed, 11 Oct 2023 12:41:25 -0700 Subject: [PATCH 02/23] [epociask/debug] Output timestamp and l1 span range --- indexer/database/blocks.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/indexer/database/blocks.go b/indexer/database/blocks.go index 0e24b2edf629..6fbaad5710c8 100644 --- a/indexer/database/blocks.go +++ b/indexer/database/blocks.go @@ -201,7 +201,8 @@ func (db *blocksDB) LatestObservedEpoch(fromL1Height *big.Int, maxL1Range uint64 result := db.gorm.Where(l1QueryFilter).Order("timestamp DESC").Take(&l1Header) if result.Error != nil { if errors.Is(result.Error, gorm.ErrRecordNotFound) { - log.Warn("Could not fetch latest L1 block header in bridge processor") + log.Warn("Could not fetch latest L1 block header in bridge processor", "from_timestamp", + fromTimestamp, "max_l1_range", maxL1Range) return nil, nil } return nil, result.Error From 79599a07d507d5456bfd83e39599fcbc86a72477 Mon Sep 17 00:00:00 2001 From: Ethen Pociask Date: Wed, 11 Oct 2023 15:11:00 -0700 Subject: [PATCH 03/23] [indexer.bridge_offset_fix] Added starting height --- indexer/processors/bridge.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/indexer/processors/bridge.go b/indexer/processors/bridge.go index 48f6c4dc87bc..590e000759b8 100644 --- a/indexer/processors/bridge.go +++ b/indexer/processors/bridge.go @@ -98,6 +98,8 @@ func (b *BridgeProcessor) run() error { var lastEpoch *big.Int if b.LatestL1Header != nil { lastEpoch = b.LatestL1Header.Number + } else { + lastEpoch = big.NewInt(int64(b.chainConfig.L1StartingHeight)) } latestEpoch, err := b.db.Blocks.LatestObservedEpoch(lastEpoch, maxEpochRange) From 82f3fe0ad5df0a6de37ba6c9e4ff14e2661590d7 Mon Sep 17 00:00:00 2001 From: Ethen Pociask Date: Wed, 11 Oct 2023 15:45:39 -0700 Subject: [PATCH 04/23] [indexer.bridge_offset_fix] Log starting height --- indexer/processors/bridge.go | 1 + 1 file changed, 1 insertion(+) diff --git a/indexer/processors/bridge.go b/indexer/processors/bridge.go index 590e000759b8..7735fa4a8008 100644 --- a/indexer/processors/bridge.go +++ b/indexer/processors/bridge.go @@ -99,6 +99,7 @@ func (b *BridgeProcessor) run() error { if b.LatestL1Header != nil { lastEpoch = b.LatestL1Header.Number } else { + b.log.Info("starting processing from supplied genesis epoch", "l1_starting_number", b.chainConfig.L1StartingHeight) lastEpoch = big.NewInt(int64(b.chainConfig.L1StartingHeight)) } From fbc8a125c24c99f0f9b8b50e6c1e5f4d0ee06239 Mon Sep 17 00:00:00 2001 From: Ethen Pociask Date: Wed, 11 Oct 2023 17:56:47 -0700 Subject: [PATCH 05/23] [indexer.bridge_offset_fix] More bridge logging --- indexer/database/blocks.go | 5 ++++- indexer/processors/bridge.go | 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/indexer/database/blocks.go b/indexer/database/blocks.go index 6fbaad5710c8..33c9ec49f0cc 100644 --- a/indexer/database/blocks.go +++ b/indexer/database/blocks.go @@ -175,12 +175,13 @@ func (db *blocksDB) LatestObservedEpoch(fromL1Height *big.Int, maxL1Range uint64 fromL1Height = bigint.Zero } - // Lower Bound (the default `fromTimestamp = 0` suffices genesis representation) + // Lower Bound (the default `fromTimestamp = l1_starting_heigh` (default=0) suffices genesis representation) if fromL1Height.BitLen() > 0 { var header L1BlockHeader result := db.gorm.Where("number = ?", fromL1Height).Take(&header) if result.Error != nil { if errors.Is(result.Error, gorm.ErrRecordNotFound) { + log.Warn("Could not fetch latest L1 block header in bridge processor", "number", fromL1Height) return nil, nil } return nil, result.Error @@ -197,6 +198,7 @@ func (db *blocksDB) LatestObservedEpoch(fromL1Height *big.Int, maxL1Range uint64 l1QueryFilter = fmt.Sprintf("%s AND number <= %d", l1QueryFilter, maxHeight) } + // Fetch most recent header from l1_block_headers table var l1Header L1BlockHeader result := db.gorm.Where(l1QueryFilter).Order("timestamp DESC").Take(&l1Header) if result.Error != nil { @@ -210,6 +212,7 @@ func (db *blocksDB) LatestObservedEpoch(fromL1Height *big.Int, maxL1Range uint64 toTimestamp = l1Header.Timestamp + // Fetch most recent header from l2_block_headers table var l2Header L2BlockHeader result = db.gorm.Where("timestamp <= ?", toTimestamp).Order("timestamp DESC").Take(&l2Header) if result.Error != nil { diff --git a/indexer/processors/bridge.go b/indexer/processors/bridge.go index 7735fa4a8008..c8d25a288faa 100644 --- a/indexer/processors/bridge.go +++ b/indexer/processors/bridge.go @@ -130,11 +130,11 @@ func (b *BridgeProcessor) run() error { } if b.LatestL1Header != nil && latestEpoch.L1BlockHeader.Number.Cmp(b.LatestL1Header.Number) <= 0 { b.log.Error("non-increasing l1 block height observed", "latest_bridge_l1_block_number", b.LatestL1Header.Number, "latest_epoch_l1_block_number", latestEpoch.L1BlockHeader.Number) - return errors.New("non-increasing l1 block heght observed") + return errors.New("non-increasing l1 block height observed") } if b.LatestL2Header != nil && latestEpoch.L2BlockHeader.Number.Cmp(b.LatestL2Header.Number) <= 0 { b.log.Error("non-increasing l2 block height observed", "latest_bridge_l2_block_number", b.LatestL2Header.Number, "latest_epoch_l2_block_number", latestEpoch.L2BlockHeader.Number) - return errors.New("non-increasing l2 block heght observed") + return errors.New("non-increasing l2 block height observed") } toL1Height, toL2Height := latestEpoch.L1BlockHeader.Number, latestEpoch.L2BlockHeader.Number From dac8f106728936079b3f2544db40f7f3e81a595d Mon Sep 17 00:00:00 2001 From: Ethen Pociask Date: Wed, 11 Oct 2023 23:32:45 -0700 Subject: [PATCH 06/23] [indexer.bridge_offset_fix] Reset state --- indexer/migrations/20230523_create_schema.sql | 7 +++++++ indexer/node/header_traversal.go | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/indexer/migrations/20230523_create_schema.sql b/indexer/migrations/20230523_create_schema.sql index b13eb9dc6fab..dffc4c4a5d10 100644 --- a/indexer/migrations/20230523_create_schema.sql +++ b/indexer/migrations/20230523_create_schema.sql @@ -16,6 +16,13 @@ END $$; * BLOCK DATA */ + +DROP TABLE IF EXISTS l1_block_headers CASCADE; +DROP TABLE IF EXISTS l2_block_headers CASCADE; +DROP TABLE IF EXISTS l1_contract_events CASCADE; +DROP TABLE IF EXISTS l2_contract_events CASCADE; + + CREATE TABLE IF NOT EXISTS l1_block_headers ( -- Searchable fields hash VARCHAR PRIMARY KEY, diff --git a/indexer/node/header_traversal.go b/indexer/node/header_traversal.go index 43d33ff4f812..001dc1745295 100644 --- a/indexer/node/header_traversal.go +++ b/indexer/node/header_traversal.go @@ -33,7 +33,7 @@ func (f *HeaderTraversal) LastHeader() *types.Header { return f.lastHeader } -// NextFinalizedHeaders retrives the next set of headers that have been +// NextFinalizedHeaders retrieves the next set of headers that have been // marked as finalized by the connected client, bounded by the supplied size func (f *HeaderTraversal) NextFinalizedHeaders(maxSize uint64) ([]types.Header, error) { latestBlockHeader, err := f.ethClient.BlockHeaderByNumber(nil) From aae3935cd927aea7f5b6b03abca7e6346576e31c Mon Sep 17 00:00:00 2001 From: Ethen Pociask Date: Thu, 12 Oct 2023 02:14:29 -0700 Subject: [PATCH 07/23] [indexer.bridge_offset_fix] Added code comments to header_traversal.go --- indexer/node/header_traversal.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/indexer/node/header_traversal.go b/indexer/node/header_traversal.go index 001dc1745295..e925747f2185 100644 --- a/indexer/node/header_traversal.go +++ b/indexer/node/header_traversal.go @@ -49,7 +49,7 @@ func (f *HeaderTraversal) NextFinalizedHeaders(maxSize uint64) ([]types.Header, if f.lastHeader != nil { cmp := f.lastHeader.Number.Cmp(endHeight) - if cmp == 0 { + if cmp == 0 { // We're synced to head and there are no new headers return nil, nil } else if cmp > 0 { return nil, ErrHeaderTraversalAheadOfProvider @@ -61,6 +61,7 @@ func (f *HeaderTraversal) NextFinalizedHeaders(maxSize uint64) ([]types.Header, nextHeight = new(big.Int).Add(f.lastHeader.Number, bigint.One) } + // endHeight = (nextHeight - endHeight) <= maxSize endHeight = bigint.Clamp(nextHeight, endHeight, maxSize) headers, err := f.ethClient.BlockHeadersByRange(nextHeight, endHeight) if err != nil { From e178ea1a59e4f435f4fbe344acdf6404340a8465 Mon Sep 17 00:00:00 2001 From: Ethen Pociask Date: Thu, 12 Oct 2023 12:13:09 -0700 Subject: [PATCH 08/23] [indexer.bridge_offset_fix] Added processor:bridge tag to epoch fetcing function --- indexer/database/blocks.go | 9 +++++---- indexer/processors/bridge.go | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/indexer/database/blocks.go b/indexer/database/blocks.go index 33c9ec49f0cc..c75313d8450c 100644 --- a/indexer/database/blocks.go +++ b/indexer/database/blocks.go @@ -181,7 +181,8 @@ func (db *blocksDB) LatestObservedEpoch(fromL1Height *big.Int, maxL1Range uint64 result := db.gorm.Where("number = ?", fromL1Height).Take(&header) if result.Error != nil { if errors.Is(result.Error, gorm.ErrRecordNotFound) { - log.Warn("Could not fetch latest L1 block header in bridge processor", "number", fromL1Height) + log.Warn("Could not fetch latest L1 block header in bridge processor", "number", fromL1Height, + "processor", "bridge") return nil, nil } return nil, result.Error @@ -204,7 +205,7 @@ func (db *blocksDB) LatestObservedEpoch(fromL1Height *big.Int, maxL1Range uint64 if result.Error != nil { if errors.Is(result.Error, gorm.ErrRecordNotFound) { log.Warn("Could not fetch latest L1 block header in bridge processor", "from_timestamp", - fromTimestamp, "max_l1_range", maxL1Range) + fromTimestamp, "max_l1_range", maxL1Range, "processor", "bridge") return nil, nil } return nil, result.Error @@ -217,7 +218,7 @@ func (db *blocksDB) LatestObservedEpoch(fromL1Height *big.Int, maxL1Range uint64 result = db.gorm.Where("timestamp <= ?", toTimestamp).Order("timestamp DESC").Take(&l2Header) if result.Error != nil { if errors.Is(result.Error, gorm.ErrRecordNotFound) { - log.Warn("Could not fetch latest L2 block header in bridge processor") + log.Warn("Could not fetch latest L2 block header in bridge processor", "processor", "bridge") return nil, nil } return nil, result.Error @@ -240,7 +241,7 @@ func (db *blocksDB) LatestObservedEpoch(fromL1Height *big.Int, maxL1Range uint64 result := query.Take(&epoch) if result.Error != nil { if errors.Is(result.Error, gorm.ErrRecordNotFound) { - log.Warn("Could not fetch latest observed epoch in bridge processor") + log.Warn("Could not fetch latest observed epoch in bridge processor", "processor", "bridge") return nil, nil } return nil, result.Error diff --git a/indexer/processors/bridge.go b/indexer/processors/bridge.go index c8d25a288faa..1996712b41ec 100644 --- a/indexer/processors/bridge.go +++ b/indexer/processors/bridge.go @@ -88,7 +88,7 @@ func (b *BridgeProcessor) Start(ctx context.Context) error { // Runs the processing loop. In order to ensure all seen bridge finalization events // can be correlated with bridge initiated events, we establish a shared marker between -// L1 and L2 when processing events. The lastest shared indexed time (epochs) between +// L1 and L2 when processing events. The latest shared indexed time (epochs) between // L1 and L2 serves as this shared marker. func (b *BridgeProcessor) run() error { // In the event where we have a large number of un-observed epochs, we cap the search From cde1ab96ffea485359a953138065d3f6c64cd86f Mon Sep 17 00:00:00 2001 From: Ethen Pociask Date: Thu, 12 Oct 2023 14:20:41 -0700 Subject: [PATCH 09/23] [indexer.bridge_offset_fix] Updated starting height logic --- indexer/README.md | 12 +++++++++--- indexer/database/blocks.go | 19 +++++++++++++------ indexer/processors/bridge.go | 3 --- 3 files changed, 22 insertions(+), 12 deletions(-) diff --git a/indexer/README.md b/indexer/README.md index 17ba2dbbcbe3..0cea037f9160 100644 --- a/indexer/README.md +++ b/indexer/README.md @@ -54,11 +54,17 @@ The indexer service is responsible for polling and processing real-time batches * Process and persist new bridge events * Synchronize L1 proven/finalized withdrawals with their L2 initialization counterparts -#### API + +### L1 Polling +L1 blocks are only indexed if they contain: +- L1 system contract events +- Batch commitment events + +#### API The indexer service runs a lightweight health server adjacently to the main service. The health server exposes a single endpoint `/healthz` that can be used to check the health of the indexer service. The health assessment doesn't check dependency health (ie. database) but rather checks the health of the indexer service itself. ### Database The indexer service currently supports a Postgres database for storing L1/L2 OP Stack chain data. The most up-to-date database schemas can be found in the `./migrations` directory. -## Metrics -The indexer services exposes a set of Prometheus metrics that can be used to monitor the health of the service. The metrics are exposed via the `/metrics` endpoint on the health server. \ No newline at end of file +## Metrics +The indexer services exposes a set of Prometheus metrics that can be used to monitor the health of the service. The metrics are exposed via the `/metrics` endpoint on the health server. \ No newline at end of file diff --git a/indexer/database/blocks.go b/indexer/database/blocks.go index c75313d8450c..52e180c3ec4f 100644 --- a/indexer/database/blocks.go +++ b/indexer/database/blocks.go @@ -5,7 +5,6 @@ import ( "fmt" "math/big" - "github.com/ethereum-optimism/optimism/indexer/bigint" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/log" @@ -171,14 +170,11 @@ func (db *blocksDB) LatestObservedEpoch(fromL1Height *big.Int, maxL1Range uint64 // We use timestamps since that translates to both L1 & L2 var fromTimestamp, toTimestamp uint64 - if fromL1Height == nil { - fromL1Height = bigint.Zero - } - // Lower Bound (the default `fromTimestamp = l1_starting_heigh` (default=0) suffices genesis representation) - if fromL1Height.BitLen() > 0 { + if fromL1Height != nil { var header L1BlockHeader result := db.gorm.Where("number = ?", fromL1Height).Take(&header) + // TODO - Embed logging to db if result.Error != nil { if errors.Is(result.Error, gorm.ErrRecordNotFound) { log.Warn("Could not fetch latest L1 block header in bridge processor", "number", fromL1Height, @@ -189,6 +185,17 @@ func (db *blocksDB) LatestObservedEpoch(fromL1Height *big.Int, maxL1Range uint64 } fromTimestamp = header.Timestamp + } else { + var header L1BlockHeader + result := db.gorm.Order("number desc").Take(&header) + if result.Error != nil { + if errors.Is(result.Error, gorm.ErrRecordNotFound) { + log.Warn("Could not fetch latest L1 block header in bridge processor", "number", fromL1Height, + "processor", "bridge") + return nil, nil + } + return nil, result.Error + } } // Upper Bound (lowest timestamp indexed between L1/L2 bounded by `maxL1Range`) diff --git a/indexer/processors/bridge.go b/indexer/processors/bridge.go index 1996712b41ec..34b5b95dcfb0 100644 --- a/indexer/processors/bridge.go +++ b/indexer/processors/bridge.go @@ -98,9 +98,6 @@ func (b *BridgeProcessor) run() error { var lastEpoch *big.Int if b.LatestL1Header != nil { lastEpoch = b.LatestL1Header.Number - } else { - b.log.Info("starting processing from supplied genesis epoch", "l1_starting_number", b.chainConfig.L1StartingHeight) - lastEpoch = big.NewInt(int64(b.chainConfig.L1StartingHeight)) } latestEpoch, err := b.db.Blocks.LatestObservedEpoch(lastEpoch, maxEpochRange) From c039ddbcf914549cff32820e6744c011f2c37804 Mon Sep 17 00:00:00 2001 From: Ethen Pociask Date: Thu, 12 Oct 2023 16:07:42 -0700 Subject: [PATCH 10/23] [indexer.bridge_offset_fix] Added header to global function scope in LatestObservedEpoch --- indexer/database/blocks.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/indexer/database/blocks.go b/indexer/database/blocks.go index 52e180c3ec4f..31d0138dbaa5 100644 --- a/indexer/database/blocks.go +++ b/indexer/database/blocks.go @@ -171,13 +171,13 @@ func (db *blocksDB) LatestObservedEpoch(fromL1Height *big.Int, maxL1Range uint64 var fromTimestamp, toTimestamp uint64 // Lower Bound (the default `fromTimestamp = l1_starting_heigh` (default=0) suffices genesis representation) + var header L1BlockHeader if fromL1Height != nil { - var header L1BlockHeader result := db.gorm.Where("number = ?", fromL1Height).Take(&header) // TODO - Embed logging to db if result.Error != nil { if errors.Is(result.Error, gorm.ErrRecordNotFound) { - log.Warn("Could not fetch latest L1 block header in bridge processor", "number", fromL1Height, + log.Warn("Could not fetch latest provided L1 block header in bridge processor", "number", fromL1Height, "processor", "bridge") return nil, nil } @@ -186,11 +186,10 @@ func (db *blocksDB) LatestObservedEpoch(fromL1Height *big.Int, maxL1Range uint64 fromTimestamp = header.Timestamp } else { - var header L1BlockHeader result := db.gorm.Order("number desc").Take(&header) if result.Error != nil { if errors.Is(result.Error, gorm.ErrRecordNotFound) { - log.Warn("Could not fetch latest L1 block header in bridge processor", "number", fromL1Height, + log.Warn("Could not fetch recent L1 block heade", "number", fromL1Height, "processor", "bridge") return nil, nil } From 3c978c014f0eb0b271d0495e895d0d43929d111e Mon Sep 17 00:00:00 2001 From: Ethen Pociask Date: Thu, 12 Oct 2023 16:29:18 -0700 Subject: [PATCH 11/23] [indexer.bridge_offset_fix] Setting timestamp appropietly --- indexer/database/blocks.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/indexer/database/blocks.go b/indexer/database/blocks.go index 31d0138dbaa5..0bbb152e084a 100644 --- a/indexer/database/blocks.go +++ b/indexer/database/blocks.go @@ -189,12 +189,14 @@ func (db *blocksDB) LatestObservedEpoch(fromL1Height *big.Int, maxL1Range uint64 result := db.gorm.Order("number desc").Take(&header) if result.Error != nil { if errors.Is(result.Error, gorm.ErrRecordNotFound) { - log.Warn("Could not fetch recent L1 block heade", "number", fromL1Height, + log.Warn("Could not fetch recent L1 block header", "number", fromL1Height, "processor", "bridge") return nil, nil } return nil, result.Error } + + fromL1Height = header.Number } // Upper Bound (lowest timestamp indexed between L1/L2 bounded by `maxL1Range`) From 14cd641404ee1e5f789d1f56ed2aa227fdea2e74 Mon Sep 17 00:00:00 2001 From: Ethen Pociask Date: Thu, 12 Oct 2023 17:01:00 -0700 Subject: [PATCH 12/23] [indexer.bridge_offset_fix] Nil check for mint value in deposit tx --- indexer/processors/bridge.go | 5 ++++- indexer/processors/contracts/optimism_portal.go | 5 +++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/indexer/processors/bridge.go b/indexer/processors/bridge.go index 34b5b95dcfb0..1e408b176fc7 100644 --- a/indexer/processors/bridge.go +++ b/indexer/processors/bridge.go @@ -201,24 +201,27 @@ func (b *BridgeProcessor) run() error { // First, find all possible initiated bridge events if err := bridge.L1ProcessInitiatedBridgeEvents(l1BridgeLog, tx, b.metrics, b.chainConfig.L1Contracts, fromL1Height, toL1Height); err != nil { + batchLog.Error("failed to index l1 initiated bridge events", "err", err) return err } if err := bridge.L2ProcessInitiatedBridgeEvents(l2BridgeLog, tx, b.metrics, b.chainConfig.L2Contracts, fromL2Height, toL2Height); err != nil { + batchLog.Error("failed to index l2 initiated bridge events", "err", err) return err } // Now all finalization events can find their counterpart. if err := bridge.L1ProcessFinalizedBridgeEvents(l1BridgeLog, tx, b.metrics, b.chainConfig.L1Contracts, fromL1Height, toL1Height); err != nil { + batchLog.Error("failed to index l1 finalized bridge events", "err", err) return err } if err := bridge.L2ProcessFinalizedBridgeEvents(l2BridgeLog, tx, b.metrics, b.chainConfig.L2Contracts, fromL2Height, toL2Height); err != nil { + batchLog.Error("failed to index l2 finalized bridge events", "err", err) return err } // a-ok return nil }); err != nil { - batchLog.Error("failed to index bridge events", "err", err) return err } diff --git a/indexer/processors/contracts/optimism_portal.go b/indexer/processors/contracts/optimism_portal.go index 0e618c982e5a..0cbacbf97ff0 100644 --- a/indexer/processors/contracts/optimism_portal.go +++ b/indexer/processors/contracts/optimism_portal.go @@ -64,6 +64,11 @@ func OptimismPortalTransactionDepositEvents(contractAddress common.Address, db * return nil, err } + mint := depositTx.Mint + if mint == nil { + mint = big.NewInt(0) + } + optimismPortalTxDeposits[i] = OptimismPortalTransactionDepositEvent{ Event: &transactionDepositEvents[i].ContractEvent, DepositTx: depositTx, From a35a963405c32151374b70ca3e04edc156d8d956 Mon Sep 17 00:00:00 2001 From: Ethen Pociask Date: Thu, 12 Oct 2023 17:16:31 -0700 Subject: [PATCH 13/23] [indexer.bridge_offset_fix] blank slate DB --- indexer/migrations/20230523_create_schema.sql | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/indexer/migrations/20230523_create_schema.sql b/indexer/migrations/20230523_create_schema.sql index dffc4c4a5d10..bd1599dd0012 100644 --- a/indexer/migrations/20230523_create_schema.sql +++ b/indexer/migrations/20230523_create_schema.sql @@ -21,6 +21,14 @@ DROP TABLE IF EXISTS l1_block_headers CASCADE; DROP TABLE IF EXISTS l2_block_headers CASCADE; DROP TABLE IF EXISTS l1_contract_events CASCADE; DROP TABLE IF EXISTS l2_contract_events CASCADE; +DROP TABLE IF EXISTS l1_transaction_deposits CASCADE; +DROP TABLE IF EXISTS l2_transaction_withdrawals CASCADE; +DROP TABLE IF EXISTS l1_bridge_messages CASCADE; +DROP TABLE IF EXISTS l2_bridge_messages CASCADE; +DROP TABLE IF EXISTS l1_bridged_tokens CASCADE; +DROP TABLE IF EXISTS l2_bridged_tokens CASCADE; +DROP TABLE IF EXISTS l1_bridge_deposits CASCADE; +DROP TABLE IF EXISTS l2_bridge_withdrawals CASCADE; CREATE TABLE IF NOT EXISTS l1_block_headers ( From f5a58f73031d0c0b229b90fe62622e6ab511bb23 Mon Sep 17 00:00:00 2001 From: Ethen Pociask Date: Thu, 12 Oct 2023 17:32:36 -0700 Subject: [PATCH 14/23] [indexer.bridge_offset_fix] updated insertion query to take mint --- indexer/processors/contracts/optimism_portal.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/indexer/processors/contracts/optimism_portal.go b/indexer/processors/contracts/optimism_portal.go index 0cbacbf97ff0..42cd4e6f8079 100644 --- a/indexer/processors/contracts/optimism_portal.go +++ b/indexer/processors/contracts/optimism_portal.go @@ -76,7 +76,7 @@ func OptimismPortalTransactionDepositEvents(contractAddress common.Address, db * Tx: database.Transaction{ FromAddress: txDeposit.From, ToAddress: txDeposit.To, - Amount: depositTx.Mint, + Amount: mint, Data: depositTx.Data, Timestamp: transactionDepositEvents[i].Timestamp, }, From a86737dfd3ef8961c485586d7b9f2b790a67f460 Mon Sep 17 00:00:00 2001 From: Ethen Pociask Date: Thu, 12 Oct 2023 18:18:25 -0700 Subject: [PATCH 15/23] [indexer.bridge_offset_fix] Addressed PR feedback --- indexer/README.md | 4 +--- indexer/database/blocks.go | 9 --------- indexer/migrations/20230523_create_schema.sql | 15 --------------- 3 files changed, 1 insertion(+), 27 deletions(-) diff --git a/indexer/README.md b/indexer/README.md index 0cea037f9160..d5e6a96dad07 100644 --- a/indexer/README.md +++ b/indexer/README.md @@ -56,9 +56,7 @@ The indexer service is responsible for polling and processing real-time batches ### L1 Polling -L1 blocks are only indexed if they contain: -- L1 system contract events -- Batch commitment events +L1 blocks are only indexed if they contain L1 system contract events. This is done to reduce the amount of unnecessary data that is indexed. Because of this, the `l1_block_headers` table will not contain every L1 block header. #### API The indexer service runs a lightweight health server adjacently to the main service. The health server exposes a single endpoint `/healthz` that can be used to check the health of the indexer service. The health assessment doesn't check dependency health (ie. database) but rather checks the health of the indexer service itself. diff --git a/indexer/database/blocks.go b/indexer/database/blocks.go index 0bbb152e084a..518a029a0f42 100644 --- a/indexer/database/blocks.go +++ b/indexer/database/blocks.go @@ -7,7 +7,6 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/log" "gorm.io/gorm" ) @@ -177,8 +176,6 @@ func (db *blocksDB) LatestObservedEpoch(fromL1Height *big.Int, maxL1Range uint64 // TODO - Embed logging to db if result.Error != nil { if errors.Is(result.Error, gorm.ErrRecordNotFound) { - log.Warn("Could not fetch latest provided L1 block header in bridge processor", "number", fromL1Height, - "processor", "bridge") return nil, nil } return nil, result.Error @@ -189,8 +186,6 @@ func (db *blocksDB) LatestObservedEpoch(fromL1Height *big.Int, maxL1Range uint64 result := db.gorm.Order("number desc").Take(&header) if result.Error != nil { if errors.Is(result.Error, gorm.ErrRecordNotFound) { - log.Warn("Could not fetch recent L1 block header", "number", fromL1Height, - "processor", "bridge") return nil, nil } return nil, result.Error @@ -212,8 +207,6 @@ func (db *blocksDB) LatestObservedEpoch(fromL1Height *big.Int, maxL1Range uint64 result := db.gorm.Where(l1QueryFilter).Order("timestamp DESC").Take(&l1Header) if result.Error != nil { if errors.Is(result.Error, gorm.ErrRecordNotFound) { - log.Warn("Could not fetch latest L1 block header in bridge processor", "from_timestamp", - fromTimestamp, "max_l1_range", maxL1Range, "processor", "bridge") return nil, nil } return nil, result.Error @@ -226,7 +219,6 @@ func (db *blocksDB) LatestObservedEpoch(fromL1Height *big.Int, maxL1Range uint64 result = db.gorm.Where("timestamp <= ?", toTimestamp).Order("timestamp DESC").Take(&l2Header) if result.Error != nil { if errors.Is(result.Error, gorm.ErrRecordNotFound) { - log.Warn("Could not fetch latest L2 block header in bridge processor", "processor", "bridge") return nil, nil } return nil, result.Error @@ -249,7 +241,6 @@ func (db *blocksDB) LatestObservedEpoch(fromL1Height *big.Int, maxL1Range uint64 result := query.Take(&epoch) if result.Error != nil { if errors.Is(result.Error, gorm.ErrRecordNotFound) { - log.Warn("Could not fetch latest observed epoch in bridge processor", "processor", "bridge") return nil, nil } return nil, result.Error diff --git a/indexer/migrations/20230523_create_schema.sql b/indexer/migrations/20230523_create_schema.sql index bd1599dd0012..b13eb9dc6fab 100644 --- a/indexer/migrations/20230523_create_schema.sql +++ b/indexer/migrations/20230523_create_schema.sql @@ -16,21 +16,6 @@ END $$; * BLOCK DATA */ - -DROP TABLE IF EXISTS l1_block_headers CASCADE; -DROP TABLE IF EXISTS l2_block_headers CASCADE; -DROP TABLE IF EXISTS l1_contract_events CASCADE; -DROP TABLE IF EXISTS l2_contract_events CASCADE; -DROP TABLE IF EXISTS l1_transaction_deposits CASCADE; -DROP TABLE IF EXISTS l2_transaction_withdrawals CASCADE; -DROP TABLE IF EXISTS l1_bridge_messages CASCADE; -DROP TABLE IF EXISTS l2_bridge_messages CASCADE; -DROP TABLE IF EXISTS l1_bridged_tokens CASCADE; -DROP TABLE IF EXISTS l2_bridged_tokens CASCADE; -DROP TABLE IF EXISTS l1_bridge_deposits CASCADE; -DROP TABLE IF EXISTS l2_bridge_withdrawals CASCADE; - - CREATE TABLE IF NOT EXISTS l1_block_headers ( -- Searchable fields hash VARCHAR PRIMARY KEY, From 38f74230dc5b19a5eb1da22a73c96c0c1977b11b Mon Sep 17 00:00:00 2001 From: Ethen Pociask Date: Thu, 12 Oct 2023 18:18:37 -0700 Subject: [PATCH 16/23] [indexer.bridge_offset_fix] Remove .vscode launch file --- .vscode/launch.json | 22 ---------------------- 1 file changed, 22 deletions(-) delete mode 100644 .vscode/launch.json diff --git a/.vscode/launch.json b/.vscode/launch.json deleted file mode 100644 index 27d5932f59b1..000000000000 --- a/.vscode/launch.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - // Use IntelliSense to learn about possible attributes. - // Hover to view descriptions of existing attributes. - // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 - "version": "0.2.0", - "configurations": [ - { - - "type": "go", - "request": "launch", - "name": "Launch Program", - "program": "${workspaceFolder}/indexer/cmd/indexer", - "mode": "debug", - "env": { - "INDEXER_CONFIG": "/Users/ethen.pociask@coinbase.com/coinbase/indexer.toml" - }, - "args": [ - "index" - ] - } - ] -} From a20a742acfe7ac1d51622cf05b3bd22e1aeefc40 Mon Sep 17 00:00:00 2001 From: Ethen Pociask Date: Thu, 12 Oct 2023 18:21:37 -0700 Subject: [PATCH 17/23] [indexer.bridge_offset_fix] debug logs in bridge.go --- indexer/processors/bridge.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/indexer/processors/bridge.go b/indexer/processors/bridge.go index 1e408b176fc7..7370ac024e5b 100644 --- a/indexer/processors/bridge.go +++ b/indexer/processors/bridge.go @@ -169,17 +169,21 @@ func (b *BridgeProcessor) run() error { // First, find all possible initiated bridge events if err := bridge.LegacyL1ProcessInitiatedBridgeEvents(l1BridgeLog, tx, b.metrics, b.chainConfig.L1Contracts, legacyFromL1Height, legacyToL1Height); err != nil { + batchLog.Error("failed to index legacy l1 initiated bridge events", "err", err) return err } if err := bridge.LegacyL2ProcessInitiatedBridgeEvents(l2BridgeLog, tx, b.metrics, b.chainConfig.L2Contracts, legacyFromL2Height, legacyToL2Height); err != nil { + batchLog.Error("failed to index legacy l2 initiated bridge events", "err", err) return err } // Now that all initiated events have been indexed, it is ensured that all finalization can find their counterpart. if err := bridge.LegacyL1ProcessFinalizedBridgeEvents(l1BridgeLog, tx, b.metrics, b.l1Etl.EthClient, b.chainConfig.L1Contracts, legacyFromL1Height, legacyToL1Height); err != nil { + batchLog.Error("failed to index legacy l1 finalized bridge events", "err", err) return err } if err := bridge.LegacyL2ProcessFinalizedBridgeEvents(l2BridgeLog, tx, b.metrics, b.chainConfig.L2Contracts, legacyFromL2Height, legacyToL2Height); err != nil { + batchLog.Error("failed to index legacy l2 finalized bridge events", "err", err) return err } From 62461c0dd05bb24085cdc036fbc6985e3276f12e Mon Sep 17 00:00:00 2001 From: Ethen Pociask Date: Thu, 12 Oct 2023 18:21:48 -0700 Subject: [PATCH 18/23] [indexer.bridge_offset_fix] debug logs in bridge.go --- indexer/processors/bridge.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/indexer/processors/bridge.go b/indexer/processors/bridge.go index 7370ac024e5b..3471b550a961 100644 --- a/indexer/processors/bridge.go +++ b/indexer/processors/bridge.go @@ -183,7 +183,7 @@ func (b *BridgeProcessor) run() error { return err } if err := bridge.LegacyL2ProcessFinalizedBridgeEvents(l2BridgeLog, tx, b.metrics, b.chainConfig.L2Contracts, legacyFromL2Height, legacyToL2Height); err != nil { - batchLog.Error("failed to index legacy l2 finalized bridge events", "err", err) + batchLog.Error("failed to index legacy l2l finalized bridge events", "err", err) return err } From 193c80ee19aede3768a579cf6a4cecf0e827f7e3 Mon Sep 17 00:00:00 2001 From: Ethen Pociask Date: Thu, 12 Oct 2023 22:24:07 -0700 Subject: [PATCH 19/23] [indexer.bridge_offset_fix] Output chain config --- indexer/config/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/indexer/config/config.go b/indexer/config/config.go index f35fbbffe206..8cef5058007b 100644 --- a/indexer/config/config.go +++ b/indexer/config/config.go @@ -210,6 +210,6 @@ func LoadConfig(log log.Logger, path string) (Config, error) { cfg.Chain.L2HeaderBufferSize = defaultHeaderBufferSize } - log.Info("loaded config") + log.Info("loaded config", "config", cfg.Chain) return cfg, nil } From ae838bd78c3deac8b350713debd4bd3bbd841e9e Mon Sep 17 00:00:00 2001 From: Ethen Pociask Date: Thu, 12 Oct 2023 22:54:16 -0700 Subject: [PATCH 20/23] [indexer.bridge_offset_fix] testing purge --- indexer/migrations/20230523_create_schema.sql | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/indexer/migrations/20230523_create_schema.sql b/indexer/migrations/20230523_create_schema.sql index b13eb9dc6fab..bd1599dd0012 100644 --- a/indexer/migrations/20230523_create_schema.sql +++ b/indexer/migrations/20230523_create_schema.sql @@ -16,6 +16,21 @@ END $$; * BLOCK DATA */ + +DROP TABLE IF EXISTS l1_block_headers CASCADE; +DROP TABLE IF EXISTS l2_block_headers CASCADE; +DROP TABLE IF EXISTS l1_contract_events CASCADE; +DROP TABLE IF EXISTS l2_contract_events CASCADE; +DROP TABLE IF EXISTS l1_transaction_deposits CASCADE; +DROP TABLE IF EXISTS l2_transaction_withdrawals CASCADE; +DROP TABLE IF EXISTS l1_bridge_messages CASCADE; +DROP TABLE IF EXISTS l2_bridge_messages CASCADE; +DROP TABLE IF EXISTS l1_bridged_tokens CASCADE; +DROP TABLE IF EXISTS l2_bridged_tokens CASCADE; +DROP TABLE IF EXISTS l1_bridge_deposits CASCADE; +DROP TABLE IF EXISTS l2_bridge_withdrawals CASCADE; + + CREATE TABLE IF NOT EXISTS l1_block_headers ( -- Searchable fields hash VARCHAR PRIMARY KEY, From dd0f9ccfe5f610a39b369b5cadf9f4a755c43ce9 Mon Sep 17 00:00:00 2001 From: Ethen Pociask Date: Fri, 13 Oct 2023 11:03:18 -0700 Subject: [PATCH 21/23] [indexer.bridge_offset_fix] Return on duplicate insertion errors --- indexer/etl/l1_etl.go | 11 +++++++++-- indexer/node/header_traversal.go | 1 + 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/indexer/etl/l1_etl.go b/indexer/etl/l1_etl.go index a4613928b077..1324c607b558 100644 --- a/indexer/etl/l1_etl.go +++ b/indexer/etl/l1_etl.go @@ -122,7 +122,7 @@ func (l1Etl *L1ETL) Start(ctx context.Context) error { // Continually try to persist this batch. If it fails after 10 attempts, we simply error out retryStrategy := &retry.ExponentialStrategy{Min: 1000, Max: 20_000, MaxJitter: 250} if _, err := retry.Do[interface{}](ctx, 10, retryStrategy, func() (interface{}, error) { - if err := l1Etl.db.Transaction(func(tx *database.DB) error { + err := l1Etl.db.Transaction(func(tx *database.DB) error { if err := tx.Blocks.StoreL1BlockHeaders(l1BlockHeaders); err != nil { return err } @@ -131,7 +131,14 @@ func (l1Etl *L1ETL) Start(ctx context.Context) error { return err } return nil - }); err != nil { + }) + + if err != nil && strings.Contains(err.Error(), "duplicate key value violates unique constraint") { + batch.Logger.Warn("duplicate key error, ignoring insertion", "err", err) + return nil, nil + } + + if err != nil { batch.Logger.Error("unable to persist batch", "err", err) return nil, err } diff --git a/indexer/node/header_traversal.go b/indexer/node/header_traversal.go index e925747f2185..5a79d0cbbc41 100644 --- a/indexer/node/header_traversal.go +++ b/indexer/node/header_traversal.go @@ -15,6 +15,7 @@ var ( ) type HeaderTraversal struct { + started bool ethClient EthClient lastHeader *types.Header From d60548802b4056b3d8eb659911e99c4880e77b30 Mon Sep 17 00:00:00 2001 From: Ethen Pociask Date: Fri, 13 Oct 2023 13:40:56 -0700 Subject: [PATCH 22/23] [indexer.bridge_offset_fix] Addressing PR feedback --- indexer/etl/l1_etl.go | 11 ++--------- indexer/migrations/20230523_create_schema.sql | 15 --------------- indexer/node/header_traversal.go | 1 - indexer/processors/contracts/optimism_portal.go | 3 ++- 4 files changed, 4 insertions(+), 26 deletions(-) diff --git a/indexer/etl/l1_etl.go b/indexer/etl/l1_etl.go index 1324c607b558..a4613928b077 100644 --- a/indexer/etl/l1_etl.go +++ b/indexer/etl/l1_etl.go @@ -122,7 +122,7 @@ func (l1Etl *L1ETL) Start(ctx context.Context) error { // Continually try to persist this batch. If it fails after 10 attempts, we simply error out retryStrategy := &retry.ExponentialStrategy{Min: 1000, Max: 20_000, MaxJitter: 250} if _, err := retry.Do[interface{}](ctx, 10, retryStrategy, func() (interface{}, error) { - err := l1Etl.db.Transaction(func(tx *database.DB) error { + if err := l1Etl.db.Transaction(func(tx *database.DB) error { if err := tx.Blocks.StoreL1BlockHeaders(l1BlockHeaders); err != nil { return err } @@ -131,14 +131,7 @@ func (l1Etl *L1ETL) Start(ctx context.Context) error { return err } return nil - }) - - if err != nil && strings.Contains(err.Error(), "duplicate key value violates unique constraint") { - batch.Logger.Warn("duplicate key error, ignoring insertion", "err", err) - return nil, nil - } - - if err != nil { + }); err != nil { batch.Logger.Error("unable to persist batch", "err", err) return nil, err } diff --git a/indexer/migrations/20230523_create_schema.sql b/indexer/migrations/20230523_create_schema.sql index bd1599dd0012..b13eb9dc6fab 100644 --- a/indexer/migrations/20230523_create_schema.sql +++ b/indexer/migrations/20230523_create_schema.sql @@ -16,21 +16,6 @@ END $$; * BLOCK DATA */ - -DROP TABLE IF EXISTS l1_block_headers CASCADE; -DROP TABLE IF EXISTS l2_block_headers CASCADE; -DROP TABLE IF EXISTS l1_contract_events CASCADE; -DROP TABLE IF EXISTS l2_contract_events CASCADE; -DROP TABLE IF EXISTS l1_transaction_deposits CASCADE; -DROP TABLE IF EXISTS l2_transaction_withdrawals CASCADE; -DROP TABLE IF EXISTS l1_bridge_messages CASCADE; -DROP TABLE IF EXISTS l2_bridge_messages CASCADE; -DROP TABLE IF EXISTS l1_bridged_tokens CASCADE; -DROP TABLE IF EXISTS l2_bridged_tokens CASCADE; -DROP TABLE IF EXISTS l1_bridge_deposits CASCADE; -DROP TABLE IF EXISTS l2_bridge_withdrawals CASCADE; - - CREATE TABLE IF NOT EXISTS l1_block_headers ( -- Searchable fields hash VARCHAR PRIMARY KEY, diff --git a/indexer/node/header_traversal.go b/indexer/node/header_traversal.go index 5a79d0cbbc41..e925747f2185 100644 --- a/indexer/node/header_traversal.go +++ b/indexer/node/header_traversal.go @@ -15,7 +15,6 @@ var ( ) type HeaderTraversal struct { - started bool ethClient EthClient lastHeader *types.Header diff --git a/indexer/processors/contracts/optimism_portal.go b/indexer/processors/contracts/optimism_portal.go index 42cd4e6f8079..ea56714c4b09 100644 --- a/indexer/processors/contracts/optimism_portal.go +++ b/indexer/processors/contracts/optimism_portal.go @@ -4,6 +4,7 @@ import ( "errors" "math/big" + "github.com/ethereum-optimism/optimism/indexer/bigint" "github.com/ethereum-optimism/optimism/indexer/database" "github.com/ethereum-optimism/optimism/op-bindings/bindings" "github.com/ethereum-optimism/optimism/op-node/rollup/derive" @@ -66,7 +67,7 @@ func OptimismPortalTransactionDepositEvents(contractAddress common.Address, db * mint := depositTx.Mint if mint == nil { - mint = big.NewInt(0) + mint = bigint.Zero } optimismPortalTxDeposits[i] = OptimismPortalTransactionDepositEvent{ From b0b37b2428ff8fa5fb8a9e37ba276d58c07dc0cc Mon Sep 17 00:00:00 2001 From: Ethen Pociask Date: Fri, 13 Oct 2023 13:42:42 -0700 Subject: [PATCH 23/23] [indexer.bridge_offset_fix] Update blocks.go --- indexer/database/blocks.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/indexer/database/blocks.go b/indexer/database/blocks.go index 518a029a0f42..f1afcbc2c893 100644 --- a/indexer/database/blocks.go +++ b/indexer/database/blocks.go @@ -169,7 +169,7 @@ func (db *blocksDB) LatestObservedEpoch(fromL1Height *big.Int, maxL1Range uint64 // We use timestamps since that translates to both L1 & L2 var fromTimestamp, toTimestamp uint64 - // Lower Bound (the default `fromTimestamp = l1_starting_heigh` (default=0) suffices genesis representation) + // Lower Bound (the default `fromTimestamp = l1_starting_height` (default=0) suffices genesis representation) var header L1BlockHeader if fromL1Height != nil { result := db.gorm.Where("number = ?", fromL1Height).Take(&header)