From 882e17ae9687856daf1faa35d5d89ce7deabbe73 Mon Sep 17 00:00:00 2001 From: yutianwu Date: Thu, 25 Apr 2024 14:40:54 +0800 Subject: [PATCH] feat: implement fast node (#75) Co-authored-by: Owen <103096885+owen-reorg@users.noreply.github.com> --- build/ci.go | 3 +- cmd/geth/main.go | 1 + cmd/geth/snapshot.go | 72 ++++++++++++++- cmd/utils/flags.go | 15 +++- core/blockchain.go | 59 ++++++++----- core/blockchain_reader.go | 19 +++- core/genesis.go | 7 +- core/state/database.go | 20 +++++ core/state/pruner/pruner.go | 100 +++++++++++++++++++++ core/state/snapshot/journal.go | 5 +- core/state/snapshot/snapshot.go | 3 +- core/state/state_object.go | 7 ++ core/state/statedb.go | 150 +++++++++++++++++++++----------- core/state/statedb_test.go | 7 +- core/state_processor.go | 1 + eth/backend.go | 1 + eth/ethconfig/config.go | 2 + eth/ethconfig/gen_config.go | 6 ++ eth/handler.go | 2 +- eth/sync.go | 2 +- go.mod | 7 +- go.sum | 6 +- light/trie.go | 8 ++ miner/worker.go | 25 +++--- tests/state_test_util.go | 3 +- trie/database.go | 5 ++ trie/dummy_trie.go | 88 +++++++++++++++++++ 27 files changed, 521 insertions(+), 103 deletions(-) create mode 100644 trie/dummy_trie.go diff --git a/build/ci.go b/build/ci.go index 1e8b6f9d93..9845c0f293 100644 --- a/build/ci.go +++ b/build/ci.go @@ -53,6 +53,7 @@ import ( "time" "github.com/cespare/cp" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto/signify" "github.com/ethereum/go-ethereum/internal/build" @@ -304,7 +305,7 @@ func doTest(cmdline []string) { gotest := tc.Go("test") // CI needs a bit more time for the statetests (default 10m). - gotest.Args = append(gotest.Args, "-timeout=20m") + gotest.Args = append(gotest.Args, "-timeout=50m") // Enable CKZG backend in CI. gotest.Args = append(gotest.Args, "-tags=ckzg") diff --git a/cmd/geth/main.go b/cmd/geth/main.go index 050e3514bf..e74cdcdfb4 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -115,6 +115,7 @@ var ( utils.CacheSnapshotFlag, utils.CacheNoPrefetchFlag, utils.CachePreimagesFlag, + utils.AllowInsecureNoTriesFlag, utils.CacheLogSizeFlag, utils.FDLimitFlag, utils.CryptoKZGFlag, diff --git a/cmd/geth/snapshot.go b/cmd/geth/snapshot.go index 6413482511..04a9ded52d 100644 --- a/cmd/geth/snapshot.go +++ b/cmd/geth/snapshot.go @@ -23,19 +23,23 @@ import ( "os" "time" + cli "github.com/urfave/cli/v2" + "github.com/ethereum/go-ethereum/cmd/utils" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/state/pruner" "github.com/ethereum/go-ethereum/core/state/snapshot" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/eth/ethconfig" "github.com/ethereum/go-ethereum/internal/flags" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" - cli "github.com/urfave/cli/v2" ) var ( @@ -147,6 +151,29 @@ as the backend data source, making this command a lot faster. The argument is interpreted as block number or hash. If none is provided, the latest block is used. +`, + }, + { + Name: "insecure-prune-all", + Usage: "Prune all trie state data except genesis block, it will break storage for fullnode, only suitable for fast node " + + "who do not need trie storage at all", + ArgsUsage: "", + Action: pruneAllState, + Category: "MISCELLANEOUS COMMANDS", + Flags: []cli.Flag{ + utils.DataDirFlag, + utils.AncientFlag, + }, + Description: ` +will prune all historical trie state data except genesis block. +All trie nodes will be deleted from the database. + +It expects the genesis file as argument. + +WARNING: It's necessary to delete the trie clean cache after the pruning. +If you specify another directory for the trie clean cache via "--cache.trie.journal" +during the use of Geth, please also specify it here for correct deletion. Otherwise +the trie clean cache with default directory will be deleted. `, }, }, @@ -635,3 +662,46 @@ func checkAccount(ctx *cli.Context) error { log.Info("Checked the snapshot journalled storage", "time", common.PrettyDuration(time.Since(start))) return nil } + +func pruneAllState(ctx *cli.Context) error { + stack, _ := makeConfigNode(ctx) + defer stack.Close() + + genesisPath := ctx.Args().First() + if len(genesisPath) == 0 { + utils.Fatalf("Must supply path to genesis JSON file") + } + file, err := os.Open(genesisPath) + if err != nil { + utils.Fatalf("Failed to read genesis file: %v", err) + } + defer file.Close() + + g := new(core.Genesis) + if err := json.NewDecoder(file).Decode(g); err != nil { + cfg := gethConfig{ + Eth: ethconfig.Defaults, + Node: defaultNodeConfig(), + Metrics: metrics.DefaultConfig, + } + + // Load config file. + if err := loadConfig(genesisPath, &cfg); err != nil { + utils.Fatalf("%v", err) + } + g = cfg.Eth.Genesis + } + + chaindb := utils.MakeChainDatabase(ctx, stack, false) + defer chaindb.Close() + pruner, err := pruner.NewAllPruner(chaindb) + if err != nil { + log.Error("Failed to open snapshot tree", "err", err) + return err + } + if err = pruner.PruneAll(g); err != nil { + log.Error("Failed to prune state", "err", err) + return err + } + return nil +} diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index a3a4641641..f52f418e31 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -35,6 +35,10 @@ import ( "strings" "time" + pcsclite "github.com/gballet/go-libpcsclite" + gopsutil "github.com/shirou/gopsutil/mem" + "github.com/urfave/cli/v2" + "github.com/ethereum/go-ethereum/accounts" "github.com/ethereum/go-ethereum/accounts/keystore" "github.com/ethereum/go-ethereum/common" @@ -74,9 +78,6 @@ import ( "github.com/ethereum/go-ethereum/trie" "github.com/ethereum/go-ethereum/trie/triedb/hashdb" "github.com/ethereum/go-ethereum/trie/triedb/pathdb" - pcsclite "github.com/gballet/go-libpcsclite" - gopsutil "github.com/shirou/gopsutil/mem" - "github.com/urfave/cli/v2" ) // These are all the command line flags we support. @@ -269,6 +270,11 @@ var ( Value: 2048, Category: flags.EthCategory, } + AllowInsecureNoTriesFlag = &cli.BoolFlag{ + Name: "allow-insecure-no-tries", + Usage: `Disable the tries state root verification, the state consistency is no longer 100% guaranteed. Do not enable it unless you know exactly what the consequence it will cause.`, + Category: flags.EthCategory, + } OverrideCancun = &cli.Uint64Flag{ Name: "override.cancun", Usage: "Manually specify the Cancun fork timestamp, overriding the bundled setting", @@ -1906,6 +1912,9 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { if ctx.IsSet(CacheLogSizeFlag.Name) { cfg.FilterLogCacheSize = ctx.Int(CacheLogSizeFlag.Name) } + if ctx.IsSet(AllowInsecureNoTriesFlag.Name) { + cfg.NoTries = ctx.Bool(AllowInsecureNoTriesFlag.Name) + } if !ctx.Bool(SnapshotFlag.Name) { // If snap-sync is requested, this flag is also required if cfg.SyncMode == downloader.SnapSync { diff --git a/core/blockchain.go b/core/blockchain.go index 1dd7490ee3..d2664fcf98 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -28,6 +28,8 @@ import ( "sync/atomic" "time" + "golang.org/x/exp/slices" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/lru" "github.com/ethereum/go-ethereum/common/mclock" @@ -50,7 +52,6 @@ import ( "github.com/ethereum/go-ethereum/trie" "github.com/ethereum/go-ethereum/trie/triedb/hashdb" "github.com/ethereum/go-ethereum/trie/triedb/pathdb" - "golang.org/x/exp/slices" ) var ( @@ -83,13 +84,13 @@ var ( blockExecutionTimer = metrics.NewRegisteredTimer("chain/execution", nil) blockWriteTimer = metrics.NewRegisteredTimer("chain/write", nil) - blockWriteExternalTimer = metrics.NewRegisteredTimer("chain/block/write/external", nil) - stateCommitExternalTimer = metrics.NewRegisteredTimer("chain/state/commit/external", nil) + blockWriteExternalTimer = metrics.NewRegisteredTimer("chain/block/write/external", nil) + stateCommitExternalTimer = metrics.NewRegisteredTimer("chain/state/commit/external", nil) triedbCommitExternalTimer = metrics.NewRegisteredTimer("chain/triedb/commit/external", nil) - innerExecutionTimer = metrics.NewRegisteredTimer("chain/inner/execution", nil) + innerExecutionTimer = metrics.NewRegisteredTimer("chain/inner/execution", nil) blockGasUsedGauge = metrics.NewRegisteredGauge("chain/block/gas/used", nil) - mgaspsGauge = metrics.NewRegisteredGauge("chain/mgas/ps", nil) + mgaspsGauge = metrics.NewRegisteredGauge("chain/mgas/ps", nil) blockReorgMeter = metrics.NewRegisteredMeter("chain/reorg/executes", nil) blockReorgAddMeter = metrics.NewRegisteredMeter("chain/reorg/add", nil) @@ -152,6 +153,7 @@ type CacheConfig struct { TrieTimeLimit time.Duration // Time limit after which to flush the current in-memory trie to disk SnapshotLimit int // Memory allowance (MB) to use for caching snapshot entries in memory Preimages bool // Whether to store preimage of trie key to the disk + NoTries bool // Insecure settings. Do not have any tries in databases if enabled. StateHistory uint64 // Number of blocks from head whose state histories are reserved. StateScheme string // Scheme used to store ethereum states and merkle tree nodes on top PathNodeBuffer pathdb.NodeBufferType // Type of trienodebuffer to cache trie nodes in disklayer @@ -164,7 +166,10 @@ type CacheConfig struct { // triedbConfig derives the configures for trie database. func (c *CacheConfig) triedbConfig() *trie.Config { - config := &trie.Config{Preimages: c.Preimages} + config := &trie.Config{ + Preimages: c.Preimages, + NoTries: c.NoTries, + } if c.StateScheme == rawdb.HashScheme { config.HashDB = &hashdb.Config{ CleanCacheSize: c.TrieCleanLimit * 1024 * 1024, @@ -367,7 +372,7 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, genesis *Genesis // Make sure the state associated with the block is available, or log out // if there is no available state, waiting for state sync. head := bc.CurrentBlock() - if !bc.HasState(head.Root) { + if !bc.NoTries() && !bc.HasState(head.Root) { if head.Number.Uint64() == 0 { // The genesis state is missing, which is only possible in the path-based // scheme. This situation occurs when the initial state sync is not finished @@ -478,6 +483,7 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, genesis *Genesis Recovery: recover, NoBuild: bc.cacheConfig.SnapshotNoBuild, AsyncBuild: !bc.cacheConfig.SnapshotWait, + NoTries: bc.stateCache.NoTries(), } bc.snaps, _ = snapshot.New(snapconfig, bc.db, bc.triedb, head.Root) } @@ -862,7 +868,7 @@ func (bc *BlockChain) SnapSyncCommitHead(hash common.Hash) error { return err } } - if !bc.HasState(root) { + if !bc.NoTries() && !bc.HasState(root) { return fmt.Errorf("non existent state [%x..]", root[:4]) } // If all checks out, manually set the head block. @@ -1460,9 +1466,10 @@ func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types. } blockWriteExternalTimer.UpdateSince(start) log.Debug("blockWriteExternalTimer", "duration", common.PrettyDuration(time.Since(start)), "hash", block.Hash()) - + // Commit all cached state changes into underlying memory database. start = time.Now() + state.SetExpectedStateRoot(block.Root()) root, err := state.Commit(block.NumberU64(), bc.chainConfig.IsEIP158(block.Number())) if err != nil { return err @@ -1477,10 +1484,10 @@ func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types. } // If we're running an archive node, always flush start = time.Now() - defer func () { + defer func() { triedbCommitExternalTimer.UpdateSince(start) log.Debug("triedbCommitExternalTimer", "duration", common.PrettyDuration(time.Since(start)), "hash", block.Hash()) - } () + }() if bc.cacheConfig.TrieDirtyDisabled { return bc.triedb.Commit(root, false) } @@ -1785,7 +1792,7 @@ func (bc *BlockChain) insertChain(chain types.Blocks, setHead bool) (int, error) } }() - defer func () { + defer func() { DebugInnerExecutionDuration = 0 }() for ; block != nil && err == nil || errors.Is(err, ErrKnownBlock); block, err = it.next() { @@ -1887,6 +1894,8 @@ func (bc *BlockChain) insertChain(chain types.Blocks, setHead bool) (int, error) } } + statedb.SetExpectedStateRoot(block.Root()) + // Process block using the parent state as reference point pstart = time.Now() receipts, logs, usedGas, err = bc.processor.Process(block, statedb, bc.vmConfig) @@ -1908,16 +1917,16 @@ func (bc *BlockChain) insertChain(chain types.Blocks, setHead bool) (int, error) proctime := time.Since(start) // processing + validation // Update the metrics touched during block processing and validation - accountReadTimer.Update(statedb.AccountReads) // Account reads are complete(in processing) - storageReadTimer.Update(statedb.StorageReads) // Storage reads are complete(in processing) - snapshotAccountReadTimer.Update(statedb.SnapshotAccountReads) // Account reads are complete(in processing) - snapshotStorageReadTimer.Update(statedb.SnapshotStorageReads) // Storage reads are complete(in processing) - accountUpdateTimer.Update(statedb.AccountUpdates) // Account updates are complete(in validation) - storageUpdateTimer.Update(statedb.StorageUpdates) // Storage updates are complete(in validation) - accountHashTimer.Update(statedb.AccountHashes) // Account hashes are complete(in validation) - storageHashTimer.Update(statedb.StorageHashes) // Storage hashes are complete(in validation) - blockExecutionTimer.Update(ptime) // The time spent on block execution - blockValidationTimer.Update(vtime) // The time spent on block validation + accountReadTimer.Update(statedb.AccountReads) // Account reads are complete(in processing) + storageReadTimer.Update(statedb.StorageReads) // Storage reads are complete(in processing) + snapshotAccountReadTimer.Update(statedb.SnapshotAccountReads) // Account reads are complete(in processing) + snapshotStorageReadTimer.Update(statedb.SnapshotStorageReads) // Storage reads are complete(in processing) + accountUpdateTimer.Update(statedb.AccountUpdates) // Account updates are complete(in validation) + storageUpdateTimer.Update(statedb.StorageUpdates) // Storage updates are complete(in validation) + accountHashTimer.Update(statedb.AccountHashes) // Account hashes are complete(in validation) + storageHashTimer.Update(statedb.StorageHashes) // Storage hashes are complete(in validation) + blockExecutionTimer.Update(ptime) // The time spent on block execution + blockValidationTimer.Update(vtime) // The time spent on block validation innerExecutionTimer.Update(DebugInnerExecutionDuration) @@ -1959,7 +1968,7 @@ func (bc *BlockChain) insertChain(chain types.Blocks, setHead bool) (int, error) } trieDiffNodes, trieBufNodes, trieImmutableBufNodes, _ := bc.triedb.Size() stats.report(chain, it.index, snapDiffItems, snapBufItems, trieDiffNodes, trieBufNodes, trieImmutableBufNodes, setHead) - blockGasUsedGauge.Update(int64(block.GasUsed())/1000000) + blockGasUsedGauge.Update(int64(block.GasUsed()) / 1000000) if !setHead { // After merge we expect few side chains. Simply count @@ -2675,3 +2684,7 @@ func (bc *BlockChain) SetTrieFlushInterval(interval time.Duration) { func (bc *BlockChain) GetTrieFlushInterval() time.Duration { return time.Duration(bc.flushInterval.Load()) } + +func (bc *BlockChain) NoTries() bool { + return bc.stateCache.NoTries() +} diff --git a/core/blockchain_reader.go b/core/blockchain_reader.go index 30fbcb883b..e98f462380 100644 --- a/core/blockchain_reader.go +++ b/core/blockchain_reader.go @@ -17,6 +17,7 @@ package core import ( + "errors" "math/big" "github.com/ethereum/go-ethereum/common" @@ -278,6 +279,9 @@ func (bc *BlockChain) GetTd(hash common.Hash, number uint64) *big.Int { // HasState checks if state trie is fully present in the database or not. func (bc *BlockChain) HasState(hash common.Hash) bool { + if bc.NoTries() { + return bc.snaps != nil && bc.snaps.Snapshot(hash) != nil + } _, err := bc.stateCache.OpenTrie(hash) return err == nil } @@ -334,7 +338,20 @@ func (bc *BlockChain) State() (*state.StateDB, error) { // StateAt returns a new mutable state based on a particular point in time. func (bc *BlockChain) StateAt(root common.Hash) (*state.StateDB, error) { - return state.New(root, bc.stateCache, bc.snaps) + stateDb, err := state.New(root, bc.stateCache, bc.snaps) + if err != nil { + return nil, err + } + + // If there's no trie and the specified snapshot is not available, getting + // any state will by default return nil. + // Instead of that, it will be more useful to return an error to indicate + // the state is not available. + if stateDb.NoTrie() && stateDb.GetSnap() == nil { + return nil, errors.New("state is not available") + } + + return stateDb, err } // Config retrieves the chain's fork configuration. diff --git a/core/genesis.go b/core/genesis.go index 6f924ab65a..c0bc087a79 100644 --- a/core/genesis.go +++ b/core/genesis.go @@ -162,6 +162,11 @@ func (ga *GenesisAlloc) hash() (common.Hash, error) { // states will be persisted into the given database. Also, the genesis state // specification will be flushed as well. func (ga *GenesisAlloc) flush(db ethdb.Database, triedb *trie.Database, blockhash common.Hash) error { + triedbConfig := triedb.Config() + if triedbConfig != nil { + triedbConfig.NoTries = false + } + statedb, err := state.New(types.EmptyRootHash, state.NewDatabaseWithNodeDB(db, triedb), nil) if err != nil { return err @@ -346,7 +351,7 @@ func SetupGenesisBlockWithOverride(db ethdb.Database, triedb *trie.Database, gen // is initialized with an external ancient store. Commit genesis state // in this case. header := rawdb.ReadHeader(db, stored, 0) - if header.Root != types.EmptyRootHash && !triedb.Initialized(header.Root) { + if header.Root != types.EmptyRootHash && !triedb.Initialized(header.Root) && !triedb.Config().NoTries { if genesis == nil { genesis = DefaultGenesisBlock() } diff --git a/core/state/database.go b/core/state/database.go index 9467c8f72e..e18d7b9b69 100644 --- a/core/state/database.go +++ b/core/state/database.go @@ -60,6 +60,9 @@ type Database interface { // TrieDB returns the underlying trie database for managing trie nodes. TrieDB() *trie.Database + + // NoTries returns whether the database has tries storage. + NoTries() bool } // Trie is a Ethereum Merkle Patricia trie. @@ -148,6 +151,7 @@ func NewDatabaseWithConfig(db ethdb.Database, config *trie.Config) Database { codeSizeCache: lru.NewCache[common.Hash, int](codeSizeCacheSize), codeCache: lru.NewSizeConstrainedCache[common.Hash, []byte](codeCacheSize), triedb: trie.NewDatabase(db, config), + noTries: config != nil && config.NoTries, } } @@ -158,6 +162,7 @@ func NewDatabaseWithNodeDB(db ethdb.Database, triedb *trie.Database) Database { codeSizeCache: lru.NewCache[common.Hash, int](codeSizeCacheSize), codeCache: lru.NewSizeConstrainedCache[common.Hash, []byte](codeCacheSize), triedb: triedb, + noTries: triedb != nil && triedb.Config() != nil && triedb.Config().NoTries, } } @@ -166,10 +171,15 @@ type cachingDB struct { codeSizeCache *lru.Cache[common.Hash, int] codeCache *lru.SizeConstrainedCache[common.Hash, []byte] triedb *trie.Database + noTries bool } // OpenTrie opens the main account trie at a specific root hash. func (db *cachingDB) OpenTrie(root common.Hash) (Trie, error) { + if db.noTries { + return trie.NewEmptyTrie(), nil + } + tr, err := trie.NewStateTrie(trie.StateTrieID(root), db.triedb) if err != nil { return nil, err @@ -179,6 +189,10 @@ func (db *cachingDB) OpenTrie(root common.Hash) (Trie, error) { // OpenStorageTrie opens the storage trie of an account. func (db *cachingDB) OpenStorageTrie(stateRoot common.Hash, address common.Address, root common.Hash) (Trie, error) { + if db.noTries { + return trie.NewEmptyTrie(), nil + } + tr, err := trie.NewStateTrie(trie.StorageTrieID(stateRoot, crypto.Keccak256Hash(address.Bytes()), root), db.triedb) if err != nil { return nil, err @@ -191,6 +205,8 @@ func (db *cachingDB) CopyTrie(t Trie) Trie { switch t := t.(type) { case *trie.StateTrie: return t.Copy() + case *trie.EmptyTrie: + return t.Copy() default: panic(fmt.Errorf("unknown trie type %T", t)) } @@ -246,3 +262,7 @@ func (db *cachingDB) DiskDB() ethdb.KeyValueStore { func (db *cachingDB) TrieDB() *trie.Database { return db.triedb } + +func (db *cachingDB) NoTries() bool { + return db.noTries +} diff --git a/core/state/pruner/pruner.go b/core/state/pruner/pruner.go index 6f17e052b3..dfc6164da5 100644 --- a/core/state/pruner/pruner.go +++ b/core/state/pruner/pruner.go @@ -28,7 +28,9 @@ import ( "time" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/state/snapshot" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethdb" @@ -79,6 +81,104 @@ type Pruner struct { snaptree *snapshot.Tree } +func NewAllPruner(db ethdb.Database) (*Pruner, error) { + headBlock := rawdb.ReadHeadBlock(db) + if headBlock == nil { + return nil, errors.New("Failed to load head block") + } + return &Pruner{ + db: db, + }, nil +} + +func (p *Pruner) PruneAll(genesis *core.Genesis) error { + return pruneAll(p.db, genesis) +} + +func pruneAll(pruneDB ethdb.Database, g *core.Genesis) error { + var ( + count int + size common.StorageSize + pstart = time.Now() + logged = time.Now() + batch = pruneDB.NewBatch() + iter = pruneDB.NewIterator(nil, nil) + ) + start := time.Now() + for iter.Next() { + key := iter.Key() + if len(key) == common.HashLength { + count += 1 + size += common.StorageSize(len(key) + len(iter.Value())) + batch.Delete(key) + + var eta time.Duration // Realistically will never remain uninited + if done := binary.BigEndian.Uint64(key[:8]); done > 0 { + var ( + left = math.MaxUint64 - binary.BigEndian.Uint64(key[:8]) + speed = done/uint64(time.Since(pstart)/time.Millisecond+1) + 1 // +1s to avoid division by zero + ) + eta = time.Duration(left/speed) * time.Millisecond + } + if time.Since(logged) > 8*time.Second { + log.Info("Pruning state data", "nodes", count, "size", size, + "elapsed", common.PrettyDuration(time.Since(pstart)), "eta", common.PrettyDuration(eta)) + logged = time.Now() + } + // Recreate the iterator after every batch commit in order + // to allow the underlying compactor to delete the entries. + if batch.ValueSize() >= ethdb.IdealBatchSize { + batch.Write() + batch.Reset() + + iter.Release() + iter = pruneDB.NewIterator(nil, key) + } + } + } + if batch.ValueSize() > 0 { + batch.Write() + batch.Reset() + } + iter.Release() + log.Info("Pruned state data", "nodes", count, "size", size, "elapsed", common.PrettyDuration(time.Since(pstart))) + + // Start compactions, will remove the deleted data from the disk immediately. + // Note for small pruning, the compaction is skipped. + if count >= rangeCompactionThreshold { + cstart := time.Now() + for b := 0x00; b <= 0xf0; b += 0x10 { + var ( + start = []byte{byte(b)} + end = []byte{byte(b + 0x10)} + ) + if b == 0xf0 { + end = nil + } + log.Info("Compacting database", "range", fmt.Sprintf("%#x-%#x", start, end), "elapsed", common.PrettyDuration(time.Since(cstart))) + if err := pruneDB.Compact(start, end); err != nil { + log.Error("Database compaction failed", "error", err) + return err + } + } + log.Info("Database compaction finished", "elapsed", common.PrettyDuration(time.Since(cstart))) + } + statedb, _ := state.New(common.Hash{}, state.NewDatabase(pruneDB), nil) + for addr, account := range g.Alloc { + statedb.AddBalance(addr, account.Balance) + statedb.SetCode(addr, account.Code) + statedb.SetNonce(addr, account.Nonce) + for key, value := range account.Storage { + statedb.SetState(addr, key, value) + } + } + root := statedb.IntermediateRoot(false) + statedb.Commit(0, false) + statedb.Database().TrieDB().Commit(root, true) + log.Info("State pruning successful", "pruned", size, "elapsed", common.PrettyDuration(time.Since(start))) + return nil +} + // NewPruner creates the pruner instance. func NewPruner(db ethdb.Database, config Config) (*Pruner, error) { headBlock := rawdb.ReadHeadBlock(db) diff --git a/core/state/snapshot/journal.go b/core/state/snapshot/journal.go index 4d070208f5..696f831a36 100644 --- a/core/state/snapshot/journal.go +++ b/core/state/snapshot/journal.go @@ -120,7 +120,7 @@ func loadAndParseJournal(db ethdb.KeyValueStore, base *diskLayer) (snapshot, jou } // loadSnapshot loads a pre-existing state snapshot backed by a key-value store. -func loadSnapshot(diskdb ethdb.KeyValueStore, triedb *trie.Database, root common.Hash, cache int, recovery bool, noBuild bool) (snapshot, bool, error) { +func loadSnapshot(diskdb ethdb.KeyValueStore, triedb *trie.Database, root common.Hash, cache int, recovery bool, noBuild bool, withoutTrie bool) (snapshot, bool, error) { // If snapshotting is disabled (initial sync in progress), don't do anything, // wait for the chain to permit us to do something meaningful if rawdb.ReadSnapshotDisabled(diskdb) { @@ -152,6 +152,9 @@ func loadSnapshot(diskdb ethdb.KeyValueStore, triedb *trie.Database, root common // which is below the snapshot. In this case the snapshot can be recovered // by re-executing blocks but right now it's unavailable. if head := snapshot.Root(); head != root { + if withoutTrie { + return snapshot, false, nil + } // If it's legacy snapshot, or it's new-format snapshot but // it's not in recovery mode, returns the error here for // rebuilding the entire snapshot forcibly. diff --git a/core/state/snapshot/snapshot.go b/core/state/snapshot/snapshot.go index e30a0005cc..0328bbb18f 100644 --- a/core/state/snapshot/snapshot.go +++ b/core/state/snapshot/snapshot.go @@ -154,6 +154,7 @@ type Config struct { Recovery bool // Indicator that the snapshots is in the recovery mode NoBuild bool // Indicator that the snapshots generation is disallowed AsyncBuild bool // The snapshot generation is allowed to be constructed asynchronously + NoTries bool // Indicator that the snapshot tries are disabled } // Tree is an Ethereum state snapshot tree. It consists of one persistent base @@ -201,7 +202,7 @@ func New(config Config, diskdb ethdb.KeyValueStore, triedb *trie.Database, root layers: make(map[common.Hash]snapshot), } // Attempt to load a previously persisted snapshot and rebuild one if failed - head, disabled, err := loadSnapshot(diskdb, triedb, root, config.CacheSize, config.Recovery, config.NoBuild) + head, disabled, err := loadSnapshot(diskdb, triedb, root, config.CacheSize, config.Recovery, config.NoBuild, config.NoTries) if disabled { log.Warn("Snapshot maintenance disabled (syncing)") return snap, nil diff --git a/core/state/state_object.go b/core/state/state_object.go index 11fcb01871..98f4d90af4 100644 --- a/core/state/state_object.go +++ b/core/state/state_object.go @@ -360,6 +360,13 @@ func (s *stateObject) updateTrie() (Trie, error) { // updateRoot flushes all cached storage mutations to trie, recalculating the // new storage trie root. func (s *stateObject) updateRoot() { + // If node runs in no trie mode, set root to empty. + defer func() { + if s.db.db.NoTries() { + s.data.Root = types.EmptyRootHash + } + }() + // Flush cached storage mutations into trie, short circuit if any error // is occurred or there is not change in the trie. tr, err := s.updateTrie() diff --git a/core/state/statedb.go b/core/state/statedb.go index d28cd29b30..372098fe77 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -62,6 +62,7 @@ type StateDB struct { db Database prefetcher *triePrefetcher trie Trie + noTrie bool hasher crypto.KeccakState snaps *snapshot.Tree // Nil if snapshot is not available snap snapshot.Snapshot // Nil if snapshot is not available @@ -69,6 +70,7 @@ type StateDB struct { // originalRoot is the pre-state root, before any changes were made. // It will be updated when the Commit is called. originalRoot common.Hash + expectedRoot common.Hash // The state root in the block header // These maps hold the state changes (including the corresponding // original value) that occurred in this **block**. @@ -169,6 +171,7 @@ func New(root common.Hash, db Database, snaps *snapshot.Tree) (*StateDB, error) if sdb.snaps != nil { sdb.snap = sdb.snaps.Snapshot(root) } + _, sdb.noTrie = tr.(*trie.EmptyTrie) return sdb, nil } @@ -176,6 +179,10 @@ func New(root common.Hash, db Database, snaps *snapshot.Tree) (*StateDB, error) // state trie concurrently while the state is mutated so that when we reach the // commit phase, most of the needed data is already hot. func (s *StateDB) StartPrefetcher(namespace string) { + if s.noTrie { + return + } + if s.prefetcher != nil { s.prefetcher.close() s.prefetcher = nil @@ -188,6 +195,10 @@ func (s *StateDB) StartPrefetcher(namespace string) { // StopPrefetcher terminates a running prefetcher and reports any leftover stats // from the gathered metrics. func (s *StateDB) StopPrefetcher() { + if s.noTrie { + return + } + if s.prefetcher != nil { s.prefetcher.close() s.prefetcher = nil @@ -500,18 +511,21 @@ func (s *StateDB) GetTransientState(addr common.Address, key common.Hash) common // updateStateObject writes the given object to the trie. func (s *StateDB) updateStateObject(obj *stateObject) { - // Track the amount of time wasted on updating the account from the trie - if metrics.EnabledExpensive { - defer func(start time.Time) { s.AccountUpdates += time.Since(start) }(time.Now()) - } - // Encode the account and update the account trie - addr := obj.Address() - if err := s.trie.UpdateAccount(addr, &obj.data); err != nil { - s.setError(fmt.Errorf("updateStateObject (%x) error: %v", addr[:], err)) - } - if obj.dirtyCode { - s.trie.UpdateContractCode(obj.Address(), common.BytesToHash(obj.CodeHash()), obj.code) + if !s.noTrie { + // Track the amount of time wasted on updating the account from the trie + if metrics.EnabledExpensive { + defer func(start time.Time) { s.AccountUpdates += time.Since(start) }(time.Now()) + } + // Encode the account and update the account trie + addr := obj.Address() + if err := s.trie.UpdateAccount(addr, &obj.data); err != nil { + s.setError(fmt.Errorf("updateStateObject (%x) error: %v", addr[:], err)) + } + if obj.dirtyCode { + s.trie.UpdateContractCode(obj.Address(), common.BytesToHash(obj.CodeHash()), obj.code) + } } + // Cache the data until commit. Note, this update mechanism is not symmetric // to the deletion, because whereas it is enough to track account updates // at commit time, deletions need tracking at transaction boundary level to @@ -532,6 +546,10 @@ func (s *StateDB) updateStateObject(obj *stateObject) { // deleteStateObject removes the given object from the state trie. func (s *StateDB) deleteStateObject(obj *stateObject) { + if s.noTrie { + return + } + // Track the amount of time wasted on deleting the account from the trie if metrics.EnabledExpensive { defer func(start time.Time) { s.AccountUpdates += time.Since(start) }(time.Now()) @@ -908,6 +926,7 @@ func (s *StateDB) IntermediateRoot(deleteEmptyObjects bool) common.Hash { s.trie = trie } } + usedAddrs := make([][]byte, 0, len(s.stateObjectsPending)) for addr := range s.stateObjectsPending { if obj := s.stateObjects[addr]; obj.deleted { @@ -922,6 +941,7 @@ func (s *StateDB) IntermediateRoot(deleteEmptyObjects bool) common.Hash { if prefetcher != nil { prefetcher.used(common.Hash{}, s.originalRoot, usedAddrs) } + if len(s.stateObjectsPending) > 0 { s.stateObjectsPending = make(map[common.Address]struct{}) } @@ -929,7 +949,12 @@ func (s *StateDB) IntermediateRoot(deleteEmptyObjects bool) common.Hash { if metrics.EnabledExpensive { defer func(start time.Time) { s.AccountHashes += time.Since(start) }(time.Now()) } - return s.trie.Hash() + + if s.noTrie { + return s.expectedRoot + } else { + return s.trie.Hash() + } } // SetTxContext sets the current transaction hash and index which are @@ -1168,7 +1193,7 @@ func (s *StateDB) Commit(block uint64, deleteEmptyObjects bool) (common.Hash, er return common.Hash{}, fmt.Errorf("commit aborted due to earlier error: %v", s.dbErr) } // Finalize any pending changes and merge everything into the tries - s.IntermediateRoot(deleteEmptyObjects) + root := s.IntermediateRoot(deleteEmptyObjects) // Commit objects to the trie, measuring the elapsed time var ( @@ -1195,21 +1220,23 @@ func (s *StateDB) Commit(block uint64, deleteEmptyObjects bool) (common.Hash, er rawdb.WriteCode(codeWriter, common.BytesToHash(obj.CodeHash()), obj.code) obj.dirtyCode = false } - // Write any storage changes in the state object to its storage trie - set, err := obj.commit() - if err != nil { - return common.Hash{}, err - } - // Merge the dirty nodes of storage trie into global set. It is possible - // that the account was destructed and then resurrected in the same block. - // In this case, the node set is shared by both accounts. - if set != nil { - if err := nodes.Merge(set); err != nil { + if !s.noTrie { + // Write any storage changes in the state object to its storage trie + set, err := obj.commit() + if err != nil { return common.Hash{}, err } - updates, deleted := set.Size() - storageTrieNodesUpdated += updates - storageTrieNodesDeleted += deleted + // Merge the dirty nodes of storage trie into global set. It is possible + // that the account was destructed and then resurrected in the same block. + // In this case, the node set is shared by both accounts. + if set != nil { + if err := nodes.Merge(set); err != nil { + return common.Hash{}, err + } + updates, deleted := set.Size() + storageTrieNodesUpdated += updates + storageTrieNodesDeleted += deleted + } } } if codeWriter.ValueSize() > 0 { @@ -1222,17 +1249,21 @@ func (s *StateDB) Commit(block uint64, deleteEmptyObjects bool) (common.Hash, er if metrics.EnabledExpensive { start = time.Now() } - root, set, err := s.trie.Commit(true) - if err != nil { - return common.Hash{}, err - } - // Merge the dirty nodes of account trie into global set - if set != nil { - if err := nodes.Merge(set); err != nil { + + if !s.noTrie { + _, set, err := s.trie.Commit(true) + if err != nil { return common.Hash{}, err } - accountTrieNodesUpdated, accountTrieNodesDeleted = set.Size() + // Merge the dirty nodes of account trie into global set + if set != nil { + if err := nodes.Merge(set); err != nil { + return common.Hash{}, err + } + accountTrieNodesUpdated, accountTrieNodesDeleted = set.Size() + } } + if metrics.EnabledExpensive { s.AccountCommits += time.Since(start) @@ -1253,14 +1284,14 @@ func (s *StateDB) Commit(block uint64, deleteEmptyObjects bool) (common.Hash, er // Only update if there's a state transition (skip empty Clique blocks) if parent := s.snap.Root(); parent != root { if err := s.snaps.Update(root, parent, s.convertAccountSet(s.stateObjectsDestruct), s.accounts, s.storages); err != nil { - log.Warn("Failed to update snapshot tree", "from", parent, "to", root, "err", err) + log.Warn("Failed to update snapshot tree", "from", parent, "to", s.expectedRoot, "err", err) } // Keep 128 diff layers in the memory, persistent layer is 129th. // - head layer is paired with HEAD state // - head-1 layer is paired with HEAD-1 state // - head-127 layer(bottom-most diff layer) is paired with HEAD-127 state if err := s.snaps.Cap(root, 128); err != nil { - log.Warn("Failed to cap snapshot tree", "root", root, "layers", 128, "err", err) + log.Warn("Failed to cap snapshot tree", "root", s.expectedRoot, "layers", 128, "err", err) } } if metrics.EnabledExpensive { @@ -1268,25 +1299,29 @@ func (s *StateDB) Commit(block uint64, deleteEmptyObjects bool) (common.Hash, er } s.snap = nil } + if root == (common.Hash{}) { root = types.EmptyRootHash } - origin := s.originalRoot - if origin == (common.Hash{}) { - origin = types.EmptyRootHash - } - if root != origin { - start := time.Now() - set := triestate.New(s.accountsOrigin, s.storagesOrigin, incomplete) - if err := s.db.TrieDB().Update(root, origin, block, nodes, set); err != nil { - return common.Hash{}, err - } - s.originalRoot = root - if metrics.EnabledExpensive { - s.TrieDBCommits += time.Since(start) + + if !s.noTrie { + origin := s.originalRoot + if origin == (common.Hash{}) { + origin = types.EmptyRootHash } - if s.onCommit != nil { - s.onCommit(set) + if root != origin { + start := time.Now() + set := triestate.New(s.accountsOrigin, s.storagesOrigin, incomplete) + if err := s.db.TrieDB().Update(root, origin, block, nodes, set); err != nil { + return common.Hash{}, err + } + s.originalRoot = root + if metrics.EnabledExpensive { + s.TrieDBCommits += time.Since(start) + } + if s.onCommit != nil { + s.onCommit(set) + } } } // Clear all internal flags at the end of commit operation. @@ -1389,6 +1424,19 @@ func (s *StateDB) convertAccountSet(set map[common.Address]*types.StateAccount) return ret } +func (s *StateDB) NoTrie() bool { + return s.noTrie +} + +func (s *StateDB) GetSnap() snapshot.Snapshot { + return s.snap +} + +// Mark that the block is processed by diff layer +func (s *StateDB) SetExpectedStateRoot(root common.Hash) { + s.expectedRoot = root +} + // copySet returns a deep-copied set. func copySet[k comparable](set map[k][]byte) map[k][]byte { copied := make(map[k][]byte, len(set)) diff --git a/core/state/statedb_test.go b/core/state/statedb_test.go index ad829a0c8f..48238d29e3 100644 --- a/core/state/statedb_test.go +++ b/core/state/statedb_test.go @@ -30,6 +30,8 @@ import ( "testing" "testing/quick" + "github.com/holiman/uint256" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state/snapshot" @@ -40,7 +42,6 @@ import ( "github.com/ethereum/go-ethereum/trie/triedb/hashdb" "github.com/ethereum/go-ethereum/trie/triedb/pathdb" "github.com/ethereum/go-ethereum/trie/trienode" - "github.com/holiman/uint256" ) // Tests that updating a state trie does not leak any database writes prior to @@ -1124,7 +1125,9 @@ func TestResetObject(t *testing.T) { state.CreateAccount(addr) state.SetBalance(addr, big.NewInt(2)) state.SetState(addr, slotB, common.BytesToHash([]byte{0x2})) - root, _ := state.Commit(0, true) + root := state.IntermediateRoot(true) + state.SetExpectedStateRoot(root) + root, _ = state.Commit(0, true) // Ensure the original account is wiped properly snap := snaps.Snapshot(root) diff --git a/core/state_processor.go b/core/state_processor.go index f84007cbbe..5339f66e7a 100644 --- a/core/state_processor.go +++ b/core/state_processor.go @@ -102,6 +102,7 @@ func (p *StateProcessor) Process(block *types.Block, statedb *state.StateDB, cfg if err != nil { return nil, nil, 0, fmt.Errorf("could not apply tx %d [%v]: %w", i, tx.Hash().Hex(), err) } + receipts = append(receipts, receipt) allLogs = append(allLogs, receipt.Logs...) if metrics.EnabledExpensive { diff --git a/eth/backend.go b/eth/backend.go index 765b96caa5..abe1f5ec79 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -204,6 +204,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { TrieTimeLimit: config.TrieTimeout, SnapshotLimit: config.SnapshotCache, Preimages: config.Preimages, + NoTries: config.NoTries, StateHistory: config.StateHistory, StateScheme: scheme, TrieCommitInterval: config.TrieCommitInterval, diff --git a/eth/ethconfig/config.go b/eth/ethconfig/config.go index aa7a8bc8a2..f4d18edaa6 100644 --- a/eth/ethconfig/config.go +++ b/eth/ethconfig/config.go @@ -95,6 +95,7 @@ var OpBNBDefaults = Config{ TrieDirtyCache: 256, TrieTimeout: 60 * time.Minute, TrieCommitInterval: 3600, + NoTries: false, SnapshotCache: 102, FilterLogCacheSize: 32, Miner: miner.DefaultConfig, @@ -165,6 +166,7 @@ type Config struct { TrieCommitInterval uint64 // Define a block height interval, commit trie every TrieCommitInterval block height. SnapshotCache int Preimages bool + NoTries bool // This is the number of blocks for which logs will be cached in the filter system. FilterLogCacheSize int diff --git a/eth/ethconfig/gen_config.go b/eth/ethconfig/gen_config.go index f93e697f83..bc64a42310 100644 --- a/eth/ethconfig/gen_config.go +++ b/eth/ethconfig/gen_config.go @@ -48,6 +48,7 @@ func (c Config) MarshalTOML() (interface{}, error) { TrieCommitInterval uint64 SnapshotCache int Preimages bool + NoTries bool FilterLogCacheSize int Miner miner.Config TxPool legacypool.Config @@ -100,6 +101,7 @@ func (c Config) MarshalTOML() (interface{}, error) { enc.TrieCommitInterval = c.TrieCommitInterval enc.SnapshotCache = c.SnapshotCache enc.Preimages = c.Preimages + enc.NoTries = c.NoTries enc.FilterLogCacheSize = c.FilterLogCacheSize enc.Miner = c.Miner enc.TxPool = c.TxPool @@ -156,6 +158,7 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { TrieCommitInterval *uint64 SnapshotCache *int Preimages *bool + NoTries *bool FilterLogCacheSize *int Miner *miner.Config TxPool *legacypool.Config @@ -271,6 +274,9 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { if dec.Preimages != nil { c.Preimages = *dec.Preimages } + if dec.NoTries != nil { + c.NoTries = *dec.NoTries + } if dec.FilterLogCacheSize != nil { c.FilterLogCacheSize = *dec.FilterLogCacheSize } diff --git a/eth/handler.go b/eth/handler.go index d04f47ce92..0a168c61f6 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -173,7 +173,7 @@ func newHandler(config *handlerConfig) (*handler, error) { if fullBlock.Number.Uint64() == 0 && snapBlock.Number.Uint64() > 0 { h.snapSync.Store(true) log.Warn("Switch sync mode from full sync to snap sync", "reason", "snap sync incomplete") - } else if !h.chain.HasState(fullBlock.Root) { + } else if !h.chain.NoTries() && !h.chain.HasState(fullBlock.Root) { h.snapSync.Store(true) log.Warn("Switch sync mode from full sync to snap sync", "reason", "head state missing") } diff --git a/eth/sync.go b/eth/sync.go index c7ba7c93d6..a9732147fa 100644 --- a/eth/sync.go +++ b/eth/sync.go @@ -209,7 +209,7 @@ func (cs *chainSyncer) modeAndLocalHead() (downloader.SyncMode, *big.Int) { // We are in a full sync, but the associated head state is missing. To complete // the head state, forcefully rerun the snap sync. Note it doesn't mean the // persistent state is corrupted, just mismatch with the head block. - if !cs.handler.chain.HasState(head.Root) { + if !cs.handler.chain.NoTries() && !cs.handler.chain.HasState(head.Root) { block := cs.handler.chain.CurrentSnapBlock() td := cs.handler.chain.GetTd(block.Hash(), block.Number.Uint64()) log.Info("Reenabled snap sync as chain is stateless") diff --git a/go.mod b/go.mod index 1ce804bfa4..1003a1179e 100644 --- a/go.mod +++ b/go.mod @@ -111,7 +111,7 @@ require ( github.com/dustin/go-humanize v1.0.1 // indirect github.com/ferranbt/fastssz v0.0.0-20210905181407-59cf6761a7d5 // indirect github.com/garslo/gogen v0.0.0-20170306192744-1d203ffc1f61 // indirect - github.com/getsentry/sentry-go v0.18.0 // indirect + github.com/getsentry/sentry-go v0.27.0 // indirect github.com/go-kit/kit v0.12.0 // indirect github.com/go-kit/log v0.2.1 // indirect github.com/go-logfmt/logfmt v0.6.0 // indirect @@ -180,6 +180,9 @@ require ( rsc.io/tmplfunc v0.0.3 // indirect ) -replace github.com/cometbft/cometbft => github.com/bnb-chain/greenfield-cometbft v1.0.0 +replace ( + github.com/cometbft/cometbft => github.com/bnb-chain/greenfield-cometbft v1.0.0 + github.com/wercker/journalhook => github.com/wercker/journalhook v0.0.0-20230927020745-64542ffa4117 +) //replace github.com/ethereum-optimism/superchain-registry/superchain => ../superchain-registry/superchain diff --git a/go.sum b/go.sum index bb9fdca4ef..ed670c0401 100644 --- a/go.sum +++ b/go.sum @@ -409,8 +409,8 @@ github.com/gballet/go-verkle v0.0.0-20230607174250-df487255f46b/go.mod h1:CDncRY github.com/getkin/kin-openapi v0.53.0/go.mod h1:7Yn5whZr5kJi6t+kShccXS8ae1APpYTW6yheSwk8Yi4= github.com/getkin/kin-openapi v0.61.0/go.mod h1:7Yn5whZr5kJi6t+kShccXS8ae1APpYTW6yheSwk8Yi4= github.com/getsentry/sentry-go v0.12.0/go.mod h1:NSap0JBYWzHND8oMbyi0+XZhUalc1TBdRL1M71JZW2c= -github.com/getsentry/sentry-go v0.18.0 h1:MtBW5H9QgdcJabtZcuJG80BMOwaBpkRDZkxRkNC1sN0= -github.com/getsentry/sentry-go v0.18.0/go.mod h1:Kgon4Mby+FJ7ZWHFUAZgVaIa8sxHtnRJRLTXZr51aKQ= +github.com/getsentry/sentry-go v0.27.0 h1:Pv98CIbtB3LkMWmXi4Joa5OOcwbmnX88sF5qbK3r3Ps= +github.com/getsentry/sentry-go v0.27.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY= github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gin-contrib/sse v0.0.0-20190301062529-5545eab6dad3/go.mod h1:VJ0WA2NBN22VlZ2dKZQPAPnyWw5XTlK1KymzLKsr59s= @@ -1440,7 +1440,7 @@ github.com/wealdtech/go-eth2-types/v2 v2.5.2/go.mod h1:8lkNUbgklSQ4LZ2oMSuxSdR7W github.com/wealdtech/go-eth2-util v1.6.3/go.mod h1:0hFMj/qtio288oZFHmAbCnPQ9OB3c4WFzs5NVPKTY4k= github.com/wealdtech/go-eth2-wallet-encryptor-keystorev4 v1.1.3/go.mod h1:qiIimacW5NhVRy8o+YxWo9YrecXqDAKKbL0+sOa0SJ4= github.com/wealdtech/go-eth2-wallet-types/v2 v2.8.2/go.mod h1:k6kmiKWSWBTd4OxFifTEkPaBLhZspnO2KFD5XJY9nqg= -github.com/wercker/journalhook v0.0.0-20180428041537-5d0a5ae867b3/go.mod h1:XCsSkdKK4gwBMNrOCZWww0pX6AOt+2gYc5Z6jBRrNVg= +github.com/wercker/journalhook v0.0.0-20230927020745-64542ffa4117/go.mod h1:XCsSkdKK4gwBMNrOCZWww0pX6AOt+2gYc5Z6jBRrNVg= github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1/go.mod h1:8UvriyWtv5Q5EOgjHaSseUEdkQfvwFv1I/In/O2M9gc= github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc/go.mod h1:bopw91TMyo8J3tvftk8xmU2kPmlrt4nScJQZU2hE5EM= github.com/whyrusleeping/mdns v0.0.0-20190826153040-b9b60ed33aa9/go.mod h1:j4l84WPFclQPj320J9gp0XwNKBb3U0zt5CBqjPp22G4= diff --git a/light/trie.go b/light/trie.go index 1847f1e71b..c567ad32f9 100644 --- a/light/trie.go +++ b/light/trie.go @@ -87,6 +87,10 @@ func (db *odrDatabase) ContractCode(addr common.Address, codeHash common.Hash) ( return req.Data, err } +func (db *odrDatabase) NoTries() bool { + return false +} + func (db *odrDatabase) ContractCodeSize(addr common.Address, codeHash common.Hash) (int, error) { code, err := db.ContractCode(addr, codeHash) return len(code), err @@ -203,6 +207,10 @@ func (t *odrTrie) Prove(key []byte, proofDb ethdb.KeyValueWriter) error { return errors.New("not implemented, needs client/server interface split") } +func (t *odrTrie) NoTries() bool { + return false +} + // do tries and retries to execute a function until it returns with no error or // an error type other than MissingNodeError func (t *odrTrie) do(key []byte, fn func() error) error { diff --git a/miner/worker.go b/miner/worker.go index 8b03825cbf..6663797044 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -732,6 +732,7 @@ func (w *worker) resultLoop() { logs = append(logs, receipt.Logs...) } // Commit block and state to database. + task.state.SetExpectedStateRoot(block.Root()) _, err := w.chain.WriteBlockAndSetHead(block, receipts, logs, task.state, true) if err != nil { log.Error("Failed writing block to chain", "err", err) @@ -1129,7 +1130,7 @@ func (w *worker) fillTransactions(interrupt *atomic.Int32, env *environment) err func (w *worker) generateWork(genParams *generateParams) *newPayloadResult { // TODO delete after debug performance metrics core.DebugInnerExecutionDuration = 0 - defer func () { + defer func() { core.DebugInnerExecutionDuration = 0 }() @@ -1192,21 +1193,25 @@ func (w *worker) generateWork(genParams *generateParams) *newPayloadResult { if err != nil { return &newPayloadResult{err: err} } + if block.Root() == (common.Hash{}) { + return &newPayloadResult{err: fmt.Errorf("empty block root")} + } + assembleBlockTimer.UpdateSince(start) log.Debug("assembleBlockTimer", "duration", common.PrettyDuration(time.Since(start)), "parentHash", genParams.parentHash) - accountReadTimer.Update(work.state.AccountReads) // Account reads are complete(in commit txs) - storageReadTimer.Update(work.state.StorageReads) // Storage reads are complete(in commit txs) - snapshotAccountReadTimer.Update(work.state.SnapshotAccountReads) // Account reads are complete(in commit txs) - snapshotStorageReadTimer.Update(work.state.SnapshotStorageReads) // Storage reads are complete(in commit txs) - accountUpdateTimer.Update(work.state.AccountUpdates) // Account updates are complete(in FinalizeAndAssemble) - storageUpdateTimer.Update(work.state.StorageUpdates) // Storage updates are complete(in FinalizeAndAssemble) - accountHashTimer.Update(work.state.AccountHashes) // Account hashes are complete(in FinalizeAndAssemble) - storageHashTimer.Update(work.state.StorageHashes) // Storage hashes are complete(in FinalizeAndAssemble) + accountReadTimer.Update(work.state.AccountReads) // Account reads are complete(in commit txs) + storageReadTimer.Update(work.state.StorageReads) // Storage reads are complete(in commit txs) + snapshotAccountReadTimer.Update(work.state.SnapshotAccountReads) // Account reads are complete(in commit txs) + snapshotStorageReadTimer.Update(work.state.SnapshotStorageReads) // Storage reads are complete(in commit txs) + accountUpdateTimer.Update(work.state.AccountUpdates) // Account updates are complete(in FinalizeAndAssemble) + storageUpdateTimer.Update(work.state.StorageUpdates) // Storage updates are complete(in FinalizeAndAssemble) + accountHashTimer.Update(work.state.AccountHashes) // Account hashes are complete(in FinalizeAndAssemble) + storageHashTimer.Update(work.state.StorageHashes) // Storage hashes are complete(in FinalizeAndAssemble) innerExecutionTimer.Update(core.DebugInnerExecutionDuration) - log.Debug("build payload statedb metrics", "parentHash", genParams.parentHash, "accountReads", common.PrettyDuration(work.state.AccountReads), "storageReads", common.PrettyDuration(work.state.StorageReads), "snapshotAccountReads", common.PrettyDuration(work.state.SnapshotAccountReads), "snapshotStorageReads", common.PrettyDuration(work.state.SnapshotStorageReads), "accountUpdates", common.PrettyDuration(work.state.AccountUpdates), "storageUpdates", common.PrettyDuration(work.state.StorageUpdates), "accountHashes", common.PrettyDuration(work.state.AccountHashes), "storageHashes", common.PrettyDuration(work.state.StorageHashes)) + log.Debug("build payload statedb metrics", "parentHash", genParams.parentHash, "accountReads", common.PrettyDuration(work.state.AccountReads), "storageReads", common.PrettyDuration(work.state.StorageReads), "snapshotAccountReads", common.PrettyDuration(work.state.SnapshotAccountReads), "snapshotStorageReads", common.PrettyDuration(work.state.SnapshotStorageReads), "accountUpdates", common.PrettyDuration(work.state.AccountUpdates), "storageUpdates", common.PrettyDuration(work.state.StorageUpdates), "accountHashes", common.PrettyDuration(work.state.AccountHashes), "storageHashes", common.PrettyDuration(work.state.StorageHashes)) return &newPayloadResult{ block: block, diff --git a/tests/state_test_util.go b/tests/state_test_util.go index 7fd97b917c..20a0bfd35f 100644 --- a/tests/state_test_util.go +++ b/tests/state_test_util.go @@ -25,6 +25,8 @@ import ( "strconv" "strings" + "golang.org/x/crypto/sha3" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/common/math" @@ -41,7 +43,6 @@ import ( "github.com/ethereum/go-ethereum/trie" "github.com/ethereum/go-ethereum/trie/triedb/hashdb" "github.com/ethereum/go-ethereum/trie/triedb/pathdb" - "golang.org/x/crypto/sha3" ) // StateTest checks transaction processing without block context. diff --git a/trie/database.go b/trie/database.go index 256db6d6ec..ebbac4426c 100644 --- a/trie/database.go +++ b/trie/database.go @@ -35,6 +35,7 @@ type Config struct { Preimages bool // Flag whether the preimage of node key is recorded HashDB *hashdb.Config // Configs for hash-based scheme PathDB *pathdb.Config // Configs for experimental path-based scheme + NoTries bool } // HashDefaults represents a config for using hash-based scheme with @@ -363,3 +364,7 @@ func (db *Database) Head() common.Hash { } return pdb.Head() } + +func (db *Database) Config() *Config { + return db.config +} diff --git a/trie/dummy_trie.go b/trie/dummy_trie.go new file mode 100644 index 0000000000..cb796c6342 --- /dev/null +++ b/trie/dummy_trie.go @@ -0,0 +1,88 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package trie + +import ( + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/trie/trienode" +) + +type EmptyTrie struct{} + +// NewSecure creates a dummy trie +func NewEmptyTrie() *EmptyTrie { + return &EmptyTrie{} +} + +func (t *EmptyTrie) GetKey(shaKey []byte) []byte { + return nil +} + +func (t *EmptyTrie) GetStorage(_ common.Address, key []byte) ([]byte, error) { + return nil, nil +} + +func (t *EmptyTrie) GetAccount(address common.Address) (*types.StateAccount, error) { + return nil, nil +} + +func (t *EmptyTrie) UpdateStorage(_ common.Address, key, value []byte) error { + return nil +} + +// TryUpdateAccount abstract an account write in the trie. +func (t *EmptyTrie) UpdateAccount(address common.Address, account *types.StateAccount) error { + return nil +} + +func (t *EmptyTrie) UpdateContractCode(_ common.Address, _ common.Hash, _ []byte) error { + return nil +} + +func (t *EmptyTrie) DeleteStorage(_ common.Address, key []byte) error { + return nil +} + +func (t *EmptyTrie) DeleteAccount(address common.Address) error { + return nil +} + +func (t *EmptyTrie) Commit(collectLeaf bool) (common.Hash, *trienode.NodeSet, error) { + return common.Hash{}, nil, nil +} + +func (t *EmptyTrie) Hash() common.Hash { + return common.Hash{} +} + +// NodeIterator returns an iterator that returns nodes of the underlying trie. Iteration +// starts at the key after the given start key. +func (t *EmptyTrie) NodeIterator(startKey []byte) (NodeIterator, error) { + return nil, nil +} + +func (t *EmptyTrie) Prove(key []byte, proofDb ethdb.KeyValueWriter) error { + return nil +} + +// Copy returns a copy of SecureTrie. +func (t *EmptyTrie) Copy() *EmptyTrie { + cpy := *t + return &cpy +}