From 4c21c90e3ab53e714ca49401710d9664a65320aa Mon Sep 17 00:00:00 2001 From: will-2012 <117156346+will-2012@users.noreply.github.com> Date: Mon, 29 Jan 2024 11:24:16 +0800 Subject: [PATCH 1/2] feat: sync pebble db from bsc (#51) Co-authored-by: wayen <2229306838@qq.com> --- ethdb/pebble/pebble.go | 64 +++++++++++++++++++++++++++++++----------- go.mod | 17 +++++------ go.sum | 27 ++++++++++-------- 3 files changed, 72 insertions(+), 36 deletions(-) diff --git a/ethdb/pebble/pebble.go b/ethdb/pebble/pebble.go index 4e374c9e28..8fdd71006b 100644 --- a/ethdb/pebble/pebble.go +++ b/ethdb/pebble/pebble.go @@ -27,6 +27,7 @@ import ( "sync/atomic" "time" + "github.com/cockroachdb/errors" "github.com/cockroachdb/pebble" "github.com/cockroachdb/pebble/bloom" "github.com/ethereum/go-ethereum/common" @@ -47,6 +48,9 @@ const ( // metricsGatheringInterval specifies the interval to retrieve pebble database // compaction, io and pause stats to report to the user. metricsGatheringInterval = 3 * time.Second + + // numLevels is the level number of pebble sst files + numLevels = 7 ) // Database is a persistent key-value store based on the pebble storage engine. @@ -115,6 +119,16 @@ func (d *Database) onWriteStallEnd() { atomic.AddInt64(&d.writeDelayTime, int64(time.Since(d.writeDelayStartTime))) } +// panicLogger is just a noop logger to disable Pebble's internal logger. +type panicLogger struct{} + +func (l panicLogger) Infof(format string, args ...interface{}) { +} + +func (l panicLogger) Fatalf(format string, args ...interface{}) { + panic(errors.Errorf("fatal: "+format, args...)) +} + // New returns a wrapped pebble DB object. The namespace is the prefix that the // metrics reporting should use for surfacing internal stats. func New(file string, cache int, handles int, namespace string, readonly bool) (*Database, error) { @@ -126,7 +140,6 @@ func New(file string, cache int, handles int, namespace string, readonly bool) ( handles = minHandles } logger := log.New("database", file) - logger.Info("Allocated cache and file handles", "cache", common.StorageSize(cache*1024*1024), "handles", handles) // The max memtable size is limited by the uint32 offsets stored in // internal/arenaskl.node, DeferredBatchOp, and flushableBatchEntry. @@ -137,9 +150,20 @@ func New(file string, cache int, handles int, namespace string, readonly bool) ( // including a frozen memory table and another live one. memTableLimit := 2 memTableSize := cache * 1024 * 1024 / 2 / memTableLimit - if memTableSize > maxMemTableSize { - memTableSize = maxMemTableSize + + // The memory table size is currently capped at maxMemTableSize-1 due to a + // known bug in the pebble where maxMemTableSize is not recognized as a + // valid size. + // + // TODO use the maxMemTableSize as the maximum table size once the issue + // in pebble is fixed. + if memTableSize >= maxMemTableSize { + memTableSize = maxMemTableSize - 1 } + + logger.Info("Allocated cache and file handles", "cache", common.StorageSize(cache*1024*1024), + "handles", handles, "memory table", common.StorageSize(memTableSize)) + db := &Database{ fn: file, log: logger, @@ -154,7 +178,7 @@ func New(file string, cache int, handles int, namespace string, readonly bool) ( // The size of memory table(as well as the write buffer). // Note, there may have more than two memory tables in the system. - MemTableSize: memTableSize, + MemTableSize: uint64(memTableSize), // MemTableStopWritesThreshold places a hard limit on the size // of the existent MemTables(including the frozen one). @@ -169,15 +193,6 @@ func New(file string, cache int, handles int, namespace string, readonly bool) ( // Per-level options. Options for at least one level must be specified. The // options for the last level are used for all subsequent levels. - Levels: []pebble.LevelOptions{ - {TargetFileSize: 2 * 1024 * 1024, FilterPolicy: bloom.FilterPolicy(10)}, - {TargetFileSize: 2 * 1024 * 1024, FilterPolicy: bloom.FilterPolicy(10)}, - {TargetFileSize: 2 * 1024 * 1024, FilterPolicy: bloom.FilterPolicy(10)}, - {TargetFileSize: 2 * 1024 * 1024, FilterPolicy: bloom.FilterPolicy(10)}, - {TargetFileSize: 2 * 1024 * 1024, FilterPolicy: bloom.FilterPolicy(10)}, - {TargetFileSize: 2 * 1024 * 1024, FilterPolicy: bloom.FilterPolicy(10)}, - {TargetFileSize: 2 * 1024 * 1024, FilterPolicy: bloom.FilterPolicy(10)}, - }, ReadOnly: readonly, EventListener: &pebble.EventListener{ CompactionBegin: db.onCompactionBegin, @@ -185,7 +200,22 @@ func New(file string, cache int, handles int, namespace string, readonly bool) ( WriteStallBegin: db.onWriteStallBegin, WriteStallEnd: db.onWriteStallEnd, }, + Levels: make([]pebble.LevelOptions, numLevels), + Logger: panicLogger{}, // TODO(karalabe): Delete when this is upstreamed in Pebble } + + for i := 0; i < len(opt.Levels); i++ { + l := &opt.Levels[i] + l.BlockSize = 32 << 10 // 32 KB + l.IndexBlockSize = 256 << 10 // 256 KB + l.FilterPolicy = bloom.FilterPolicy(10) + l.FilterType = pebble.TableFilter + if i > 0 { + l.TargetFileSize = opt.Levels[i-1].TargetFileSize * 2 + } + l.EnsureDefaults() + } + // Disable seek compaction explicitly. Check https://github.com/ethereum/go-ethereum/pull/20130 // for more details. opt.Experimental.ReadSamplingMultiplier = -1 @@ -279,9 +309,10 @@ func (d *Database) NewBatch() ethdb.Batch { // It's not supported by pebble, but pebble has better memory allocation strategy // which turns out a lot faster than leveldb. It's performant enough to construct // batch object without any pre-allocated space. -func (d *Database) NewBatchWithSize(_ int) ethdb.Batch { +func (d *Database) NewBatchWithSize(size int) ethdb.Batch { return &batch{ - b: d.db.NewBatch(), + b: d.db.NewBatchWithSize(size), + db: d, } } @@ -478,6 +509,7 @@ func (d *Database) meter(refresh time.Duration) { // when Write is called. A batch cannot be used concurrently. type batch struct { b *pebble.Batch + db *Database size int } @@ -543,7 +575,7 @@ type pebbleIterator struct { // of database content with a particular key prefix, starting at a particular // initial key (or after, if it does not exist). func (d *Database) NewIterator(prefix []byte, start []byte) ethdb.Iterator { - iter := d.db.NewIter(&pebble.IterOptions{ + iter, _ := d.db.NewIter(&pebble.IterOptions{ LowerBound: append(prefix, start...), UpperBound: upperBound(prefix), }) diff --git a/go.mod b/go.mod index 261f0ff4e8..7f4eb93465 100644 --- a/go.mod +++ b/go.mod @@ -12,9 +12,10 @@ require ( github.com/btcsuite/btcd/btcec/v2 v2.3.2 github.com/cespare/cp v1.1.1 github.com/cloudflare/cloudflare-go v0.14.0 - github.com/cockroachdb/pebble v0.0.0-20230209160836-829675f94811 - github.com/cometbft/cometbft v0.37.2 - github.com/consensys/gnark-crypto v0.9.1-0.20230105202408-1a7a29904a7c + github.com/cockroachdb/errors v1.9.1 + github.com/cockroachdb/pebble v0.0.0-20230928194634-aa077af62593 + github.com/cometbft/cometbft v0.37.0 + github.com/consensys/gnark-crypto v0.10.0 github.com/davecgh/go-spew v1.1.1 github.com/deckarep/golang-set/v2 v2.1.0 github.com/docker/docker v23.0.0-rc.1+incompatible @@ -62,9 +63,9 @@ require ( github.com/tyler-smith/go-bip39 v1.1.0 github.com/urfave/cli/v2 v2.23.7 golang.org/x/crypto v0.7.0 - golang.org/x/exp v0.0.0-20230321023759-10a507213a29 + golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df golang.org/x/sync v0.1.0 - golang.org/x/sys v0.7.0 + golang.org/x/sys v0.11.0 golang.org/x/text v0.9.0 golang.org/x/time v0.0.0-20220922220347-f3bd1da661af golang.org/x/tools v0.8.0 @@ -82,12 +83,12 @@ require ( github.com/aws/aws-sdk-go-v2/service/sts v1.1.1 // indirect github.com/aws/smithy-go v1.14.2 // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1 // indirect + github.com/bits-and-blooms/bitset v1.5.0 // indirect github.com/cespare/xxhash v1.1.0 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect - github.com/cockroachdb/errors v1.9.1 // indirect github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect github.com/cockroachdb/redact v1.1.3 // indirect + github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 // indirect github.com/consensys/bavard v0.1.13 // indirect github.com/cosmos/gogoproto v1.4.1 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect @@ -149,7 +150,7 @@ require ( github.com/tklauser/numcpus v0.6.0 // indirect github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect github.com/yusufpapurcu/wmi v1.2.2 // indirect - golang.org/x/mod v0.10.0 // indirect + golang.org/x/mod v0.11.0 // indirect golang.org/x/net v0.9.0 // indirect google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect google.golang.org/grpc v1.56.1 // indirect diff --git a/go.sum b/go.sum index 1e90b92e64..34c033446f 100644 --- a/go.sum +++ b/go.sum @@ -154,6 +154,8 @@ github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+Ce github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/bits-and-blooms/bitset v1.5.0 h1:NpE8frKRLGHIcEzkR+gZhiioW1+WbYV6fKwD6ZIpQT8= +github.com/bits-and-blooms/bitset v1.5.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= github.com/bmizerany/pat v0.0.0-20170815010413-6226ea591a40/go.mod h1:8rLXio+WjiTceGBHIoTvn60HIbs7Hm7bcHjyrSqYB9c= github.com/bnb-chain/greenfield-cometbft v1.0.0 h1:0r6hOJWD/+es0gxP/exKuN/krgXAr3LCn5/XlcgDWr8= github.com/bnb-chain/greenfield-cometbft v1.0.0/go.mod h1:f35mk/r5ab6yvzlqEWZt68LfUje68sYgMpVlt2CUYMk= @@ -170,7 +172,6 @@ github.com/btcsuite/btcd/btcec/v2 v2.3.2 h1:5n0X6hX0Zk+6omWcihdYvdAlGf2DfasC0GMf github.com/btcsuite/btcd/btcec/v2 v2.3.2/go.mod h1:zYzJ8etWJQIv1Ogk7OzpWjowwOdXY1W/17j2MW85J04= github.com/btcsuite/btcd/btcutil v1.1.2 h1:XLMbX8JQEiwMcYft2EGi8zPUkoa0abKIU6/BJSRsjzQ= github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1 h1:q0rUy8C/TYNBQS1+CGKw68tLOFYSNEs0TFnxxnS9+4U= -github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1/go.mod h1:7SFka0XMvUgj3hfZtydOrQY2mwhPclbT2snogU7SQQc= github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA= github.com/btcsuite/btcutil v0.0.0-20190207003914-4c204d697803/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg= github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg= @@ -209,17 +210,19 @@ github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGX github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= -github.com/cockroachdb/datadriven v1.0.2 h1:H9MtNqVoVhvd9nCBwOyDjUEdZCREqbIdCJD93PBm/jA= github.com/cockroachdb/datadriven v1.0.2/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU= +github.com/cockroachdb/datadriven v1.0.3-0.20230413201302-be42291fc80f h1:otljaYPt5hWxV3MUfO5dFPFiOXg9CyG5/kCfayTqsJ4= github.com/cockroachdb/errors v1.9.1 h1:yFVvsI0VxmRShfawbt/laCIDy/mtTqqnvoNgiy5bEV8= github.com/cockroachdb/errors v1.9.1/go.mod h1:2sxOtL2WIc096WSZqZ5h8fa17rdDq9HZOZLBCor4mBk= github.com/cockroachdb/logtags v0.0.0-20211118104740-dabe8e521a4f/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs= github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZeQy818SGhaone5OnYfxFR/+AzdY3sf5aE= github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs= -github.com/cockroachdb/pebble v0.0.0-20230209160836-829675f94811 h1:ytcWPaNPhNoGMWEhDvS3zToKcDpRsLuRolQJBVGdozk= -github.com/cockroachdb/pebble v0.0.0-20230209160836-829675f94811/go.mod h1:Nb5lgvnQ2+oGlE/EyZy4+2/CxRh9KfvCXnag1vtpxVM= +github.com/cockroachdb/pebble v0.0.0-20230928194634-aa077af62593 h1:aPEJyR4rPBvDmeyi+l/FS/VtA00IWvjeFvjen1m1l1A= +github.com/cockroachdb/pebble v0.0.0-20230928194634-aa077af62593/go.mod h1:6hk1eMY/u5t+Cf18q5lFMUA1Rc+Sm5I6Ra1QuPyxXCo= github.com/cockroachdb/redact v1.1.3 h1:AKZds10rFSIj7qADf0g46UixK8NNLwWTNdCIGS5wfSQ= github.com/cockroachdb/redact v1.1.3/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= +github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 h1:zuQyyAKVxetITBuuhv3BI9cMrmStnpT18zmgmTxunpo= +github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06/go.mod h1:7nc4anLGjupUW/PeY5qiNYsdNXj7zopG+eqsS7To5IQ= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM= github.com/cometbft/cometbft-db v0.7.0 h1:uBjbrBx4QzU0zOEnU8KxoDl18dMNgDh+zZRUE0ucsbo= @@ -227,8 +230,8 @@ github.com/consensys/bavard v0.1.8-0.20210406032232-f3452dc9b572/go.mod h1:Bpd0/ github.com/consensys/bavard v0.1.13 h1:oLhMLOFGTLdlda/kma4VOJazblc7IM5y5QPd2A/YjhQ= github.com/consensys/bavard v0.1.13/go.mod h1:9ItSMtA/dXMAiL7BG6bqW2m3NdSEObYWoH223nGHukI= github.com/consensys/gnark-crypto v0.4.1-0.20210426202927-39ac3d4b3f1f/go.mod h1:815PAHg3wvysy0SyIqanF8gZ0Y1wjk/hrDHD/iT88+Q= -github.com/consensys/gnark-crypto v0.9.1-0.20230105202408-1a7a29904a7c h1:llSLg4o9EgH3SrXky+Q5BqEYqV76NGKo07K5Ps2pIKo= -github.com/consensys/gnark-crypto v0.9.1-0.20230105202408-1a7a29904a7c/go.mod h1:CkbdF9hbRidRJYMRzmfX8TMOr95I2pYXRHF18MzRrvA= +github.com/consensys/gnark-crypto v0.10.0 h1:zRh22SR7o4K35SoNqouS9J/TKHTyU2QWaj5ldehyXtA= +github.com/consensys/gnark-crypto v0.10.0/go.mod h1:Iq/P3HHl0ElSjsg2E1gsMwhAyxnxoKK5nVyZKd+/KhU= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= @@ -1479,8 +1482,8 @@ golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EH golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/exp v0.0.0-20200331195152-e8c3332aa8e5/go.mod h1:4M0jN8W1tt0AVLNr8HDosyJCDCDuyL9N9+3m7wDWgKw= golang.org/x/exp v0.0.0-20200513190911-00229845015e/go.mod h1:4M0jN8W1tt0AVLNr8HDosyJCDCDuyL9N9+3m7wDWgKw= -golang.org/x/exp v0.0.0-20230321023759-10a507213a29 h1:ooxPy7fPvB4kwsA2h+iBNHkAbp/4JxTSwCmvdjEYmug= -golang.org/x/exp v0.0.0-20230321023759-10a507213a29/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= +golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df h1:UA2aFVmmsIlefxMk29Dp2juaUSth8Pyn3Tq5Y5mJGME= +golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= @@ -1507,8 +1510,8 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= -golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk= -golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.11.0 h1:bUO06HqtnRcc/7l71XBe4WcqTZ+3AH1J59zWDDwLKgU= +golang.org/x/mod v0.11.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180719180050-a680a1efc54d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1710,8 +1713,8 @@ golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= -golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= From 371bc4440496bcd358f66e1e8f680f04619d500f Mon Sep 17 00:00:00 2001 From: will-2012 <117156346+will-2012@users.noreply.github.com> Date: Mon, 29 Jan 2024 11:25:10 +0800 Subject: [PATCH 2/2] feat: sync inspect mpt tool from bsc (#52) Co-authored-by: Fynn Co-authored-by: will@2012 --- cmd/geth/dbcmd.go | 89 +++++++++++++ trie/inspect_trie.go | 296 +++++++++++++++++++++++++++++++++++++++++++ trie/trie.go | 11 ++ 3 files changed, 396 insertions(+) create mode 100644 trie/inspect_trie.go diff --git a/cmd/geth/dbcmd.go b/cmd/geth/dbcmd.go index b409b19260..e182b7ed08 100644 --- a/cmd/geth/dbcmd.go +++ b/cmd/geth/dbcmd.go @@ -30,6 +30,7 @@ import ( "github.com/ethereum/go-ethereum/cmd/utils" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/common/math" "github.com/ethereum/go-ethereum/console/prompt" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state/snapshot" @@ -62,6 +63,7 @@ Remove blockchain and state databases`, dbCompactCmd, dbGetCmd, dbDeleteCmd, + dbInspectTrieCmd, dbPutCmd, dbGetSlotsCmd, dbDumpFreezerIndex, @@ -81,6 +83,17 @@ Remove blockchain and state databases`, Usage: "Inspect the storage size for each type of data in the database", Description: `This commands iterates the entire database. If the optional 'prefix' and 'start' arguments are provided, then the iteration is limited to the given subset of data.`, } + dbInspectTrieCmd = &cli.Command{ + Action: inspectTrie, + Name: "inspect-trie", + ArgsUsage: " ", + Flags: []cli.Flag{ + utils.DataDirFlag, + utils.SyncModeFlag, + }, + Usage: "Inspect the MPT tree of the account and contract.", + Description: `This commands iterates the entrie WorldState.`, + } dbCheckStateContentCmd = &cli.Command{ Action: checkStateContent, Name: "check-state-content", @@ -255,6 +268,82 @@ func confirmAndRemoveDB(database string, kind string) { } } +func inspectTrie(ctx *cli.Context) error { + if ctx.NArg() < 1 { + return fmt.Errorf("required arguments: %v", ctx.Command.ArgsUsage) + } + + if ctx.NArg() > 3 { + return fmt.Errorf("Max 3 arguments: %v", ctx.Command.ArgsUsage) + } + + var ( + blockNumber uint64 + trieRootHash common.Hash + jobnum uint64 + ) + + stack, _ := makeConfigNode(ctx) + defer stack.Close() + + db := utils.MakeChainDatabase(ctx, stack, true) + defer db.Close() + + var headerBlockHash common.Hash + if ctx.NArg() >= 1 { + if ctx.Args().Get(0) == "latest" { + headerHash := rawdb.ReadHeadHeaderHash(db) + blockNumber = *(rawdb.ReadHeaderNumber(db, headerHash)) + } else if ctx.Args().Get(0) == "snapshot" { + trieRootHash = rawdb.ReadSnapshotRoot(db) + blockNumber = math.MaxUint64 + } else { + var err error + blockNumber, err = strconv.ParseUint(ctx.Args().Get(0), 10, 64) + if err != nil { + return fmt.Errorf("failed to Parse blocknum, Args[0]: %v, err: %v", ctx.Args().Get(0), err) + } + } + + if ctx.NArg() == 1 { + jobnum = 1000 + } else { + var err error + jobnum, err = strconv.ParseUint(ctx.Args().Get(1), 10, 64) + if err != nil { + return fmt.Errorf("failed to Parse jobnum, Args[1]: %v, err: %v", ctx.Args().Get(1), err) + } + } + + if blockNumber != math.MaxUint64 { + headerBlockHash = rawdb.ReadCanonicalHash(db, blockNumber) + if headerBlockHash == (common.Hash{}) { + return fmt.Errorf("ReadHeadBlockHash empty hash") + } + blockHeader := rawdb.ReadHeader(db, headerBlockHash, blockNumber) + trieRootHash = blockHeader.Root + } + if (trieRootHash == common.Hash{}) { + log.Error("Empty root hash") + } + fmt.Printf("ReadBlockHeader, root: %v, blocknum: %v\n", trieRootHash, blockNumber) + + triedb := trie.NewDatabase(db) + theTrie, err := trie.New(trie.TrieID(trieRootHash), triedb) + if err != nil { + fmt.Printf("fail to new trie tree, err: %v, rootHash: %v\n", err, trieRootHash.String()) + return err + } + theInspect, err := trie.NewInspector(theTrie, triedb, trieRootHash, blockNumber, jobnum) + if err != nil { + return err + } + theInspect.Run() + theInspect.DisplayResult() + } + return nil +} + func inspect(ctx *cli.Context) error { var ( prefix []byte diff --git a/trie/inspect_trie.go b/trie/inspect_trie.go new file mode 100644 index 0000000000..f0f706f049 --- /dev/null +++ b/trie/inspect_trie.go @@ -0,0 +1,296 @@ +package trie + +import ( + "bytes" + "errors" + "fmt" + "math/big" + "os" + "runtime" + "sort" + "strconv" + "sync" + "sync/atomic" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/rlp" + "github.com/olekukonko/tablewriter" + + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/types" + "golang.org/x/sync/semaphore" +) + +const ( + DEFAULT_TRIEDBCACHE_SIZE = 1024 * 1024 * 1024 +) + +type Account struct { + Nonce uint64 + Balance *big.Int + Root common.Hash // merkle root of the storage trie + CodeHash []byte +} + +type Inspector struct { + trie *Trie // traverse trie + db *Database + stateRootHash common.Hash + blocknum uint64 + root node // root of triedb + totalNum uint64 + wg sync.WaitGroup + statLock sync.RWMutex + result map[string]*TrieTreeStat + sem *semaphore.Weighted + eoaAccountNums uint64 +} + +type TrieTreeStat struct { + isAccountTrie bool + theNodeStatByLevel [15]NodeStat + totalNodeStat NodeStat +} + +type NodeStat struct { + ShortNodeCnt uint64 + FullNodeCnt uint64 + ValueNodeCnt uint64 +} + +func (trieStat *TrieTreeStat) AtomicAdd(theNode node, height uint32) { + switch (theNode).(type) { + case *shortNode: + atomic.AddUint64(&trieStat.totalNodeStat.ShortNodeCnt, 1) + atomic.AddUint64(&(trieStat.theNodeStatByLevel[height].ShortNodeCnt), 1) + case *fullNode: + atomic.AddUint64(&trieStat.totalNodeStat.FullNodeCnt, 1) + atomic.AddUint64(&trieStat.theNodeStatByLevel[height].FullNodeCnt, 1) + case valueNode: + atomic.AddUint64(&trieStat.totalNodeStat.ValueNodeCnt, 1) + atomic.AddUint64(&((trieStat.theNodeStatByLevel[height]).ValueNodeCnt), 1) + default: + panic(errors.New("Invalid node type to statistics")) + } +} + +func (trieStat *TrieTreeStat) Display(ownerAddress string, treeType string) { + table := tablewriter.NewWriter(os.Stdout) + table.SetHeader([]string{"-", "Level", "ShortNodeCnt", "FullNodeCnt", "ValueNodeCnt"}) + if ownerAddress == "" { + table.SetCaption(true, fmt.Sprintf("%v", treeType)) + } else { + table.SetCaption(true, fmt.Sprintf("%v-%v", treeType, ownerAddress)) + } + table.SetAlignment(1) + for i := 0; i < len(trieStat.theNodeStatByLevel); i++ { + nodeStat := trieStat.theNodeStatByLevel[i] + if nodeStat.FullNodeCnt == 0 && nodeStat.ShortNodeCnt == 0 && nodeStat.ValueNodeCnt == 0 { + break + } + table.AppendBulk([][]string{ + {"-", strconv.Itoa(i), nodeStat.ShortNodeCount(), nodeStat.FullNodeCount(), nodeStat.ValueNodeCount()}, + }) + } + table.AppendBulk([][]string{ + {"Total", "-", trieStat.totalNodeStat.ShortNodeCount(), trieStat.totalNodeStat.FullNodeCount(), trieStat.totalNodeStat.ValueNodeCount()}, + }) + table.Render() +} + +func Uint64ToString(cnt uint64) string { + return fmt.Sprintf("%v", cnt) +} + +func (nodeStat *NodeStat) ShortNodeCount() string { + return Uint64ToString(nodeStat.ShortNodeCnt) +} + +func (nodeStat *NodeStat) FullNodeCount() string { + return Uint64ToString(nodeStat.FullNodeCnt) +} +func (nodeStat *NodeStat) ValueNodeCount() string { + return Uint64ToString(nodeStat.ValueNodeCnt) +} + +// NewInspector return a inspector obj +func NewInspector(tr *Trie, db *Database, stateRootHash common.Hash, blocknum uint64, jobnum uint64) (*Inspector, error) { + if tr == nil { + return nil, errors.New("trie is nil") + } + + if tr.root == nil { + return nil, errors.New("trie root is nil") + } + + ins := &Inspector{ + trie: tr, + db: db, + stateRootHash: stateRootHash, + blocknum: blocknum, + root: tr.root, + result: make(map[string]*TrieTreeStat), + totalNum: (uint64)(0), + wg: sync.WaitGroup{}, + sem: semaphore.NewWeighted(int64(jobnum)), + eoaAccountNums: 0, + } + + return ins, nil +} + +// Run statistics, external call +func (inspect *Inspector) Run() { + accountTrieStat := &TrieTreeStat{ + isAccountTrie: true, + } + if inspect.db.Scheme() == rawdb.HashScheme { + ticker := time.NewTicker(30 * time.Second) + go func() { + defer ticker.Stop() + for range ticker.C { + inspect.db.Cap(DEFAULT_TRIEDBCACHE_SIZE) + } + }() + } + + if _, ok := inspect.result[""]; !ok { + inspect.result[""] = accountTrieStat + } + log.Info("Find Account Trie Tree", "rootHash: ", inspect.trie.Hash().String(), "BlockNum: ", inspect.blocknum) + + inspect.ConcurrentTraversal(inspect.trie, accountTrieStat, inspect.root, 0, []byte{}) + inspect.wg.Wait() +} + +func (inspect *Inspector) SubConcurrentTraversal(theTrie *Trie, theTrieTreeStat *TrieTreeStat, theNode node, height uint32, path []byte) { + inspect.ConcurrentTraversal(theTrie, theTrieTreeStat, theNode, height, path) + inspect.wg.Done() +} + +func (inspect *Inspector) ConcurrentTraversal(theTrie *Trie, theTrieTreeStat *TrieTreeStat, theNode node, height uint32, path []byte) { + // print process progress + total_num := atomic.AddUint64(&inspect.totalNum, 1) + if total_num%100000 == 0 { + fmt.Printf("Complete progress: %v, go routines Num: %v\n", total_num, runtime.NumGoroutine()) + } + + // nil node + if theNode == nil { + return + } + + switch current := (theNode).(type) { + case *shortNode: + inspect.ConcurrentTraversal(theTrie, theTrieTreeStat, current.Val, height, append(path, current.Key...)) + case *fullNode: + for idx, child := range current.Children { + if child == nil { + continue + } + childPath := append(path, byte(idx)) + if inspect.sem.TryAcquire(1) { + inspect.wg.Add(1) + dst := make([]byte, len(childPath)) + copy(dst, childPath) + go inspect.SubConcurrentTraversal(theTrie, theTrieTreeStat, child, height+1, dst) + } else { + inspect.ConcurrentTraversal(theTrie, theTrieTreeStat, child, height+1, childPath) + } + } + case hashNode: + n, err := theTrie.resloveWithoutTrack(current, path) + if err != nil { + fmt.Printf("Resolve HashNode error: %v, TrieRoot: %v, Height: %v, Path: %v\n", err, theTrie.Hash().String(), height+1, path) + return + } + inspect.ConcurrentTraversal(theTrie, theTrieTreeStat, n, height, path) + return + case valueNode: + if !hasTerm(path) { + break + } + var account Account + if err := rlp.Decode(bytes.NewReader(current), &account); err != nil { + break + } + if common.BytesToHash(account.CodeHash) == types.EmptyCodeHash { + inspect.eoaAccountNums++ + } + if account.Root == (common.Hash{}) || account.Root == types.EmptyRootHash { + break + } + ownerAddress := common.BytesToHash(hexToCompact(path)) + contractTrie, err := New(StorageTrieID(inspect.stateRootHash, ownerAddress, account.Root), inspect.db) + if err != nil { + fmt.Printf("New contract trie node: %v, error: %v, Height: %v, Path: %v\n", theNode, err, height, path) + break + } + contractTrie.tracer.reset() + trieStat := &TrieTreeStat{ + isAccountTrie: false, + } + + inspect.statLock.Lock() + if _, ok := inspect.result[ownerAddress.String()]; !ok { + inspect.result[ownerAddress.String()] = trieStat + } + inspect.statLock.Unlock() + + // log.Info("Find Contract Trie Tree, rootHash: ", contractTrie.Hash().String(), "") + inspect.wg.Add(1) + go inspect.SubConcurrentTraversal(contractTrie, trieStat, contractTrie.root, 0, []byte{}) + default: + panic(errors.New("Invalid node type to traverse.")) + } + theTrieTreeStat.AtomicAdd(theNode, height) +} + +func (inspect *Inspector) DisplayResult() { + // display root hash + if _, ok := inspect.result[""]; !ok { + log.Info("Display result error", "missing account trie") + return + } + inspect.result[""].Display("", "AccountTrie") + + type SortedTrie struct { + totalNum uint64 + ownerAddress string + } + // display contract trie + var sortedTriesByNums []SortedTrie + var totalContactsNodeStat NodeStat + var contractTrieCnt uint64 = 0 + + for ownerAddress, stat := range inspect.result { + if ownerAddress == "" { + continue + } + contractTrieCnt++ + totalContactsNodeStat.ShortNodeCnt += stat.totalNodeStat.ShortNodeCnt + totalContactsNodeStat.FullNodeCnt += stat.totalNodeStat.FullNodeCnt + totalContactsNodeStat.ValueNodeCnt += stat.totalNodeStat.ValueNodeCnt + totalNodeCnt := stat.totalNodeStat.ShortNodeCnt + stat.totalNodeStat.ValueNodeCnt + stat.totalNodeStat.FullNodeCnt + sortedTriesByNums = append(sortedTriesByNums, SortedTrie{totalNum: totalNodeCnt, ownerAddress: ownerAddress}) + } + sort.Slice(sortedTriesByNums, func(i, j int) bool { + return sortedTriesByNums[i].totalNum > sortedTriesByNums[j].totalNum + }) + fmt.Println("EOA accounts num: ", inspect.eoaAccountNums) + // only display top 5 + for i, t := range sortedTriesByNums { + if i > 5 { + break + } + if stat, ok := inspect.result[t.ownerAddress]; !ok { + log.Error("Storage trie stat not found", "ownerAddress", t.ownerAddress) + } else { + stat.Display(t.ownerAddress, "ContractTrie") + } + } + fmt.Printf("Contract Trie, total trie num: %v, ShortNodeCnt: %v, FullNodeCnt: %v, ValueNodeCnt: %v\n", + contractTrieCnt, totalContactsNodeStat.ShortNodeCnt, totalContactsNodeStat.FullNodeCnt, totalContactsNodeStat.ValueNodeCnt) +} diff --git a/trie/trie.go b/trie/trie.go index d98fe992d5..3a83c37aa7 100644 --- a/trie/trie.go +++ b/trie/trie.go @@ -544,6 +544,17 @@ func (t *Trie) resolveAndTrack(n hashNode, prefix []byte) (node, error) { return mustDecodeNode(n, blob), nil } +func (t *Trie) resloveWithoutTrack(n node, prefix []byte) (node, error) { + if n, ok := n.(hashNode); ok { + blob, err := t.reader.nodeBlob(prefix, common.BytesToHash(n)) + if err != nil { + return nil, err + } + return mustDecodeNode(n, blob), nil + } + return n, nil +} + // Hash returns the root hash of the trie. It does not write to the // database and can be used even if the trie doesn't have one. func (t *Trie) Hash() common.Hash {