From 3053dcafb09f5b6130b731e6aee14db91c695dca Mon Sep 17 00:00:00 2001 From: Dmitry Shulyak Date: Fri, 20 Sep 2024 08:38:55 +0000 Subject: [PATCH] atx: cache poet proofs with lru (#6336) followup for https://github.com/spacemeshos/go-spacemesh/pull/6326 in current code poet proofs are fetched for every atx submitted to the node, they are relatively large (140KB) and account for sizeable chunk of all reads executed on the atx handler codepath (25%). they are also a perfect case for lru caching, they are mostly reused and replaced from epoch to epoch. basic stats in recent epochs. ``` select round_id, count(*), max(length(poet)) from poets group by round_id; 25|42|146200 26|45|145936 27|45|145738 28|45|145903 ``` --- activation/activation.go | 2 + activation/poetdb.go | 77 +++++++++++++++++++++++++++++++++------ config/mainnet.go | 1 + config/presets/testnet.go | 1 + node/node.go | 5 ++- 5 files changed, 74 insertions(+), 12 deletions(-) diff --git a/activation/activation.go b/activation/activation.go index 8c1188f70f..b77619d439 100644 --- a/activation/activation.go +++ b/activation/activation.go @@ -54,6 +54,7 @@ type PoetConfig struct { InfoCacheTTL time.Duration `mapstructure:"info-cache-ttl"` PowParamsCacheTTL time.Duration `mapstructure:"pow-params-cache-ttl"` MaxRequestRetries int `mapstructure:"retry-max"` + PoetProofsCache int `mapstructure:"poet-proofs-cache"` } func DefaultPoetConfig() PoetConfig { @@ -62,6 +63,7 @@ func DefaultPoetConfig() PoetConfig { MaxRequestRetries: 10, InfoCacheTTL: 5 * time.Minute, PowParamsCacheTTL: 5 * time.Minute, + PoetProofsCache: 200, } } diff --git a/activation/poetdb.go b/activation/poetdb.go index 3905564f89..c676e5ce38 100644 --- a/activation/poetdb.go +++ b/activation/poetdb.go @@ -5,11 +5,13 @@ import ( "encoding/hex" "fmt" + lru "github.com/hashicorp/golang-lru/v2" "github.com/spacemeshos/merkle-tree" "github.com/spacemeshos/poet/hash" "github.com/spacemeshos/poet/shared" "github.com/spacemeshos/poet/verifier" "go.uber.org/zap" + "golang.org/x/sync/singleflight" "github.com/spacemeshos/go-spacemesh/codec" "github.com/spacemeshos/go-spacemesh/common/types" @@ -19,19 +21,59 @@ import ( "github.com/spacemeshos/go-spacemesh/sql/poets" ) +// PoetDbOptions are options for PoetDb. +type PoetDbOptions struct { + cacheSize int +} + +type PoetDbOption func(*PoetDbOptions) + +// WithCacheSize sets the cache size for PoetDb. +func WithCacheSize(size int) PoetDbOption { + return func(opts *PoetDbOptions) { + opts.cacheSize = size + } +} + // PoetDb is a database for PoET proofs. type PoetDb struct { - sqlDB sql.StateDatabase - logger *zap.Logger + sqlDB sql.StateDatabase + poetProofsDbRequest singleflight.Group + poetProofsLru *lru.Cache[types.PoetProofRef, *types.PoetProofMessage] + logger *zap.Logger } // NewPoetDb returns a new PoET handler. -func NewPoetDb(db sql.StateDatabase, log *zap.Logger) *PoetDb { - return &PoetDb{sqlDB: db, logger: log} +func NewPoetDb(db sql.StateDatabase, log *zap.Logger, opts ...PoetDbOption) *PoetDb { + options := PoetDbOptions{ + // in last epochs there are 45 proofs per epoch, with each of them nearly 140KB + // 200 is set not to keep multiple epochs, but to account for unexpected growth + // select round_id, count(*), max(length(poet)) from poets group by round_id; + // 25|42|146200 + // 26|45|145936 + // 27|45|145738 + // 28|45|145903 + cacheSize: 200, + } + for _, opt := range opts { + opt(&options) + } + poetProofsLru, err := lru.New[types.PoetProofRef, *types.PoetProofMessage](options.cacheSize) + if err != nil { + log.Panic("failed to create PoET proofs LRU cache", zap.Error(err)) + } + return &PoetDb{ + sqlDB: db, + poetProofsLru: poetProofsLru, + logger: log, + } } // HasProof returns true if the database contains a proof with the given reference, or false otherwise. func (db *PoetDb) HasProof(proofRef types.PoetProofRef) bool { + if db.poetProofsLru.Contains(proofRef) { + return true + } has, err := poets.Has(db.sqlDB, proofRef) return err == nil && has } @@ -99,6 +141,7 @@ func (db *PoetDb) Validate( // StoreProof saves the poet proof in local db. func (db *PoetDb) StoreProof(ctx context.Context, ref types.PoetProofRef, proofMessage *types.PoetProofMessage) error { + db.poetProofsLru.Add(ref, proofMessage) messageBytes, err := codec.Encode(proofMessage) if err != nil { return fmt.Errorf("could not marshal proof message: %w", err) @@ -146,15 +189,27 @@ func (db *PoetDb) GetProofMessage(proofRef types.PoetProofRef) ([]byte, error) { // Proof returns full proof. func (db *PoetDb) Proof(proofRef types.PoetProofRef) (*types.PoetProof, *types.Hash32, error) { - proofMessageBytes, err := db.GetProofMessage(proofRef) + response, err, _ := db.poetProofsDbRequest.Do(string(proofRef[:]), func() (any, error) { + cachedProof, ok := db.poetProofsLru.Get(proofRef) + if ok && cachedProof != nil { + return cachedProof, nil + } + proofMessageBytes, err := db.GetProofMessage(proofRef) + if err != nil { + return nil, fmt.Errorf("could not fetch poet proof for ref %x: %w", proofRef, err) + } + var proofMessage types.PoetProofMessage + if err := codec.Decode(proofMessageBytes, &proofMessage); err != nil { + return nil, fmt.Errorf("failed to unmarshal poet proof for ref %x: %w", proofRef, err) + } + db.poetProofsLru.Add(proofRef, &proofMessage) + return &proofMessage, nil + }) if err != nil { - return nil, nil, fmt.Errorf("could not fetch poet proof for ref %x: %w", proofRef, err) - } - var proofMessage types.PoetProofMessage - if err := codec.Decode(proofMessageBytes, &proofMessage); err != nil { - return nil, nil, fmt.Errorf("failed to unmarshal poet proof for ref %x: %w", proofRef, err) + return nil, nil, err } - return &proofMessage.PoetProof, &proofMessage.Statement, nil + proof := response.(*types.PoetProofMessage) + return &proof.PoetProof, &proof.Statement, nil } func (db *PoetDb) ProofForRound(poetID []byte, roundID string) (*types.PoetProof, error) { diff --git a/config/mainnet.go b/config/mainnet.go index 3b1f8920f1..0add25c46b 100644 --- a/config/mainnet.go +++ b/config/mainnet.go @@ -173,6 +173,7 @@ func MainnetConfig() Config { RequestTimeout: 1100 * time.Second, RequestRetryDelay: 10 * time.Second, MaxRequestRetries: 10, + PoetProofsCache: 200, }, POST: activation.PostConfig{ MinNumUnits: 4, diff --git a/config/presets/testnet.go b/config/presets/testnet.go index af070e1068..f828bc12cc 100644 --- a/config/presets/testnet.go +++ b/config/presets/testnet.go @@ -126,6 +126,7 @@ func testnet() config.Config { InfoCacheTTL: 5 * time.Minute, PowParamsCacheTTL: 5 * time.Minute, + PoetProofsCache: 200, }, POST: activation.PostConfig{ MinNumUnits: 2, diff --git a/node/node.go b/node/node.go index 70ea73a48f..1ffcb89abf 100644 --- a/node/node.go +++ b/node/node.go @@ -580,7 +580,10 @@ func (app *App) initServices(ctx context.Context) error { layersPerEpoch := types.GetLayersPerEpoch() lg := app.log - poetDb := activation.NewPoetDb(app.db, app.addLogger(PoetDbLogger, lg).Zap()) + poetDb := activation.NewPoetDb( + app.db, + app.addLogger(PoetDbLogger, lg).Zap(), + activation.WithCacheSize(app.Config.POET.PoetProofsCache)) postStates := activation.NewPostStates(app.addLogger(PostLogger, lg).Zap()) opts := []activation.PostVerifierOpt{ activation.WithVerifyingOpts(app.Config.SMESHING.VerifyingOpts),