From d8d699a2b076590e73288ceed7f1bb6daf084d33 Mon Sep 17 00:00:00 2001 From: "Masih H. Derkani" Date: Wed, 18 Sep 2024 17:39:49 +0200 Subject: [PATCH] feat(f3): checkpoint tipsets that are finalized by F3 (#12460) * Checkpoint tipsets that are finalized by F3 Once a decision is received from F3, checkpoint it in chain-store. As part of checkpointing, refactor the F3 tipset wrapper to reduce type casting. Note that go-f3 module now uses Go 1.22, which requires Lotus go module to be updated accordingly. Part of https://github.com/filecoin-project/go-f3/issues/603 * Make checkpointing chainstore optional via build constraints --- CHANGELOG.md | 1 + build/buildconstants/params_2k.go | 4 + build/buildconstants/params_butterfly.go | 4 + build/buildconstants/params_calibnet.go | 4 + build/buildconstants/params_interop.go | 4 + build/buildconstants/params_mainnet.go | 4 + build/buildconstants/params_testground.go | 4 + build/openrpc/full.json | 16 +-- chain/lf3/ec.go | 151 +++++++++++++--------- chain/lf3/f3.go | 23 ++-- chain/types/tipset_key.go | 3 +- go.mod | 3 +- go.sum | 6 +- node/builder_chain.go | 2 + 14 files changed, 142 insertions(+), 87 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 049f3f1e492..d3448e8f2bb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ ## New features * Add `EthSendRawTransactionUntrusted` RPC method to be used for the gateway when accepting `EthSendRawTransaction` and `eth_sendRawTransaction`. Applies a tighter limit on the number of messages in the queue from a single sender and applies additional restrictions on nonce increments. ([filecoin-project/lotus#12431](https://github.com/filecoin-project/lotus/pull/12431)) +* [Checkpoint TipSets finalized by F3](https://github.com/filecoin-project/lotus/pull/12460): Once a decision is made by F3, the TipSet is check-pointed in `ChainStore`. As part of this change, any missing TipSets are asynchronously synced as required by the `ChainStore` checkpointing mechanism. ## Improvements diff --git a/build/buildconstants/params_2k.go b/build/buildconstants/params_2k.go index ce800e80fb8..2455a49bc2a 100644 --- a/build/buildconstants/params_2k.go +++ b/build/buildconstants/params_2k.go @@ -200,3 +200,7 @@ var F3Enabled = true const ManifestServerID = "12D3KooWHcNBkqXEBrsjoveQvj6zDF3vK5S9tAfqyYaQF1LGSJwG" var F3BootstrapEpoch abi.ChainEpoch = 1000 + +// F3Consensus set whether F3 should checkpoint tipsets finalized by F3. This +// flag has no effect if F3 is not enabled. +const F3Consensus = true diff --git a/build/buildconstants/params_butterfly.go b/build/buildconstants/params_butterfly.go index 2f9bae83a69..2ec6363d5f3 100644 --- a/build/buildconstants/params_butterfly.go +++ b/build/buildconstants/params_butterfly.go @@ -101,3 +101,7 @@ var WhitelistedBlock = cid.Undef const F3Enabled = true const ManifestServerID = "12D3KooWJr9jy4ngtJNR7JC1xgLFra3DjEtyxskRYWvBK9TC3Yn6" const F3BootstrapEpoch abi.ChainEpoch = 1000 + +// F3Consensus set whether F3 should checkpoint tipsets finalized by F3. This +// flag has no effect if F3 is not enabled. +const F3Consensus = true diff --git a/build/buildconstants/params_calibnet.go b/build/buildconstants/params_calibnet.go index 905b0171d65..8e6c2d61f6c 100644 --- a/build/buildconstants/params_calibnet.go +++ b/build/buildconstants/params_calibnet.go @@ -148,3 +148,7 @@ var WhitelistedBlock = cid.Undef const F3Enabled = true const ManifestServerID = "12D3KooWS9vD9uwm8u2uPyJV32QBAhKAmPYwmziAgr3Xzk2FU1Mr" const F3BootstrapEpoch abi.ChainEpoch = UpgradeWaffleHeight + 100 + +// F3Consensus set whether F3 should checkpoint tipsets finalized by F3. This +// flag has no effect if F3 is not enabled. +const F3Consensus = true diff --git a/build/buildconstants/params_interop.go b/build/buildconstants/params_interop.go index 79eec060031..cd4552b345b 100644 --- a/build/buildconstants/params_interop.go +++ b/build/buildconstants/params_interop.go @@ -139,3 +139,7 @@ var WhitelistedBlock = cid.Undef const F3Enabled = true const ManifestServerID = "12D3KooWQJ2rdVnG4okDUB6yHQhAjNutGNemcM7XzqC9Eo4z9Jce" const F3BootstrapEpoch abi.ChainEpoch = 1000 + +// F3Consensus set whether F3 should checkpoint tipsets finalized by F3. This +// flag has no effect if F3 is not enabled. +const F3Consensus = true diff --git a/build/buildconstants/params_mainnet.go b/build/buildconstants/params_mainnet.go index 9533e1357a9..e0fa3721ab8 100644 --- a/build/buildconstants/params_mainnet.go +++ b/build/buildconstants/params_mainnet.go @@ -173,3 +173,7 @@ var WhitelistedBlock = cid.MustParse("bafy2bzaceapyg2uyzk7vueh3xccxkuwbz3nxewjyg const F3Enabled = true const ManifestServerID = "12D3KooWENMwUF9YxvQxar7uBWJtZkA6amvK4xWmKXfSiHUo2Qq7" const F3BootstrapEpoch abi.ChainEpoch = -1 + +// F3Consensus set whether F3 should checkpoint tipsets finalized by F3. This +// flag has no effect if F3 is not enabled. +const F3Consensus = false diff --git a/build/buildconstants/params_testground.go b/build/buildconstants/params_testground.go index 7ab1f171bda..84c863f1101 100644 --- a/build/buildconstants/params_testground.go +++ b/build/buildconstants/params_testground.go @@ -126,6 +126,10 @@ var ( F3Enabled = false ManifestServerID = "" F3BootstrapEpoch abi.ChainEpoch = -1 + + // F3Consensus set whether F3 should checkpoint tipsets finalized by F3. This + // flag has no effect if F3 is not enabled. + F3Consensus = true ) func init() { diff --git a/build/openrpc/full.json b/build/openrpc/full.json index 4bae60f82a4..7094ed0f1af 100644 --- a/build/openrpc/full.json +++ b/build/openrpc/full.json @@ -6485,9 +6485,7 @@ "type": "string" }, "PowerTable": { - "media": { - "binaryEncoding": "base64" - }, + "title": "Content Identifier", "type": "string" } }, @@ -6548,9 +6546,7 @@ "type": "array" }, "PowerTable": { - "media": { - "binaryEncoding": "base64" - }, + "title": "Content Identifier", "type": "string" } }, @@ -6822,9 +6818,7 @@ "type": "string" }, "PowerTable": { - "media": { - "binaryEncoding": "base64" - }, + "title": "Content Identifier", "type": "string" } }, @@ -6885,9 +6879,7 @@ "type": "array" }, "PowerTable": { - "media": { - "binaryEncoding": "base64" - }, + "title": "Content Identifier", "type": "string" } }, diff --git a/chain/lf3/ec.go b/chain/lf3/ec.go index 69095bab8d3..ae9b5728b1e 100644 --- a/chain/lf3/ec.go +++ b/chain/lf3/ec.go @@ -12,6 +12,7 @@ import ( "github.com/filecoin-project/go-f3/gpbft" "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/lotus/chain" "github.com/filecoin-project/lotus/chain/actors/builtin/miner" "github.com/filecoin-project/lotus/chain/actors/builtin/power" "github.com/filecoin-project/lotus/chain/stmgr" @@ -20,52 +21,57 @@ import ( "github.com/filecoin-project/lotus/chain/vm" ) +var ( + _ ec.Backend = (*ecWrapper)(nil) + _ ec.TipSet = (*f3TipSet)(nil) +) + type ecWrapper struct { ChainStore *store.ChainStore + Syncer *chain.Syncer StateManager *stmgr.StateManager -} - -var _ ec.TipSet = (*f3TipSet)(nil) - -type f3TipSet types.TipSet -func (ts *f3TipSet) cast() *types.TipSet { - return (*types.TipSet)(ts) + // Checkpoint sets whether to checkpoint tipsets finalized by F3 in ChainStore. + Checkpoint bool } -func (ts *f3TipSet) String() string { - return ts.cast().String() +type f3TipSet struct { + *types.TipSet } -func (ts *f3TipSet) Key() gpbft.TipSetKey { - return ts.cast().Key().Bytes() +func (ts *f3TipSet) String() string { return ts.TipSet.String() } +func (ts *f3TipSet) Key() gpbft.TipSetKey { return ts.TipSet.Key().Bytes() } +func (ts *f3TipSet) Epoch() int64 { return int64(ts.TipSet.Height()) } + +func (ts *f3TipSet) FirstBlockHeader() *types.BlockHeader { + if ts.TipSet == nil || len(ts.TipSet.Blocks()) == 0 { + return nil + } + return ts.TipSet.Blocks()[0] } func (ts *f3TipSet) Beacon() []byte { - entries := ts.cast().Blocks()[0].BeaconEntries - if len(entries) == 0 { - // This should never happen in practice, but set beacon to a non-nil - // 32byte slice to force the message builder to generate a - // ticket. Otherwise, messages that require ticket, i.e. CONVERGE will fail - // validation due to the absence of ticket. This is a convoluted way of doing it. + switch header := ts.FirstBlockHeader(); { + case header == nil, len(header.BeaconEntries) == 0: + // This should never happen in practice, but set beacon to a non-nil 32byte slice + // to force the message builder to generate a ticket. Otherwise, messages that + // require ticket, i.e. CONVERGE will fail validation due to the absence of + // ticket. This is a convoluted way of doing it. + + // TODO: investigate if this is still necessary, or how message builder can be + // adapted to behave correctly regardless of beacon value, e.g. fail fast + // instead of building CONVERGE with empty beacon. return make([]byte, 32) + default: + return header.BeaconEntries[len(header.BeaconEntries)-1].Data } - return entries[len(entries)-1].Data -} - -func (ts *f3TipSet) Epoch() int64 { - return int64(ts.cast().Height()) } func (ts *f3TipSet) Timestamp() time.Time { - return time.Unix(int64(ts.cast().Blocks()[0].Timestamp), 0) -} - -func wrapTS(ts *types.TipSet) ec.TipSet { - if ts == nil { - return nil + if header := ts.FirstBlockHeader(); header != nil { + return time.Unix(int64(header.Timestamp), 0) } - return (*f3TipSet)(ts) + return time.Time{} } // GetTipsetByEpoch should return a tipset before the one requested if the requested @@ -75,57 +81,42 @@ func (ec *ecWrapper) GetTipsetByEpoch(ctx context.Context, epoch int64) (ec.TipS if err != nil { return nil, xerrors.Errorf("getting tipset by height: %w", err) } - return wrapTS(ts), nil + return &f3TipSet{TipSet: ts}, nil } func (ec *ecWrapper) GetTipset(ctx context.Context, tsk gpbft.TipSetKey) (ec.TipSet, error) { - tskLotus, err := types.TipSetKeyFromBytes(tsk) - if err != nil { - return nil, xerrors.Errorf("decoding tsk: %w", err) - } - - ts, err := ec.ChainStore.GetTipSetFromKey(ctx, tskLotus) + ts, err := ec.getTipSetFromF3TSK(ctx, tsk) if err != nil { return nil, xerrors.Errorf("getting tipset by key: %w", err) } - return wrapTS(ts), nil + return &f3TipSet{TipSet: ts}, nil } -func (ec *ecWrapper) GetHead(_ context.Context) (ec.TipSet, error) { - return wrapTS(ec.ChainStore.GetHeaviestTipSet()), nil +func (ec *ecWrapper) GetHead(context.Context) (ec.TipSet, error) { + head := ec.ChainStore.GetHeaviestTipSet() + if head == nil { + return nil, xerrors.New("no heaviest tipset") + } + return &f3TipSet{TipSet: head}, nil } func (ec *ecWrapper) GetParent(ctx context.Context, tsF3 ec.TipSet) (ec.TipSet, error) { - var ts *types.TipSet - if tsW, ok := tsF3.(*f3TipSet); ok { - ts = tsW.cast() - } else { - // There are only two implementations of ec.TipSet: f3TipSet, and one in fake EC - // backend. - // - // TODO: Revisit the type check here and remove as needed once testing - // is over and fake EC backend goes away. - tskLotus, err := types.TipSetKeyFromBytes(tsF3.Key()) - if err != nil { - return nil, xerrors.Errorf("decoding tsk: %w", err) - } - ts, err = ec.ChainStore.GetTipSetFromKey(ctx, tskLotus) - if err != nil { - return nil, xerrors.Errorf("getting tipset by key for get parent: %w", err) - } + ts, err := ec.toLotusTipSet(ctx, tsF3) + if err != nil { + return nil, err } parentTs, err := ec.ChainStore.GetTipSetFromKey(ctx, ts.Parents()) if err != nil { return nil, xerrors.Errorf("getting parent tipset: %w", err) } - return wrapTS(parentTs), nil + return &f3TipSet{TipSet: parentTs}, nil } func (ec *ecWrapper) GetPowerTable(ctx context.Context, tskF3 gpbft.TipSetKey) (gpbft.PowerEntries, error) { - tsk, err := types.TipSetKeyFromBytes(tskF3) + tsk, err := toLotusTipSetKey(tskF3) if err != nil { - return nil, xerrors.Errorf("decoding tsk: %w", err) + return nil, err } return ec.getPowerTableLotusTSK(ctx, tsk) } @@ -208,7 +199,7 @@ func (ec *ecWrapper) getPowerTableLotusTSK(ctx context.Context, tsk types.TipSet if waddr.Protocol() != address.BLS { return xerrors.Errorf("wrong type of worker address") } - pe.PubKey = gpbft.PubKey(waddr.Payload()) + pe.PubKey = waddr.Payload() powerEntries = append(powerEntries, pe) return nil }) @@ -219,3 +210,43 @@ func (ec *ecWrapper) getPowerTableLotusTSK(ctx context.Context, tsk types.TipSet sort.Sort(powerEntries) return powerEntries, nil } + +func (ec *ecWrapper) Finalize(ctx context.Context, key gpbft.TipSetKey) error { + if !ec.Checkpoint { + return nil // Nothing to do; checkpointing is not enabled. + } + tsk, err := toLotusTipSetKey(key) + if err != nil { + return err + } + if err = ec.Syncer.SyncCheckpoint(ctx, tsk); err != nil { + return xerrors.Errorf("checkpointing finalized tipset: %w", err) + } + return nil +} + +func (ec *ecWrapper) toLotusTipSet(ctx context.Context, ts ec.TipSet) (*types.TipSet, error) { + switch tst := ts.(type) { + case *f3TipSet: + return tst.TipSet, nil + default: + // Fall back on getting the tipset by key. This path is executed only in testing. + return ec.getTipSetFromF3TSK(ctx, ts.Key()) + } +} + +func (ec *ecWrapper) getTipSetFromF3TSK(ctx context.Context, key gpbft.TipSetKey) (*types.TipSet, error) { + tsk, err := toLotusTipSetKey(key) + if err != nil { + return nil, err + } + ts, err := ec.ChainStore.GetTipSetFromKey(ctx, tsk) + if err != nil { + return nil, xerrors.Errorf("getting tipset from key: %w", err) + } + return ts, nil +} + +func toLotusTipSetKey(key gpbft.TipSetKey) (types.TipSetKey, error) { + return types.TipSetKeyFromBytes(key) +} diff --git a/chain/lf3/f3.go b/chain/lf3/f3.go index eba612e77b1..25d0ab9f5e9 100644 --- a/chain/lf3/f3.go +++ b/chain/lf3/f3.go @@ -20,6 +20,7 @@ import ( "github.com/filecoin-project/go-f3/manifest" "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/chain" "github.com/filecoin-project/lotus/chain/stmgr" "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" @@ -35,17 +36,21 @@ type F3 struct { newLeases chan leaseRequest } +type F3ConsensusEnabled bool + type F3Params struct { fx.In - NetworkName dtypes.NetworkName - ManifestProvider manifest.ManifestProvider - PubSub *pubsub.PubSub - Host host.Host - ChainStore *store.ChainStore - StateManager *stmgr.StateManager - Datastore dtypes.MetadataDS - Wallet api.Wallet + NetworkName dtypes.NetworkName + ManifestProvider manifest.ManifestProvider + PubSub *pubsub.PubSub + Host host.Host + ChainStore *store.ChainStore + Syncer *chain.Syncer + StateManager *stmgr.StateManager + Datastore dtypes.MetadataDS + Wallet api.Wallet + F3ConsensusEnabled F3ConsensusEnabled } var log = logging.Logger("f3") @@ -56,6 +61,8 @@ func New(mctx helpers.MetricsCtx, lc fx.Lifecycle, params F3Params) (*F3, error) ec := &ecWrapper{ ChainStore: params.ChainStore, StateManager: params.StateManager, + Syncer: params.Syncer, + Checkpoint: bool(params.F3ConsensusEnabled), } verif := blssig.VerifierWithKeyOnG1() diff --git a/chain/types/tipset_key.go b/chain/types/tipset_key.go index 1a48781693e..703ff2a4c43 100644 --- a/chain/types/tipset_key.go +++ b/chain/types/tipset_key.go @@ -10,6 +10,7 @@ import ( block "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" typegen "github.com/whyrusleeping/cbor-gen" + "golang.org/x/xerrors" "github.com/filecoin-project/go-state-types/abi" ) @@ -52,7 +53,7 @@ func NewTipSetKey(cids ...cid.Cid) TipSetKey { func TipSetKeyFromBytes(encoded []byte) (TipSetKey, error) { _, err := decodeKey(encoded) if err != nil { - return EmptyTSK, err + return EmptyTSK, xerrors.Errorf("decoding tpiset key: %w", err) } return TipSetKey{string(encoded)}, nil } diff --git a/go.mod b/go.mod index 722328780ab..0b48e069d57 100644 --- a/go.mod +++ b/go.mod @@ -44,7 +44,7 @@ require ( github.com/filecoin-project/go-cbor-util v0.0.1 github.com/filecoin-project/go-commp-utils/v2 v2.1.0 github.com/filecoin-project/go-crypto v0.1.0 - github.com/filecoin-project/go-f3 v0.2.0 + github.com/filecoin-project/go-f3 v0.3.0 github.com/filecoin-project/go-fil-commcid v0.2.0 github.com/filecoin-project/go-hamt-ipld/v3 v3.4.0 github.com/filecoin-project/go-jsonrpc v0.6.0 @@ -171,7 +171,6 @@ require ( require ( github.com/GeertJohan/go.incremental v1.0.0 // indirect github.com/Jorropo/jsync v1.0.1 // indirect - github.com/Kubuxu/go-broadcast v0.0.0-20240621161059-1a8c90734cd6 // indirect github.com/PuerkitoBio/purell v1.1.1 // indirect github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect github.com/StackExchange/wmi v1.2.1 // indirect diff --git a/go.sum b/go.sum index 86208ebb19b..4d1ebb2562b 100644 --- a/go.sum +++ b/go.sum @@ -58,8 +58,6 @@ github.com/Gurpartap/async v0.0.0-20180927173644-4f7f499dd9ee h1:8doiS7ib3zi6/K1 github.com/Gurpartap/async v0.0.0-20180927173644-4f7f499dd9ee/go.mod h1:W0GbEAA4uFNYOGG2cJpmFJ04E6SD1NLELPYZB57/7AY= github.com/Jorropo/jsync v1.0.1 h1:6HgRolFZnsdfzRUj+ImB9og1JYOxQoReSywkHOGSaUU= github.com/Jorropo/jsync v1.0.1/go.mod h1:jCOZj3vrBCri3bSU3ErUYvevKlnbssrXeCivybS5ABQ= -github.com/Kubuxu/go-broadcast v0.0.0-20240621161059-1a8c90734cd6 h1:yh2/1fz3ajTaeKskSWxtSBNScdRZfQ/A5nyd9+64T6M= -github.com/Kubuxu/go-broadcast v0.0.0-20240621161059-1a8c90734cd6/go.mod h1:5LOj/fF3Oc/cvJqzDiyfx4XwtBPRWUYEz+V+b13sH5U= github.com/Kubuxu/go-os-helper v0.0.1/go.mod h1:N8B+I7vPCT80IcP58r50u4+gEEcsZETFUpAzWW2ep1Y= github.com/Kubuxu/imtui v0.0.0-20210401140320-41663d68d0fa h1:1PPxEyGdIGVkX/kqMvLJ95a1dGS1Sz7tpNEgehEYYt0= github.com/Kubuxu/imtui v0.0.0-20210401140320-41663d68d0fa/go.mod h1:WUmMvh9wMtqj1Xhf1hf3kp9RvL+y6odtdYxpyZjb90U= @@ -264,8 +262,8 @@ github.com/filecoin-project/go-commp-utils/v2 v2.1.0/go.mod h1:NbxJYlhxtWaNhlVCj github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03/go.mod h1:+viYnvGtUTgJRdy6oaeF4MTFKAfatX071MPDPBL11EQ= github.com/filecoin-project/go-crypto v0.1.0 h1:Pob2MphoipMbe/ksxZOMcQvmBHAd3sI/WEqcbpIsGI0= github.com/filecoin-project/go-crypto v0.1.0/go.mod h1:K9UFXvvoyAVvB+0Le7oGlKiT9mgA5FHOJdYQXEE8IhI= -github.com/filecoin-project/go-f3 v0.2.0 h1:Gis44+hOrDjSUEw3IDmU7CudNILi5e+bb1pgZgp680k= -github.com/filecoin-project/go-f3 v0.2.0/go.mod h1:43fBLX0iX0+Nnw4Z91wSrdfDYAd6YEDexy7GcLnIJtk= +github.com/filecoin-project/go-f3 v0.3.0 h1:GUY7+QSyWqX4MEuFtQAYkYLRM1t3GleZ0U2I87oOStM= +github.com/filecoin-project/go-f3 v0.3.0/go.mod h1:MVf4ynbRdLMnZZWK8GJCex0VH1lQvh9AwFTMEu7jX1Y= github.com/filecoin-project/go-fil-commcid v0.2.0 h1:B+5UX8XGgdg/XsdUpST4pEBviKkFOw+Fvl2bLhSKGpI= github.com/filecoin-project/go-fil-commcid v0.2.0/go.mod h1:8yigf3JDIil+/WpqR5zoKyP0jBPCOGtEqq/K1CcMy9Q= github.com/filecoin-project/go-fil-commp-hashhash v0.2.0 h1:HYIUugzjq78YvV3vC6rL95+SfC/aSTVSnZSZiDV5pCk= diff --git a/node/builder_chain.go b/node/builder_chain.go index cdf529cb743..72d8a0489ff 100644 --- a/node/builder_chain.go +++ b/node/builder_chain.go @@ -13,6 +13,7 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" + "github.com/filecoin-project/lotus/build/buildconstants" "github.com/filecoin-project/lotus/chain" "github.com/filecoin-project/lotus/chain/beacon" "github.com/filecoin-project/lotus/chain/consensus" @@ -164,6 +165,7 @@ var ChainNode = Options( ), If(build.IsF3Enabled(), + Override(new(*lf3.F3ConsensusEnabled), buildconstants.F3Consensus), Override(new(manifest.ManifestProvider), lf3.NewManifestProvider), Override(new(*lf3.F3), lf3.New), ),