From 1b9b81537b34424eac56b48c67b921ca7d994d19 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Mon, 7 Oct 2024 15:03:55 +0000 Subject: [PATCH] feat(f3): prepare for f3 bootstrap (#12552) This patch: 1. Bootstraps F3 when we hit the `F3BootstrapEpoch` (when non-negative). 2. Refuses any/all dynamic manifests once we get within one finality of said epoch. 3. Sets the F3 network name for mainnet to "filecoin". 4. Refuses any/all dynamic manifests that don't start with the expected network name prefix. --- chain/lf3/config.go | 98 +++++++++++++++++++++++++++---------- chain/lf3/manifest.go | 68 ++++++++++++++++++++----- go.mod | 2 +- go.sum | 4 +- itests/f3_test.go | 58 ++++++++++++++++++++-- itests/kit/node_opts.go | 29 ++--------- node/builder_chain.go | 6 +-- node/modules/lp2p/pubsub.go | 7 ++- 8 files changed, 195 insertions(+), 77 deletions(-) diff --git a/chain/lf3/config.go b/chain/lf3/config.go index 8c669ded94..2472ca28a8 100644 --- a/chain/lf3/config.go +++ b/chain/lf3/config.go @@ -8,6 +8,7 @@ import ( "github.com/filecoin-project/go-f3/gpbft" "github.com/filecoin-project/go-f3/manifest" + "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/lotus/build/buildconstants" "github.com/filecoin-project/lotus/chain/actors/policy" @@ -15,34 +16,81 @@ import ( ) type Config struct { - InitialManifest *manifest.Manifest + // BaseNetworkName is the base from which dynamic network names are defined and is usually + // the name of the network defined by the static manifest. This must be set correctly or, + // e.g., pubsub topic filters won't work correctly. + BaseNetworkName gpbft.NetworkName + // StaticManifest this instance's default manifest absent any dynamic manifests. Also see + // PrioritizeStaticManifest. + StaticManifest *manifest.Manifest + // DynamicManifestProvider is the peer ID of the peer authorized to send us dynamic manifest + // updates. Dynamic manifest updates can be used for testing but will not be used to affect + // finality. DynamicManifestProvider peer.ID + // PrioritizeStaticManifest means that, once we get within one finality of the static + // manifest's bootstrap epoch we'll switch to it and ignore any further dynamic manifest + // updates. This exists to enable bootstrapping F3. + PrioritizeStaticManifest bool + // TESTINGAllowDynamicFinalize allow dynamic manifests to finalize tipsets. DO NOT ENABLE + // THIS IN PRODUCTION! + AllowDynamicFinalize bool } -func NewConfig(manifestProvider peer.ID, initialPowerTable cid.Cid) func(dtypes.NetworkName) *Config { - return func(nn dtypes.NetworkName) *Config { - m := manifest.LocalDevnetManifest() - m.NetworkName = gpbft.NetworkName(nn) - m.EC.Period = time.Duration(buildconstants.BlockDelaySecs) * time.Second - m.CatchUpAlignment = time.Duration(buildconstants.BlockDelaySecs) * time.Second / 2 - if buildconstants.F3BootstrapEpoch < 0 { - // if unset, set to a sane default so we don't get scary logs and pause. - m.BootstrapEpoch = 2 * int64(policy.ChainFinality) - m.Pause = true - } else { - m.BootstrapEpoch = int64(buildconstants.F3BootstrapEpoch) - } - m.EC.Finality = int64(policy.ChainFinality) - m.CommitteeLookback = 5 - m.InitialPowerTable = initialPowerTable - m.EC.Finalize = buildconstants.F3Consensus +// NewManifest constructs a sane F3 manifest based on the passed parameters. This function does not +// look at and/or depend on the nodes build params, etc. +func NewManifest( + nn gpbft.NetworkName, + finality, bootstrapEpoch abi.ChainEpoch, + ecPeriod time.Duration, + initialPowerTable cid.Cid, +) *manifest.Manifest { + return &manifest.Manifest{ + ProtocolVersion: manifest.VersionCapability, + BootstrapEpoch: int64(bootstrapEpoch), + NetworkName: nn, + InitialPowerTable: initialPowerTable, + CommitteeLookback: manifest.DefaultCommitteeLookback, + CatchUpAlignment: ecPeriod / 2, + Gpbft: manifest.DefaultGpbftConfig, + EC: manifest.EcConfig{ + Period: ecPeriod, + Finality: int64(finality), + DelayMultiplier: manifest.DefaultEcConfig.DelayMultiplier, + BaseDecisionBackoffTable: manifest.DefaultEcConfig.BaseDecisionBackoffTable, + HeadLookback: 0, + Finalize: true, + }, + CertificateExchange: manifest.CxConfig{ + ClientRequestTimeout: manifest.DefaultCxConfig.ClientRequestTimeout, + ServerRequestTimeout: manifest.DefaultCxConfig.ServerRequestTimeout, + MinimumPollInterval: ecPeriod, + MaximumPollInterval: 4 * ecPeriod, + }, + } +} - // TODO: We're forcing this to start paused for now. We need to remove this for the final - // mainnet launch. - m.Pause = true - return &Config{ - InitialManifest: m, - DynamicManifestProvider: manifestProvider, - } +// NewConfig creates a new F3 config based on the node's build parameters and the passed network +// name. +func NewConfig(nn dtypes.NetworkName) *Config { + // Use "filecoin" as the network name on mainnet, otherwise use the network name. Yes, + // mainnet is called testnetnet in state. + if nn == "testnetnet" { + nn = "filecoin" + } + c := &Config{ + BaseNetworkName: gpbft.NetworkName(nn), + PrioritizeStaticManifest: true, + DynamicManifestProvider: buildconstants.F3ManifestServerID, + AllowDynamicFinalize: false, + } + if buildconstants.F3BootstrapEpoch >= 0 { + c.StaticManifest = NewManifest( + c.BaseNetworkName, + policy.ChainFinality, + buildconstants.F3BootstrapEpoch, + time.Duration(buildconstants.BlockDelaySecs)*time.Second, + buildconstants.F3InitialPowerTableCID, + ) } + return c } diff --git a/chain/lf3/manifest.go b/chain/lf3/manifest.go index e4a944f090..2b64cc4b22 100644 --- a/chain/lf3/manifest.go +++ b/chain/lf3/manifest.go @@ -1,45 +1,91 @@ package lf3 import ( + "context" "fmt" + "strings" "github.com/ipfs/go-datastore" "github.com/ipfs/go-datastore/namespace" pubsub "github.com/libp2p/go-libp2p-pubsub" + "golang.org/x/xerrors" + "github.com/filecoin-project/go-f3/ec" "github.com/filecoin-project/go-f3/manifest" + "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/node/modules/dtypes" + "github.com/filecoin-project/lotus/node/modules/helpers" ) +type headGetter store.ChainStore + +func (hg *headGetter) GetHead(context.Context) (ec.TipSet, error) { + head := (*store.ChainStore)(hg).GetHeaviestTipSet() + if head == nil { + return nil, xerrors.New("no heaviest tipset") + } + return &f3TipSet{TipSet: head}, nil +} + // Determines the max. number of configuration changes // that are allowed for the dynamic manifest. // If the manifest changes more than this number, the F3 // message topic will be filtered var MaxDynamicManifestChangesAllowed = 1000 -func NewManifestProvider(config *Config, ps *pubsub.PubSub, mds dtypes.MetadataDS) (manifest.ManifestProvider, error) { +func NewManifestProvider(mctx helpers.MetricsCtx, config *Config, cs *store.ChainStore, ps *pubsub.PubSub, mds dtypes.MetadataDS) (prov manifest.ManifestProvider, err error) { if config.DynamicManifestProvider == "" { - return manifest.NewStaticManifestProvider(config.InitialManifest) + if config.StaticManifest == nil { + return manifest.NoopManifestProvider{}, nil + } + return manifest.NewStaticManifestProvider(config.StaticManifest) + } + + opts := []manifest.DynamicManifestProviderOption{ + manifest.DynamicManifestProviderWithDatastore( + namespace.Wrap(mds, datastore.NewKey("/f3-dynamic-manifest")), + ), } - primaryNetworkName := config.InitialManifest.NetworkName + if config.StaticManifest != nil { + opts = append(opts, + manifest.DynamicManifestProviderWithInitialManifest(config.StaticManifest), + ) + } + + if config.AllowDynamicFinalize { + log.Error("dynamic F3 manifests are allowed to finalize tipsets, do not enable this in production!") + } + + networkNameBase := config.BaseNetworkName + "/" filter := func(m *manifest.Manifest) error { if m.EC.Finalize { - return fmt.Errorf("refusing dynamic manifest that finalizes tipsets") + if !config.AllowDynamicFinalize { + return fmt.Errorf("refusing dynamic manifest that finalizes tipsets") + } + log.Error("WARNING: loading a dynamic F3 manifest that will finalize new tipsets") } - if m.NetworkName == primaryNetworkName { + if !strings.HasPrefix(string(m.NetworkName), string(networkNameBase)) { return fmt.Errorf( - "refusing dynamic manifest with network name %q that clashes with initial manifest", - primaryNetworkName, + "refusing dynamic manifest with network name %q, must start with %q", + m.NetworkName, + networkNameBase, ) } return nil } - ds := namespace.Wrap(mds, datastore.NewKey("/f3-dynamic-manifest")) - return manifest.NewDynamicManifestProvider(ps, config.DynamicManifestProvider, - manifest.DynamicManifestProviderWithInitialManifest(config.InitialManifest), - manifest.DynamicManifestProviderWithDatastore(ds), + opts = append(opts, manifest.DynamicManifestProviderWithFilter(filter), ) + + prov, err = manifest.NewDynamicManifestProvider(ps, config.DynamicManifestProvider, opts...) + if err != nil { + return nil, err + } + if config.PrioritizeStaticManifest && config.StaticManifest != nil { + prov, err = manifest.NewFusingManifestProvider(mctx, + (*headGetter)(cs), prov, config.StaticManifest) + } + return prov, err } diff --git a/go.mod b/go.mod index 2fee44d90a..8997cbca17 100644 --- a/go.mod +++ b/go.mod @@ -43,7 +43,7 @@ require ( github.com/filecoin-project/go-cbor-util v0.0.1 github.com/filecoin-project/go-commp-utils/v2 v2.1.0 github.com/filecoin-project/go-crypto v0.1.0 - github.com/filecoin-project/go-f3 v0.4.0 + github.com/filecoin-project/go-f3 v0.5.0 github.com/filecoin-project/go-fil-commcid v0.2.0 github.com/filecoin-project/go-hamt-ipld/v3 v3.4.0 github.com/filecoin-project/go-jsonrpc v0.6.0 diff --git a/go.sum b/go.sum index ff09fd9a65..c15dfe158b 100644 --- a/go.sum +++ b/go.sum @@ -273,8 +273,8 @@ github.com/filecoin-project/go-commp-utils/v2 v2.1.0/go.mod h1:NbxJYlhxtWaNhlVCj github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03/go.mod h1:+viYnvGtUTgJRdy6oaeF4MTFKAfatX071MPDPBL11EQ= github.com/filecoin-project/go-crypto v0.1.0 h1:Pob2MphoipMbe/ksxZOMcQvmBHAd3sI/WEqcbpIsGI0= github.com/filecoin-project/go-crypto v0.1.0/go.mod h1:K9UFXvvoyAVvB+0Le7oGlKiT9mgA5FHOJdYQXEE8IhI= -github.com/filecoin-project/go-f3 v0.4.0 h1:3UUjFMmZYvytDZPI5oeMroaEGO691icQM/7XoioYVxg= -github.com/filecoin-project/go-f3 v0.4.0/go.mod h1:QoxuoK4aktNZD1R/unlhNbhV6TnlNTAbA/QODCnAjak= +github.com/filecoin-project/go-f3 v0.5.0 h1:pRw8A9moEXSxK8v4nmR/Hs09TO2Wrz9h/Al76yt+8jI= +github.com/filecoin-project/go-f3 v0.5.0/go.mod h1:QoxuoK4aktNZD1R/unlhNbhV6TnlNTAbA/QODCnAjak= github.com/filecoin-project/go-fil-commcid v0.2.0 h1:B+5UX8XGgdg/XsdUpST4pEBviKkFOw+Fvl2bLhSKGpI= github.com/filecoin-project/go-fil-commcid v0.2.0/go.mod h1:8yigf3JDIil+/WpqR5zoKyP0jBPCOGtEqq/K1CcMy9Q= github.com/filecoin-project/go-fil-commp-hashhash v0.2.0 h1:HYIUugzjq78YvV3vC6rL95+SfC/aSTVSnZSZiDV5pCk= diff --git a/itests/f3_test.go b/itests/f3_test.go index f3a1ce6561..0a96fa4d54 100644 --- a/itests/f3_test.go +++ b/itests/f3_test.go @@ -5,15 +5,18 @@ import ( "testing" "time" + "github.com/ipfs/go-cid" "github.com/libp2p/go-libp2p" pubsub "github.com/libp2p/go-libp2p-pubsub" "github.com/libp2p/go-libp2p/core/host" "github.com/stretchr/testify/require" "golang.org/x/sync/errgroup" + "github.com/filecoin-project/go-f3/gpbft" "github.com/filecoin-project/go-f3/manifest" "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/lotus/chain/lf3" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/itests/kit" "github.com/filecoin-project/lotus/node" @@ -21,8 +24,9 @@ import ( ) const ( - DefaultBootsrapEpoch = 20 - DefaultFinality = 5 + DefaultBootstrapEpoch = 20 + DefaultFinality = 5 + BaseNetworkName gpbft.NetworkName = "test" ) type testEnv struct { @@ -61,7 +65,7 @@ func TestF3_Rebootstrap(t *testing.T) { cpy := *e.m cpy.BootstrapEpoch = 25 - cpy.NetworkName += "/1" + cpy.NetworkName = BaseNetworkName + "/2" e.ms.UpdateManifest(&cpy) newManifest := e.waitTillManifestChange(&cpy, 20*time.Second) @@ -90,7 +94,7 @@ func TestF3_PauseAndRebootstrap(t *testing.T) { e.waitTillF3Runs(30 * time.Second) cpy := *e.m - cpy.NetworkName += "/1" + cpy.NetworkName = BaseNetworkName + "/2" cpy.BootstrapEpoch = 25 e.ms.UpdateManifest(&cpy) @@ -98,6 +102,35 @@ func TestF3_PauseAndRebootstrap(t *testing.T) { e.waitTillF3Rebootstrap(20 * time.Second) } +// Tests that pause/resume and rebootstrapping F3 works +func TestF3_Bootstrap(t *testing.T) { + kit.QuietMiningLogs() + + var bootstrapEpoch abi.ChainEpoch = 50 + blocktime := 100 * time.Millisecond + staticManif := lf3.NewManifest(BaseNetworkName, DefaultFinality, bootstrapEpoch, blocktime, cid.Undef) + dynamicManif := *staticManif + dynamicManif.BootstrapEpoch = 5 + dynamicManif.EC.Finalize = false + dynamicManif.NetworkName = BaseNetworkName + "/1" + + e := setupWithStaticManifest(t, staticManif, true) + e.ms.UpdateManifest(&dynamicManif) + e.waitTillManifestChange(&dynamicManif, 20*time.Second) + e.waitTillF3Instance(2, 20*time.Second) + e.waitTillManifestChange(staticManif, 20*time.Second) + e.waitTillF3Instance(2, 20*time.Second) + + // Try to switch back, we should ignore the manifest update. + e.ms.UpdateManifest(&dynamicManif) + time.Sleep(time.Second) + for _, n := range e.minerFullNodes { + m, err := n.F3GetManifest(e.testCtx) + require.NoError(e.t, err) + require.True(t, m.Equal(staticManif)) + } +} + func (e *testEnv) waitTillF3Rebootstrap(timeout time.Duration) { e.waitFor(func(n *kit.TestFullNode) bool { // the prev epoch yet, check if we already bootstrapped and from @@ -176,9 +209,16 @@ func (e *testEnv) waitFor(f func(n *kit.TestFullNode) bool, timeout time.Duratio // and the second full-node is an observer that is not directly connected to // a miner. The last return value is the manifest sender for the network. func setup(t *testing.T, blocktime time.Duration) *testEnv { + manif := lf3.NewManifest(BaseNetworkName+"/1", DefaultFinality, DefaultBootstrapEpoch, blocktime, cid.Undef) + return setupWithStaticManifest(t, manif, false) +} + +func setupWithStaticManifest(t *testing.T, manif *manifest.Manifest, testBootstrap bool) *testEnv { ctx, stopServices := context.WithCancel(context.Background()) errgrp, ctx := errgroup.WithContext(ctx) + blocktime := manif.EC.Period + t.Cleanup(func() { stopServices() require.NoError(t, errgrp.Wait()) @@ -188,7 +228,15 @@ func setup(t *testing.T, blocktime time.Duration) *testEnv { manifestServerHost, err := libp2p.New(libp2p.ListenAddrStrings("/ip4/127.0.0.1/udp/0/quic-v1")) require.NoError(t, err) - f3NOpt := kit.F3Enabled(DefaultBootsrapEpoch, blocktime, DefaultFinality, manifestServerHost.ID()) + cfg := &lf3.Config{ + BaseNetworkName: BaseNetworkName, + StaticManifest: manif, + DynamicManifestProvider: manifestServerHost.ID(), + PrioritizeStaticManifest: testBootstrap, + AllowDynamicFinalize: !testBootstrap, + } + + f3NOpt := kit.F3Enabled(cfg) f3MOpt := kit.ConstructorOpts(node.Override(node.F3Participation, modules.F3Participation)) var ( diff --git a/itests/kit/node_opts.go b/itests/kit/node_opts.go index 6755988caa..6a50e60ff7 100644 --- a/itests/kit/node_opts.go +++ b/itests/kit/node_opts.go @@ -2,11 +2,6 @@ package kit import ( "math" - "time" - - "github.com/ipfs/go-cid" - pubsub "github.com/libp2p/go-libp2p-pubsub" - "github.com/libp2p/go-libp2p/core/peer" "github.com/filecoin-project/go-f3/manifest" "github.com/filecoin-project/go-state-types/abi" @@ -215,28 +210,10 @@ func MutateSealingConfig(mut func(sc *config.SealingConfig)) NodeOpt { }))) } -func F3Enabled(bootstrapEpoch abi.ChainEpoch, blockDelay time.Duration, finality abi.ChainEpoch, manifestProvider peer.ID) NodeOpt { +func F3Enabled(cfg *lf3.Config) NodeOpt { return ConstructorOpts( - node.Override(new(*lf3.Config), func(nn dtypes.NetworkName) *lf3.Config { - c := lf3.NewConfig(manifestProvider, cid.Undef)(nn) - c.InitialManifest.Pause = false - c.InitialManifest.EC.Period = blockDelay - c.InitialManifest.Gpbft.Delta = blockDelay / 5 - c.InitialManifest.EC.Finality = int64(finality) - c.InitialManifest.BootstrapEpoch = int64(bootstrapEpoch) - c.InitialManifest.EC.HeadLookback = 0 - c.InitialManifest.EC.Finalize = true - c.InitialManifest.CatchUpAlignment = blockDelay / 2 - c.InitialManifest.CertificateExchange.MinimumPollInterval = 2 * blockDelay - c.InitialManifest.CertificateExchange.MaximumPollInterval = 10 * blockDelay - return c - }), - node.Override(new(manifest.ManifestProvider), - func(config *lf3.Config, ps *pubsub.PubSub) (manifest.ManifestProvider, error) { - return manifest.NewDynamicManifestProvider(ps, config.DynamicManifestProvider, - manifest.DynamicManifestProviderWithInitialManifest(config.InitialManifest), - ) - }), + node.Override(new(*lf3.Config), cfg), + node.Override(new(manifest.ManifestProvider), lf3.NewManifestProvider), node.Override(new(*lf3.F3), lf3.New), ) } diff --git a/node/builder_chain.go b/node/builder_chain.go index 480c5cb27f..d367ab4e3e 100644 --- a/node/builder_chain.go +++ b/node/builder_chain.go @@ -13,7 +13,6 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" - "github.com/filecoin-project/lotus/build/buildconstants" "github.com/filecoin-project/lotus/chain" "github.com/filecoin-project/lotus/chain/beacon" "github.com/filecoin-project/lotus/chain/consensus" @@ -165,10 +164,7 @@ var ChainNode = Options( ), If(build.IsF3Enabled(), - Override(new(*lf3.Config), lf3.NewConfig( - buildconstants.F3ManifestServerID, - buildconstants.F3InitialPowerTableCID, - )), + Override(new(*lf3.Config), lf3.NewConfig), Override(new(manifest.ManifestProvider), lf3.NewManifestProvider), Override(new(*lf3.F3), lf3.New), ), diff --git a/node/modules/lp2p/pubsub.go b/node/modules/lp2p/pubsub.go index 0c8f7b291f..20a222cd21 100644 --- a/node/modules/lp2p/pubsub.go +++ b/node/modules/lp2p/pubsub.go @@ -388,9 +388,12 @@ func GossipSub(in GossipIn) (service *pubsub.PubSub, err error) { allowTopics = append(allowTopics, drandTopics...) if in.F3Config != nil { - f3BaseTopicName := manifest.PubSubTopicFromNetworkName(in.F3Config.InitialManifest.NetworkName) - allowTopics = append(allowTopics, f3BaseTopicName) + if in.F3Config.StaticManifest != nil { + f3TopicName := manifest.PubSubTopicFromNetworkName(in.F3Config.StaticManifest.NetworkName) + allowTopics = append(allowTopics, f3TopicName) + } if in.F3Config.DynamicManifestProvider != "" { + f3BaseTopicName := manifest.PubSubTopicFromNetworkName(in.F3Config.BaseNetworkName) allowTopics = append(allowTopics, manifest.ManifestPubSubTopicName) for i := 0; i < lf3.MaxDynamicManifestChangesAllowed; i++ { allowTopics = append(allowTopics, fmt.Sprintf("%s/%d", f3BaseTopicName, i))