From 4f0520eb73e85ea85e0dd145953cf7520ef8bfc4 Mon Sep 17 00:00:00 2001 From: Matthias Fasching <5011972+fasmat@users.noreply.github.com> Date: Thu, 30 Nov 2023 22:50:49 +0000 Subject: [PATCH] Allow nodes to be started ready for remote smeshing (#5314) ## Motivation This changes the node startup in a way to allow a node to receive a remote PoST service connection. ## Changes - if `--smeshing-start` is provided, but no `--smeshing-coinbase` the node will panic during startup - if both are provided node operates in supervised mode (smapp usecase) - if the provided coinbase is not valid the node will also panic during startup - if `--smeshing-coinbase` is provided, but not `--smeshing-start` the node will be prepared to accept connections from remote PoST services - this additionally requires the following config parameters to be set to actually work (**TODO**: verify those during startup?) - `grpc-tls-listener` - `grpc-tls-ca-cert` - `grpc-tls-cert` - `grpc-tls-key` - if neither `--smeshing-start` nor `--smeshing-coinbase` are provided the node can still be instructed to start (and stop) the supervised PoST mode via GRPC (**required for system tests** - coinbase is passed as argument and verified by node) - if `--smeshing-coinbase` is not defined, but the TLS listener is set up correctly, remote connections should succeed but the node will not start smeshing (i.e. register at PoET, generate PoST proofs, publish ATXs, etc.) ## Test Plan TODO: add system tests with remote post service ## TODO - [x] Explain motivation or link existing issue(s) - [x] Test changes and document test plan - [x] Update documentation as needed - [ ] Update [changelog](../CHANGELOG.md) as needed --- .github/workflows/release.yml | 2 +- activation/activation.go | 2 - activation/activation_test.go | 7 +-- api/grpcserver/config.go | 2 +- node/node.go | 104 +++++++++++++++------------------- systest/cluster/nodes.go | 2 +- 6 files changed, 50 insertions(+), 69 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ea3c455bf3..fdbe51d5a5 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -55,7 +55,7 @@ jobs: shell: bash run: | make install - make build VERSION=${{ github.ref_name }} BIN_DIR_WIN=./build + make build VERSION=${{ github.ref_name }} - name: Create release archive shell: bash diff --git a/activation/activation.go b/activation/activation.go index 68ecdd6868..55e87f4cb9 100644 --- a/activation/activation.go +++ b/activation/activation.go @@ -58,7 +58,6 @@ const ( // Config defines configuration for Builder. type Config struct { - CoinbaseAccount types.Address GoldenATXID types.ATXID LayersPerEpoch uint32 RegossipInterval time.Duration @@ -143,7 +142,6 @@ func NewBuilder( b := &Builder{ parentCtx: context.Background(), signer: signer, - coinbaseAccount: conf.CoinbaseAccount, goldenATXID: conf.GoldenATXID, regossipInterval: conf.RegossipInterval, cdb: cdb, diff --git a/activation/activation_test.go b/activation/activation_test.go index c79e6d7662..f5940cf47d 100644 --- a/activation/activation_test.go +++ b/activation/activation_test.go @@ -107,7 +107,6 @@ type testAtxBuilder struct { cdb *datastore.CachedDB localDb *localsql.Database sig *signing.EdSigner - coinbase types.Address goldenATXID types.ATXID mpub *mocks.MockPublisher @@ -128,7 +127,6 @@ func newTestBuilder(tb testing.TB, opts ...BuilderOption) *testAtxBuilder { cdb: datastore.NewCachedDB(sql.InMemory(), lg), localDb: localsql.InMemory(), sig: edSigner, - coinbase: types.GenerateAddress([]byte("33333")), goldenATXID: types.ATXID(types.HexToHash32("77777")), mpub: mocks.NewMockPublisher(ctrl), mpostSvc: NewMockpostService(ctrl), @@ -142,9 +140,8 @@ func newTestBuilder(tb testing.TB, opts ...BuilderOption) *testAtxBuilder { opts = append(opts, WithValidator(tab.mValidator)) cfg := Config{ - CoinbaseAccount: tab.coinbase, - GoldenATXID: tab.goldenATXID, - LayersPerEpoch: layersPerEpoch, + GoldenATXID: tab.goldenATXID, + LayersPerEpoch: layersPerEpoch, } tab.msync.EXPECT().RegisterForATXSynced().DoAndReturn(closedChan).AnyTimes() diff --git a/api/grpcserver/config.go b/api/grpcserver/config.go index 89351a1f1a..4ae4a67d39 100644 --- a/api/grpcserver/config.go +++ b/api/grpcserver/config.go @@ -12,7 +12,7 @@ type Config struct { PrivateListener string `mapstructure:"grpc-private-listener"` TLSServices []Service TLSListener string `mapstructure:"grpc-tls-listener"` - TLSCACert string `mapstructure:"gprc-tls-ca-cert"` + TLSCACert string `mapstructure:"grpc-tls-ca-cert"` TLSCert string `mapstructure:"grpc-tls-cert"` TLSKey string `mapstructure:"grpc-tls-key"` GrpcSendMsgSize int `mapstructure:"grpc-send-msg-size"` diff --git a/node/node.go b/node/node.go index d4ada5741a..6e0a0e485a 100644 --- a/node/node.go +++ b/node/node.go @@ -191,7 +191,7 @@ func GetCommand() *cobra.Command { // os.Interrupt for all systems, especially windows, syscall.SIGTERM is mainly for docker. ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) defer cancel() - if err = run(ctx); err != nil { + if err := run(ctx); err != nil { app.log.With().Fatal(err.Error()) } }, @@ -665,7 +665,7 @@ func (app *App) initServices(ctx context.Context) error { mlog := app.addLogger(MeshLogger, lg) msh, err := mesh.NewMesh(app.cachedDB, app.atxsdata, app.clock, trtl, executor, app.conState, mlog) if err != nil { - return fmt.Errorf("failed to create mesh: %w", err) + return fmt.Errorf("create mesh: %w", err) } pruner := prune.New(app.db, app.Config.Tortoise.Hdist, app.Config.PruneActivesetsFrom, prune.WithLogger(mlog.Zap())) @@ -866,38 +866,34 @@ func (app *App) initServices(ctx context.Context) error { ) proposalBuilder.Register(app.edSgn) - if app.Config.SMESHING.Start { - u := url.URL{ - Scheme: "http", - Host: app.Config.API.PrivateListener, - } - app.Config.POSTService.NodeAddress = u.String() - - postSetupMgr, err := activation.NewPostSetupManager( - app.edSgn.NodeID(), - app.Config.POST, - app.addLogger(PostLogger, lg).Zap(), - app.cachedDB, goldenATXID, - ) - if err != nil { - app.log.Panic("failed to create post setup manager: %v", err) - } + u := url.URL{ + Scheme: "http", + Host: app.Config.API.PrivateListener, + } + app.Config.POSTService.NodeAddress = u.String() + postSetupMgr, err := activation.NewPostSetupManager( + app.edSgn.NodeID(), + app.Config.POST, + app.addLogger(PostLogger, lg).Zap(), + app.cachedDB, goldenATXID, + ) + if err != nil { + return fmt.Errorf("create post setup manager: %v", err) + } - app.postSupervisor, err = activation.NewPostSupervisor( - app.log.Zap(), - app.Config.POSTService, - app.Config.POST, - app.Config.SMESHING.ProvingOpts, - postSetupMgr, - newSyncer, - ) - if err != nil { - return fmt.Errorf("init post service: %w", err) - } + app.postSupervisor, err = activation.NewPostSupervisor( + app.log.Zap(), + app.Config.POSTService, + app.Config.POST, + app.Config.SMESHING.ProvingOpts, + postSetupMgr, + newSyncer, + ) + if err != nil { + return fmt.Errorf("init post service: %w", err) } app.grpcPostService = grpcserver.NewPostService(app.addLogger(PostServiceLogger, lg).Zap()) - nipostBuilder, err := activation.NewNIPostBuilder( poetDb, app.grpcPostService, @@ -909,29 +905,10 @@ func (app *App) initServices(ctx context.Context) error { app.clock, ) if err != nil { - app.log.Panic("failed to create nipost builder: %v", err) - } - - var coinbaseAddr types.Address - if app.Config.SMESHING.Start { - // TODO(mafa): this won't work with a remote-only setup, where `smeshing-start` is set to false - // TODO(mafa): also the way we handle coinbase means only 1 address can receive rewards, independent - // of the number of identities/post services that are managed by the node - coinbaseAddr, err = types.StringToAddress(app.Config.SMESHING.CoinbaseAccount) - if err != nil { - app.log.Panic( - "failed to parse CoinbaseAccount address `%s`: %v", - app.Config.SMESHING.CoinbaseAccount, - err, - ) - } - if coinbaseAddr.IsEmpty() { - app.log.Panic("invalid coinbase account") - } + return fmt.Errorf("create nipost builder: %w", err) } builderConfig := activation.Config{ - CoinbaseAccount: coinbaseAddr, GoldenATXID: goldenATXID, LayersPerEpoch: layersPerEpoch, RegossipInterval: app.Config.RegossipAtxInterval, @@ -1179,7 +1156,7 @@ func (app *App) listenToUpdates(ctx context.Context) { func (app *App) startServices(ctx context.Context) error { if err := app.fetcher.Start(); err != nil { - return fmt.Errorf("failed to start fetcher: %w", err) + return fmt.Errorf("start fetcher: %w", err) } app.syncer.Start() app.beaconProtocol.Start(ctx) @@ -1190,17 +1167,23 @@ func (app *App) startServices(ctx context.Context) error { return app.proposalBuilder.Run(ctx) }) - if app.Config.SMESHING.Start { + if app.Config.SMESHING.CoinbaseAccount != "" { coinbaseAddr, err := types.StringToAddress(app.Config.SMESHING.CoinbaseAccount) if err != nil { - app.log.Panic( - "failed to parse CoinbaseAccount address on start `%s`: %v", + return fmt.Errorf( + "parse CoinbaseAccount address on start `%s`: %w", app.Config.SMESHING.CoinbaseAccount, err, ) } if err := app.atxBuilder.StartSmeshing(coinbaseAddr); err != nil { - app.log.Panic("failed to start smeshing: %v", err) + return fmt.Errorf("start smeshing: %w", err) + } + } + + if app.Config.SMESHING.Start { + if app.Config.SMESHING.CoinbaseAccount == "" { + return fmt.Errorf("smeshing enabled but no coinbase account provided") } if err := app.postSupervisor.Start(app.Config.SMESHING.Opts); err != nil { return fmt.Errorf("start post service: %w", err) @@ -1352,6 +1335,9 @@ func (app *App) startAPIServices(ctx context.Context) error { if err != nil { return err } + if err := app.grpcTLSServer.Start(); err != nil { + return err + } } if len(app.Config.API.JSONListener) > 0 { @@ -1579,7 +1565,7 @@ func (app *App) setupDBs(ctx context.Context, lg log.Log) error { ) migrations, err = sql.LocalMigrations() if err != nil { - return fmt.Errorf("failed to load local migrations: %w", err) + return fmt.Errorf("load local migrations: %w", err) } localDB, err := localsql.Open("file:"+filepath.Join(dbPath, localDbFile), sql.WithMigrations(migrations), @@ -1709,14 +1695,14 @@ func (app *App) startSynchronous(ctx context.Context) (err error) { p2p.WithNodeReporter(events.ReportNodeStatusUpdate), ) if err != nil { - return fmt.Errorf("failed to initialize p2p host: %w", err) + return fmt.Errorf("initialize p2p host: %w", err) } if err := app.setupDBs(ctx, lg); err != nil { return err } if err := app.initServices(ctx); err != nil { - return fmt.Errorf("cannot start services: %w", err) + return fmt.Errorf("init services: %w", err) } if app.Config.CollectMetrics { @@ -1735,7 +1721,7 @@ func (app *App) startSynchronous(ctx context.Context) (err error) { } if err := app.startServices(ctx); err != nil { - return err + return fmt.Errorf("start services: %w", err) } // need post verifying service to start first diff --git a/systest/cluster/nodes.go b/systest/cluster/nodes.go index 04663e4d4b..27d4a8a8c8 100644 --- a/systest/cluster/nodes.go +++ b/systest/cluster/nodes.go @@ -106,7 +106,7 @@ const ( poetConfigMapName = "poet" spacemeshConfigMapName = "spacemesh" - // smeshers are splitted in 10 approximately equal buckets + // smeshers are split in 10 approximately equal buckets // to enable running chaos mesh tasks on the different parts of the cluster. buckets = 10 )