Skip to content

Commit

Permalink
Allow nodes to be started ready for remote smeshing (#5314)
Browse files Browse the repository at this point in the history
## Motivation
This changes the node startup in a way to allow a node to receive a remote PoST service connection.

## Changes
- if `--smeshing-start` is provided, but no `--smeshing-coinbase` the node will panic during startup
  - if both are provided node operates in supervised mode (smapp usecase)
  - if the provided coinbase is not valid the node will also panic during startup
- if `--smeshing-coinbase` is provided, but not `--smeshing-start` the node will be prepared to accept connections from remote PoST services
  - this additionally requires the following config parameters to be set to actually work (**TODO**: verify those during startup?)
    - `grpc-tls-listener`
    - `grpc-tls-ca-cert`
    - `grpc-tls-cert`
    - `grpc-tls-key`
- if neither `--smeshing-start` nor `--smeshing-coinbase` are provided the node can still be instructed to start (and stop) the supervised PoST mode via GRPC (**required for system tests** - coinbase is passed as argument and verified by node)
- if `--smeshing-coinbase` is not defined, but the TLS listener is set up correctly, remote connections should succeed but the node will not start smeshing (i.e. register at PoET, generate PoST proofs, publish ATXs, etc.)

## Test Plan
TODO: add system tests with remote post service

## TODO
<!-- This section should be removed when all items are complete -->
- [x] Explain motivation or link existing issue(s)
- [x] Test changes and document test plan
- [x] Update documentation as needed
- [ ] Update [changelog](../CHANGELOG.md) as needed
  • Loading branch information
fasmat committed Nov 30, 2023
1 parent 65b77b2 commit 4f0520e
Show file tree
Hide file tree
Showing 6 changed files with 50 additions and 69 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ jobs:
shell: bash
run: |
make install
make build VERSION=${{ github.ref_name }} BIN_DIR_WIN=./build
make build VERSION=${{ github.ref_name }}
- name: Create release archive
shell: bash
Expand Down
2 changes: 0 additions & 2 deletions activation/activation.go
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,6 @@ const (

// Config defines configuration for Builder.
type Config struct {
CoinbaseAccount types.Address
GoldenATXID types.ATXID
LayersPerEpoch uint32
RegossipInterval time.Duration
Expand Down Expand Up @@ -143,7 +142,6 @@ func NewBuilder(
b := &Builder{
parentCtx: context.Background(),
signer: signer,
coinbaseAccount: conf.CoinbaseAccount,
goldenATXID: conf.GoldenATXID,
regossipInterval: conf.RegossipInterval,
cdb: cdb,
Expand Down
7 changes: 2 additions & 5 deletions activation/activation_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,6 @@ type testAtxBuilder struct {
cdb *datastore.CachedDB
localDb *localsql.Database
sig *signing.EdSigner
coinbase types.Address
goldenATXID types.ATXID

mpub *mocks.MockPublisher
Expand All @@ -128,7 +127,6 @@ func newTestBuilder(tb testing.TB, opts ...BuilderOption) *testAtxBuilder {
cdb: datastore.NewCachedDB(sql.InMemory(), lg),
localDb: localsql.InMemory(),
sig: edSigner,
coinbase: types.GenerateAddress([]byte("33333")),
goldenATXID: types.ATXID(types.HexToHash32("77777")),
mpub: mocks.NewMockPublisher(ctrl),
mpostSvc: NewMockpostService(ctrl),
Expand All @@ -142,9 +140,8 @@ func newTestBuilder(tb testing.TB, opts ...BuilderOption) *testAtxBuilder {
opts = append(opts, WithValidator(tab.mValidator))

cfg := Config{
CoinbaseAccount: tab.coinbase,
GoldenATXID: tab.goldenATXID,
LayersPerEpoch: layersPerEpoch,
GoldenATXID: tab.goldenATXID,
LayersPerEpoch: layersPerEpoch,
}

tab.msync.EXPECT().RegisterForATXSynced().DoAndReturn(closedChan).AnyTimes()
Expand Down
2 changes: 1 addition & 1 deletion api/grpcserver/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ type Config struct {
PrivateListener string `mapstructure:"grpc-private-listener"`
TLSServices []Service
TLSListener string `mapstructure:"grpc-tls-listener"`
TLSCACert string `mapstructure:"gprc-tls-ca-cert"`
TLSCACert string `mapstructure:"grpc-tls-ca-cert"`
TLSCert string `mapstructure:"grpc-tls-cert"`
TLSKey string `mapstructure:"grpc-tls-key"`
GrpcSendMsgSize int `mapstructure:"grpc-send-msg-size"`
Expand Down
104 changes: 45 additions & 59 deletions node/node.go
Original file line number Diff line number Diff line change
Expand Up @@ -191,7 +191,7 @@ func GetCommand() *cobra.Command {
// os.Interrupt for all systems, especially windows, syscall.SIGTERM is mainly for docker.
ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer cancel()
if err = run(ctx); err != nil {
if err := run(ctx); err != nil {
app.log.With().Fatal(err.Error())
}
},
Expand Down Expand Up @@ -665,7 +665,7 @@ func (app *App) initServices(ctx context.Context) error {
mlog := app.addLogger(MeshLogger, lg)
msh, err := mesh.NewMesh(app.cachedDB, app.atxsdata, app.clock, trtl, executor, app.conState, mlog)
if err != nil {
return fmt.Errorf("failed to create mesh: %w", err)
return fmt.Errorf("create mesh: %w", err)
}

pruner := prune.New(app.db, app.Config.Tortoise.Hdist, app.Config.PruneActivesetsFrom, prune.WithLogger(mlog.Zap()))
Expand Down Expand Up @@ -866,38 +866,34 @@ func (app *App) initServices(ctx context.Context) error {
)
proposalBuilder.Register(app.edSgn)

if app.Config.SMESHING.Start {
u := url.URL{
Scheme: "http",
Host: app.Config.API.PrivateListener,
}
app.Config.POSTService.NodeAddress = u.String()

postSetupMgr, err := activation.NewPostSetupManager(
app.edSgn.NodeID(),
app.Config.POST,
app.addLogger(PostLogger, lg).Zap(),
app.cachedDB, goldenATXID,
)
if err != nil {
app.log.Panic("failed to create post setup manager: %v", err)
}
u := url.URL{
Scheme: "http",
Host: app.Config.API.PrivateListener,
}
app.Config.POSTService.NodeAddress = u.String()
postSetupMgr, err := activation.NewPostSetupManager(
app.edSgn.NodeID(),
app.Config.POST,
app.addLogger(PostLogger, lg).Zap(),
app.cachedDB, goldenATXID,
)
if err != nil {
return fmt.Errorf("create post setup manager: %v", err)
}

app.postSupervisor, err = activation.NewPostSupervisor(
app.log.Zap(),
app.Config.POSTService,
app.Config.POST,
app.Config.SMESHING.ProvingOpts,
postSetupMgr,
newSyncer,
)
if err != nil {
return fmt.Errorf("init post service: %w", err)
}
app.postSupervisor, err = activation.NewPostSupervisor(
app.log.Zap(),
app.Config.POSTService,
app.Config.POST,
app.Config.SMESHING.ProvingOpts,
postSetupMgr,
newSyncer,
)
if err != nil {
return fmt.Errorf("init post service: %w", err)
}

app.grpcPostService = grpcserver.NewPostService(app.addLogger(PostServiceLogger, lg).Zap())

nipostBuilder, err := activation.NewNIPostBuilder(
poetDb,
app.grpcPostService,
Expand All @@ -909,29 +905,10 @@ func (app *App) initServices(ctx context.Context) error {
app.clock,
)
if err != nil {
app.log.Panic("failed to create nipost builder: %v", err)
}

var coinbaseAddr types.Address
if app.Config.SMESHING.Start {
// TODO(mafa): this won't work with a remote-only setup, where `smeshing-start` is set to false
// TODO(mafa): also the way we handle coinbase means only 1 address can receive rewards, independent
// of the number of identities/post services that are managed by the node
coinbaseAddr, err = types.StringToAddress(app.Config.SMESHING.CoinbaseAccount)
if err != nil {
app.log.Panic(
"failed to parse CoinbaseAccount address `%s`: %v",
app.Config.SMESHING.CoinbaseAccount,
err,
)
}
if coinbaseAddr.IsEmpty() {
app.log.Panic("invalid coinbase account")
}
return fmt.Errorf("create nipost builder: %w", err)
}

builderConfig := activation.Config{
CoinbaseAccount: coinbaseAddr,
GoldenATXID: goldenATXID,
LayersPerEpoch: layersPerEpoch,
RegossipInterval: app.Config.RegossipAtxInterval,
Expand Down Expand Up @@ -1179,7 +1156,7 @@ func (app *App) listenToUpdates(ctx context.Context) {

func (app *App) startServices(ctx context.Context) error {
if err := app.fetcher.Start(); err != nil {
return fmt.Errorf("failed to start fetcher: %w", err)
return fmt.Errorf("start fetcher: %w", err)
}
app.syncer.Start()
app.beaconProtocol.Start(ctx)
Expand All @@ -1190,17 +1167,23 @@ func (app *App) startServices(ctx context.Context) error {
return app.proposalBuilder.Run(ctx)
})

if app.Config.SMESHING.Start {
if app.Config.SMESHING.CoinbaseAccount != "" {
coinbaseAddr, err := types.StringToAddress(app.Config.SMESHING.CoinbaseAccount)
if err != nil {
app.log.Panic(
"failed to parse CoinbaseAccount address on start `%s`: %v",
return fmt.Errorf(
"parse CoinbaseAccount address on start `%s`: %w",
app.Config.SMESHING.CoinbaseAccount,
err,
)
}
if err := app.atxBuilder.StartSmeshing(coinbaseAddr); err != nil {
app.log.Panic("failed to start smeshing: %v", err)
return fmt.Errorf("start smeshing: %w", err)
}
}

if app.Config.SMESHING.Start {
if app.Config.SMESHING.CoinbaseAccount == "" {
return fmt.Errorf("smeshing enabled but no coinbase account provided")
}
if err := app.postSupervisor.Start(app.Config.SMESHING.Opts); err != nil {
return fmt.Errorf("start post service: %w", err)
Expand Down Expand Up @@ -1352,6 +1335,9 @@ func (app *App) startAPIServices(ctx context.Context) error {
if err != nil {
return err
}
if err := app.grpcTLSServer.Start(); err != nil {
return err
}
}

if len(app.Config.API.JSONListener) > 0 {
Expand Down Expand Up @@ -1579,7 +1565,7 @@ func (app *App) setupDBs(ctx context.Context, lg log.Log) error {
)
migrations, err = sql.LocalMigrations()
if err != nil {
return fmt.Errorf("failed to load local migrations: %w", err)
return fmt.Errorf("load local migrations: %w", err)
}
localDB, err := localsql.Open("file:"+filepath.Join(dbPath, localDbFile),
sql.WithMigrations(migrations),
Expand Down Expand Up @@ -1709,14 +1695,14 @@ func (app *App) startSynchronous(ctx context.Context) (err error) {
p2p.WithNodeReporter(events.ReportNodeStatusUpdate),
)
if err != nil {
return fmt.Errorf("failed to initialize p2p host: %w", err)
return fmt.Errorf("initialize p2p host: %w", err)
}

if err := app.setupDBs(ctx, lg); err != nil {
return err
}
if err := app.initServices(ctx); err != nil {
return fmt.Errorf("cannot start services: %w", err)
return fmt.Errorf("init services: %w", err)
}

if app.Config.CollectMetrics {
Expand All @@ -1735,7 +1721,7 @@ func (app *App) startSynchronous(ctx context.Context) (err error) {
}

if err := app.startServices(ctx); err != nil {
return err
return fmt.Errorf("start services: %w", err)
}

// need post verifying service to start first
Expand Down
2 changes: 1 addition & 1 deletion systest/cluster/nodes.go
Original file line number Diff line number Diff line change
Expand Up @@ -106,7 +106,7 @@ const (
poetConfigMapName = "poet"
spacemeshConfigMapName = "spacemesh"

// smeshers are splitted in 10 approximately equal buckets
// smeshers are split in 10 approximately equal buckets
// to enable running chaos mesh tasks on the different parts of the cluster.
buckets = 10
)
Expand Down

0 comments on commit 4f0520e

Please sign in to comment.