diff --git a/baseapp/options.go b/baseapp/options.go index e669495f647d..fab11203e237 100644 --- a/baseapp/options.go +++ b/baseapp/options.go @@ -96,6 +96,11 @@ func SetChainID(chainID string) func(*BaseApp) { return func(app *BaseApp) { app.chainID = chainID } } +// SetStoreLoader allows us to customize the rootMultiStore initialization. +func SetStoreLoader(loader StoreLoader) func(*BaseApp) { + return func(app *BaseApp) { app.SetStoreLoader(loader) } +} + func (app *BaseApp) SetName(name string) { if app.sealed { panic("SetName() on sealed BaseApp") diff --git a/server/start.go b/server/start.go index 8a9020f43275..e3c957c7f902 100644 --- a/server/start.go +++ b/server/start.go @@ -3,9 +3,11 @@ package server // DONTCOVER import ( + "bufio" "encoding/hex" "errors" "fmt" + "io" "net" "net/http" "os" @@ -14,19 +16,23 @@ import ( "strings" "time" + "cosmossdk.io/tools/rosetta" + crgserver "cosmossdk.io/tools/rosetta/lib/server" + db "github.com/cometbft/cometbft-db" "github.com/cometbft/cometbft/abci/server" tcmd "github.com/cometbft/cometbft/cmd/cometbft/commands" + cmtjson "github.com/cometbft/cometbft/libs/json" "github.com/cometbft/cometbft/node" "github.com/cometbft/cometbft/p2p" pvm "github.com/cometbft/cometbft/privval" + cmtstate "github.com/cometbft/cometbft/proto/tendermint/state" + cmtproto "github.com/cometbft/cometbft/proto/tendermint/types" "github.com/cometbft/cometbft/proxy" "github.com/cometbft/cometbft/rpc/client/local" - "github.com/spf13/cobra" - "google.golang.org/grpc" - "google.golang.org/grpc/credentials/insecure" + sm "github.com/cometbft/cometbft/state" + "github.com/cometbft/cometbft/store" + cmttypes "github.com/cometbft/cometbft/types" - "cosmossdk.io/tools/rosetta" - crgserver "cosmossdk.io/tools/rosetta/lib/server" "github.com/cosmos/cosmos-sdk/client" "github.com/cosmos/cosmos-sdk/client/flags" "github.com/cosmos/cosmos-sdk/codec" @@ -39,6 +45,10 @@ import ( "github.com/cosmos/cosmos-sdk/telemetry" sdk "github.com/cosmos/cosmos-sdk/types" "github.com/cosmos/cosmos-sdk/types/mempool" + "github.com/cosmos/cosmos-sdk/x/genutil" + "github.com/spf13/cobra" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" ) const ( @@ -88,6 +98,14 @@ const ( // mempool flags FlagMempoolMaxTxs = "mempool.max-txs" + + // testnet keys + KeyIsTestnet = "is-testnet" + KeyNewChainID = "new-chain-ID" + KeyNewOpAddr = "new-operator-addr" + KeyNewValAddr = "new-validator-addr" + KeyUserPubKey = "user-pub-key" + KeyTriggerTestnetUpgrade = "trigger-testnet-upgrade" ) // StartCmd runs the service passed in, either stand-alone or in-process with @@ -164,49 +182,7 @@ is performed. Note, when enabled, gRPC will also be automatically enabled. }, } - cmd.Flags().String(flags.FlagHome, defaultNodeHome, "The application home directory") - cmd.Flags().Bool(flagWithTendermint, true, "Run abci app embedded in-process with tendermint") - cmd.Flags().String(flagAddress, "tcp://0.0.0.0:26658", "Listen address") - cmd.Flags().String(flagTransport, "socket", "Transport protocol: socket, grpc") - cmd.Flags().String(flagTraceStore, "", "Enable KVStore tracing to an output file") - cmd.Flags().String(FlagMinGasPrices, "", "Minimum gas prices to accept for transactions; Any fee in a tx must meet this minimum (e.g. 0.01photino;0.0001stake)") - cmd.Flags().IntSlice(FlagUnsafeSkipUpgrades, []int{}, "Skip a set of upgrade heights to continue the old binary") - cmd.Flags().Uint64(FlagHaltHeight, 0, "Block height at which to gracefully halt the chain and shutdown the node") - cmd.Flags().Uint64(FlagHaltTime, 0, "Minimum block time (in Unix seconds) at which to gracefully halt the chain and shutdown the node") - cmd.Flags().Bool(FlagInterBlockCache, true, "Enable inter-block caching") - cmd.Flags().String(flagCPUProfile, "", "Enable CPU profiling and write to the provided file") - cmd.Flags().Bool(FlagTrace, false, "Provide full stack traces for errors in ABCI Log") - cmd.Flags().String(FlagPruning, pruningtypes.PruningOptionDefault, "Pruning strategy (default|nothing|everything|custom)") - cmd.Flags().Uint64(FlagPruningKeepRecent, 0, "Number of recent heights to keep on disk (ignored if pruning is not 'custom')") - cmd.Flags().Uint64(FlagPruningInterval, 0, "Height interval at which pruned heights are removed from disk (ignored if pruning is not 'custom')") - cmd.Flags().Uint(FlagInvCheckPeriod, 0, "Assert registered invariants every N blocks") - cmd.Flags().Uint64(FlagMinRetainBlocks, 0, "Minimum block height offset during ABCI commit to prune Tendermint blocks") - - cmd.Flags().Bool(FlagAPIEnable, false, "Define if the API server should be enabled") - cmd.Flags().Bool(FlagAPISwagger, false, "Define if swagger documentation should automatically be registered (Note: the API must also be enabled)") - cmd.Flags().String(FlagAPIAddress, serverconfig.DefaultAPIAddress, "the API server address to listen on") - cmd.Flags().Uint(FlagAPIMaxOpenConnections, 1000, "Define the number of maximum open connections") - cmd.Flags().Uint(FlagRPCReadTimeout, 10, "Define the Tendermint RPC read timeout (in seconds)") - cmd.Flags().Uint(FlagRPCWriteTimeout, 0, "Define the Tendermint RPC write timeout (in seconds)") - cmd.Flags().Uint(FlagRPCMaxBodyBytes, 1000000, "Define the Tendermint maximum request body (in bytes)") - cmd.Flags().Bool(FlagAPIEnableUnsafeCORS, false, "Define if CORS should be enabled (unsafe - use it at your own risk)") - - cmd.Flags().Bool(flagGRPCOnly, false, "Start the node in gRPC query only mode (no Tendermint process is started)") - cmd.Flags().Bool(flagGRPCEnable, true, "Define if the gRPC server should be enabled") - cmd.Flags().String(flagGRPCAddress, serverconfig.DefaultGRPCAddress, "the gRPC server address to listen on") - - cmd.Flags().Bool(flagGRPCWebEnable, true, "Define if the gRPC-Web server should be enabled. (Note: gRPC must also be enabled)") - cmd.Flags().String(flagGRPCWebAddress, serverconfig.DefaultGRPCWebAddress, "The gRPC-Web server address to listen on") - - cmd.Flags().Uint64(FlagStateSyncSnapshotInterval, 0, "State sync snapshot interval") - cmd.Flags().Uint32(FlagStateSyncSnapshotKeepRecent, 2, "State sync snapshot to keep") - - cmd.Flags().Bool(FlagDisableIAVLFastNode, false, "Disable fast node for IAVL tree") - - cmd.Flags().Int(FlagMempoolMaxTxs, mempool.DefaultMaxTx, "Sets MaxTx value for the app-side mempool") - - // add support for all Tendermint-specific command line options - tcmd.AddNodeFlags(cmd) + addStartNodeFlags(cmd, defaultNodeHome) return cmd } @@ -302,7 +278,16 @@ func startInProcess(ctx *Context, clientCtx client.Context, appCreator types.App return err } - app := appCreator(ctx.Logger, db, traceWriter, ctx.Viper) + var app types.Application + + if isTestnet, ok := ctx.Viper.Get(KeyIsTestnet).(bool); ok && isTestnet { + app, err = testnetify(ctx, home, appCreator, db, traceWriter) + if err != nil { + return err + } + } else { + app = appCreator(ctx.Logger, db, traceWriter, ctx.Viper) + } nodeKey, err := p2p.LoadOrGenNodeKey(cfg.NodeKeyFile()) if err != nil { @@ -621,3 +606,299 @@ func returnCommitInfo(ctx *Context, app types.Application, version int64) error ctx.Logger.Error("your node has app hashed. Compare each module's hashes with a node that did not app hash to determine the problematic module", "commitInfo", commitInfoForHeight.String()) return nil } + +// InPlaceTestnetCreator utilizes the provided chainID and operatorAddress as well as the local private validator key to +// control the network represented in the data folder. This is useful to create testnets nearly identical to your +// mainnet environment. +func InPlaceTestnetCreator(testnetAppCreator types.AppCreator, defaultNodeHome string) *cobra.Command { + cmd := &cobra.Command{ + Use: "in-place-testnet [newChainID] [newOperatorAddress]", + Short: "Create and start a testnet from current local state", + Long: `Create and start a testnet from current local state. +After utilizing this command the network will start. If the network is stopped, +the normal "start" command should be used. Re-using this command on state that +has already been modified by this command could result in unexpected behavior. + +Additionally, the first block may take up to one minute to be committed, depending +on how old the block is. For instance, if a snapshot was taken weeks ago and we want +to turn this into a testnet, it is possible lots of pending state needs to be committed +(expiring locks, etc.). It is recommended that you should wait for this block to be committed +before stopping the daemon. + +If the --trigger-testnet-upgrade flag is set, the upgrade handler specified by the flag will be run +on the first block of the testnet. + +Regardless of whether the flag is set or not, if any new stores are introduced in the daemon being run, +those stores will be registered in order to prevent panics. Therefore, you only need to set the flag if +you want to test the upgrade handler itself. +`, + Example: "in-place-testnet localosmosis osmo12smx2wdlyttvyzvzg54y2vnqwq2qjateuf7thj", + Args: cobra.ExactArgs(2), + PreRunE: func(cmd *cobra.Command, _ []string) error { + serverCtx := GetServerContextFromCmd(cmd) + + if err := serverCtx.Viper.BindPFlags(cmd.Flags()); err != nil { + return err + } + + _, err := GetPruningOptionsFromFlags(serverCtx.Viper) + return err + }, + RunE: func(cmd *cobra.Command, args []string) error { + serverCtx := GetServerContextFromCmd(cmd) + clientCtx, err := client.GetClientQueryContext(cmd) + if err != nil { + return err + } + + newChainID := args[0] + newOperatorAddress := args[1] + + // Confirmation prompt to prevent accidental modification of state. + reader := bufio.NewReader(os.Stdin) + fmt.Println("This operation will modify state in your data folder and cannot be undone. Do you want to continue? (y/n)") + text, _ := reader.ReadString('\n') + response := strings.TrimSpace(strings.ToLower(text)) + if response != "y" && response != "yes" { + fmt.Println("Operation canceled.") + return nil + } + serverCtx.Viper.Set(KeyIsTestnet, true) + serverCtx.Viper.Set(KeyNewChainID, newChainID) + serverCtx.Viper.Set(KeyNewOpAddr, newOperatorAddress) + + return startInProcess(serverCtx, clientCtx, testnetAppCreator) + }, + } + + addStartNodeFlags(cmd, defaultNodeHome) + cmd.Flags().String(KeyTriggerTestnetUpgrade, "", "If set (example: \"v21\"), triggers the v21 upgrade handler to run on the first block of the testnet") + return cmd +} + +// testnetify modifies both state and blockStore, allowing the provided operator address and local validator key to control the network +// that the state in the data folder represents. The chainID of the local genesis file is modified to match the provided chainID. +func testnetify(ctx *Context, home string, testnetAppCreator types.AppCreator, db db.DB, traceWriter io.WriteCloser) (types.Application, error) { + config := ctx.Config + + newChainID, ok := ctx.Viper.Get(KeyNewChainID).(string) + if !ok { + return nil, fmt.Errorf("expected string for key %s", KeyNewChainID) + } + + genDocProvider := node.DefaultGenesisDocProviderFunc(config) + + // Initialize blockStore and stateDB. + blockStoreDB, err := node.DefaultDBProvider(&node.DBContext{ID: "blockstore", Config: config}) + if err != nil { + return nil, err + } + blockStore := store.NewBlockStore(blockStoreDB) + + stateDB, err := node.DefaultDBProvider(&node.DBContext{ID: "state", Config: config}) + if err != nil { + return nil, err + } + + defer blockStore.Close() + defer stateDB.Close() + + privValidator := pvm.LoadOrGenFilePV(config.PrivValidatorKeyFile(), config.PrivValidatorStateFile()) + userPubKey, err := privValidator.GetPubKey() + if err != nil { + return nil, err + } + validatorAddress := userPubKey.Address() + if err != nil { + return nil, err + } + + stateStore := sm.NewStore(stateDB, sm.StoreOptions{ + DiscardABCIResponses: config.Storage.DiscardABCIResponses, + }) + + state, genDoc, err := node.LoadStateFromDBOrGenesisDocProvider(stateDB, genDocProvider) + if err != nil { + return nil, err + } + + genDoc.ChainID = newChainID + genFilePath := config.GenesisFile() + if err = genutil.ExportGenesisFile(genDoc, genFilePath); err != nil { + return nil, err + } + + // There are times when a user stops their node between commits, resulting in a mismatch between the + // blockStore and state. For convenience, we just discard the uncommitted blockStore block and operate on + // the lastBlockHeight in state. + if blockStore.Height() != state.LastBlockHeight { + err = blockStore.DeleteLatestBlock() + if err != nil { + return nil, err + } + } + + block := blockStore.LoadBlock(blockStore.Height()) + + block.ChainID = newChainID + state.ChainID = newChainID + + block.LastBlockID = state.LastBlockID + block.LastCommit.BlockID = state.LastBlockID + + // Create a vote from our validator + vote := cmttypes.Vote{ + Type: cmtproto.PrecommitType, + Height: state.LastBlockHeight, + Round: 0, + BlockID: state.LastBlockID, + Timestamp: time.Now(), + ValidatorAddress: validatorAddress, + ValidatorIndex: 0, + Signature: []byte{}, + } + + // Sign the vote, and copy the proto changes from the act of signing to the vote itself + voteProto := vote.ToProto() + err = privValidator.SignVote(newChainID, voteProto) + if err != nil { + return nil, err + } + vote.Signature = voteProto.Signature + vote.Timestamp = voteProto.Timestamp + + // Modify the block's lastCommit to be signed only by our validator + block.LastCommit.Signatures[0].ValidatorAddress = validatorAddress + block.LastCommit.Signatures[0].Signature = vote.Signature + block.LastCommit.Signatures = []cmttypes.CommitSig{block.LastCommit.Signatures[0]} + + // Load the seenCommit of the lastBlockHeight and modify it to be signed from our validator + seenCommit := blockStore.LoadSeenCommit(state.LastBlockHeight) + seenCommit.BlockID = state.LastBlockID + seenCommit.Round = vote.Round + seenCommit.Signatures[0].Signature = vote.Signature + seenCommit.Signatures[0].ValidatorAddress = validatorAddress + seenCommit.Signatures[0].Timestamp = vote.Timestamp + seenCommit.Signatures = []cmttypes.CommitSig{seenCommit.Signatures[0]} + err = blockStore.SaveSeenCommit(state.LastBlockHeight, seenCommit) + if err != nil { + return nil, err + } + + // Create ValidatorSet struct containing just our valdiator. + newVal := &cmttypes.Validator{ + Address: validatorAddress, + PubKey: userPubKey, + VotingPower: 900000000000000, + } + newValSet := &cmttypes.ValidatorSet{ + Validators: []*cmttypes.Validator{newVal}, + Proposer: newVal, + } + + // Replace all valSets in state to be the valSet with just our validator. + state.Validators = newValSet + state.LastValidators = newValSet + state.NextValidators = newValSet + state.LastHeightValidatorsChanged = blockStore.Height() + + err = stateStore.Save(state) + if err != nil { + return nil, err + } + + // Create a ValidatorsInfo struct to store in stateDB. + valSet, err := state.Validators.ToProto() + if err != nil { + return nil, err + } + valInfo := &cmtstate.ValidatorsInfo{ + ValidatorSet: valSet, + LastHeightChanged: state.LastBlockHeight, + } + buf, err := valInfo.Marshal() + if err != nil { + return nil, err + } + + // Modfiy Validators stateDB entry. + err = stateDB.Set([]byte(fmt.Sprintf("validatorsKey:%v", blockStore.Height())), buf) + if err != nil { + return nil, err + } + + // Modify LastValidators stateDB entry. + err = stateDB.Set([]byte(fmt.Sprintf("validatorsKey:%v", blockStore.Height()-1)), buf) + if err != nil { + return nil, err + } + + // Modify NextValidators stateDB entry. + err = stateDB.Set([]byte(fmt.Sprintf("validatorsKey:%v", blockStore.Height()+1)), buf) + if err != nil { + return nil, err + } + + b, err := cmtjson.Marshal(genDoc) + if err != nil { + return nil, err + } + if err := stateDB.SetSync([]byte("genesisDoc"), b); err != nil { + return nil, err + } + + // testnetAppCreator makes any application side changes that must be made due to the above modifications. + // Also, it makes any optional application side changes to make running the testnet easier (voting times, fund accounts, etc). + ctx.Viper.Set(KeyNewValAddr, validatorAddress) + ctx.Viper.Set(KeyUserPubKey, userPubKey) + testnetApp := testnetAppCreator(ctx.Logger, db, traceWriter, ctx.Viper) + + return testnetApp, err +} + +// addStartNodeFlags should be added to any CLI commands that start the network. +func addStartNodeFlags(cmd *cobra.Command, defaultNodeHome string) { + cmd.Flags().String(flags.FlagHome, defaultNodeHome, "The application home directory") + cmd.Flags().Bool(flagWithTendermint, true, "Run abci app embedded in-process with tendermint") + cmd.Flags().String(flagAddress, "tcp://0.0.0.0:26658", "Listen address") + cmd.Flags().String(flagTransport, "socket", "Transport protocol: socket, grpc") + cmd.Flags().String(flagTraceStore, "", "Enable KVStore tracing to an output file") + cmd.Flags().String(FlagMinGasPrices, "", "Minimum gas prices to accept for transactions; Any fee in a tx must meet this minimum (e.g. 0.01photino;0.0001stake)") + cmd.Flags().IntSlice(FlagUnsafeSkipUpgrades, []int{}, "Skip a set of upgrade heights to continue the old binary") + cmd.Flags().Uint64(FlagHaltHeight, 0, "Block height at which to gracefully halt the chain and shutdown the node") + cmd.Flags().Uint64(FlagHaltTime, 0, "Minimum block time (in Unix seconds) at which to gracefully halt the chain and shutdown the node") + cmd.Flags().Bool(FlagInterBlockCache, true, "Enable inter-block caching") + cmd.Flags().String(flagCPUProfile, "", "Enable CPU profiling and write to the provided file") + cmd.Flags().Bool(FlagTrace, false, "Provide full stack traces for errors in ABCI Log") + cmd.Flags().String(FlagPruning, pruningtypes.PruningOptionDefault, "Pruning strategy (default|nothing|everything|custom)") + cmd.Flags().Uint64(FlagPruningKeepRecent, 0, "Number of recent heights to keep on disk (ignored if pruning is not 'custom')") + cmd.Flags().Uint64(FlagPruningInterval, 0, "Height interval at which pruned heights are removed from disk (ignored if pruning is not 'custom')") + cmd.Flags().Uint(FlagInvCheckPeriod, 0, "Assert registered invariants every N blocks") + cmd.Flags().Uint64(FlagMinRetainBlocks, 0, "Minimum block height offset during ABCI commit to prune Tendermint blocks") + + cmd.Flags().Bool(FlagAPIEnable, false, "Define if the API server should be enabled") + cmd.Flags().Bool(FlagAPISwagger, false, "Define if swagger documentation should automatically be registered (Note: the API must also be enabled)") + cmd.Flags().String(FlagAPIAddress, serverconfig.DefaultAPIAddress, "the API server address to listen on") + cmd.Flags().Uint(FlagAPIMaxOpenConnections, 1000, "Define the number of maximum open connections") + cmd.Flags().Uint(FlagRPCReadTimeout, 10, "Define the Tendermint RPC read timeout (in seconds)") + cmd.Flags().Uint(FlagRPCWriteTimeout, 0, "Define the Tendermint RPC write timeout (in seconds)") + cmd.Flags().Uint(FlagRPCMaxBodyBytes, 1000000, "Define the Tendermint maximum request body (in bytes)") + cmd.Flags().Bool(FlagAPIEnableUnsafeCORS, false, "Define if CORS should be enabled (unsafe - use it at your own risk)") + + cmd.Flags().Bool(flagGRPCOnly, false, "Start the node in gRPC query only mode (no Tendermint process is started)") + cmd.Flags().Bool(flagGRPCEnable, true, "Define if the gRPC server should be enabled") + cmd.Flags().String(flagGRPCAddress, serverconfig.DefaultGRPCAddress, "the gRPC server address to listen on") + + cmd.Flags().Bool(flagGRPCWebEnable, true, "Define if the gRPC-Web server should be enabled. (Note: gRPC must also be enabled)") + cmd.Flags().String(flagGRPCWebAddress, serverconfig.DefaultGRPCWebAddress, "The gRPC-Web server address to listen on") + + cmd.Flags().Uint64(FlagStateSyncSnapshotInterval, 0, "State sync snapshot interval") + cmd.Flags().Uint32(FlagStateSyncSnapshotKeepRecent, 2, "State sync snapshot to keep") + + cmd.Flags().Bool(FlagDisableIAVLFastNode, false, "Disable fast node for IAVL tree") + + cmd.Flags().Int(FlagMempoolMaxTxs, mempool.DefaultMaxTx, "Sets MaxTx value for the app-side mempool") + + // add support for all Tendermint-specific command line options + tcmd.AddNodeFlags(cmd) +} diff --git a/server/util.go b/server/util.go index 9a6ae40f786e..8b41d194aa42 100644 --- a/server/util.go +++ b/server/util.go @@ -324,6 +324,13 @@ func AddCommands(rootCmd *cobra.Command, defaultNodeHome string, appCreator type ) } +// AddTestnetCreatorCommand allows chains to create a testnet from the state existing in their node's data directory. +func AddTestnetCreatorCommand(rootCmd *cobra.Command, defaultNodeHome string, appCreator types.AppCreator, addStartFlags types.ModuleInitFlags) { + testnetCreateCmd := InPlaceTestnetCreator(appCreator, defaultNodeHome) + addStartFlags(testnetCreateCmd) + rootCmd.AddCommand(testnetCreateCmd) +} + // https://stackoverflow.com/questions/23558425/how-do-i-get-the-local-ip-address-in-go // TODO there must be a better way to get external IP func ExternalIP() (string, error) {