From eecb1ca35d851c96b9a31d39d9bbba26a70e150b Mon Sep 17 00:00:00 2001 From: klim0v Date: Mon, 27 Jan 2020 16:36:37 +0300 Subject: [PATCH 01/10] export state 1.0 for genesis 1.1 --- Gopkg.lock | 1 + cmd/export/main.go | 221 +++++++++++++++++++++++++++++++++--- cmd/export/types11/types.go | 93 +++++++++++++++ core/state/statedb.go | 127 +++++++++++++++++++++ 4 files changed, 428 insertions(+), 14 deletions(-) create mode 100644 cmd/export/types11/types.go diff --git a/Gopkg.lock b/Gopkg.lock index 27bf2d73c..b8b959dd4 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -639,6 +639,7 @@ "github.com/tendermint/tendermint/rpc/lib/server", "github.com/tendermint/tendermint/rpc/lib/types", "github.com/tendermint/tendermint/types", + "github.com/tendermint/tendermint/types/time", "golang.org/x/crypto/sha3", "golang.org/x/net/netutil", "golang.org/x/sys/cpu", diff --git a/cmd/export/main.go b/cmd/export/main.go index ea83d9f9e..dc12f58f2 100644 --- a/cmd/export/main.go +++ b/cmd/export/main.go @@ -1,19 +1,44 @@ package main import ( + "bytes" "encoding/json" + "flag" + "fmt" "github.com/MinterTeam/go-amino" "github.com/MinterTeam/minter-go-node/cmd/utils" - "github.com/MinterTeam/minter-go-node/config" - "github.com/MinterTeam/minter-go-node/core/appdb" "github.com/MinterTeam/minter-go-node/core/state" + "github.com/pkg/errors" + "github.com/tendermint/tendermint/crypto" + "github.com/tendermint/tendermint/crypto/ed25519" + "github.com/tendermint/tendermint/crypto/secp256k1" "github.com/tendermint/tendermint/libs/common" "github.com/tendermint/tendermint/libs/db" "github.com/tendermint/tendermint/types" + tmtime "github.com/tendermint/tendermint/types/time" + "golang.org/x/crypto/sha3" + "io/ioutil" + "log" "time" ) +var ( + height = flag.Uint64("height", 0, "height") + chainID = flag.String("chain_id", "", "chain_id") + genesisTime = flag.Duration("genesis_time", 0, "genesis_time") +) + func main() { + required := []string{"height", "chain_id", "genesis_time"} + flag.Parse() + seen := make(map[string]bool) + flag.Visit(func(f *flag.Flag) { seen[f.Name] = true }) + for _, req := range required { + if !seen[req] { + log.Fatalf("missing required --%s argument/flag\n", req) + } + } + err := common.EnsureDir(utils.GetMinterHome()+"/config", 0777) if err != nil { panic(err) @@ -24,16 +49,14 @@ func main() { panic(err) } - applicationDB := appdb.NewAppDB(config.GetConfig()) - height := applicationDB.GetLastHeight() - currentState, err := state.New(height, ldb, false) + currentState, err := state.New(*height, ldb, false) if err != nil { panic(err) } cdc := amino.NewCodec() - jsonBytes, err := cdc.MarshalJSONIndent(currentState.Export(height), "", " ") + jsonBytes, err := cdc.MarshalJSONIndent(currentState.Export11(*height), "", " ") if err != nil { panic(err) } @@ -41,19 +64,19 @@ func main() { appHash := [32]byte{} // Compose Genesis - genesis := types.GenesisDoc{ - GenesisTime: time.Date(2019, time.April, 2, 17, 0, 0, 0, time.UTC), - ChainID: "minter-test-network-35", - ConsensusParams: &types.ConsensusParams{ - Block: types.BlockParams{ + genesis := GenesisDoc{ + GenesisTime: time.Unix(0, 0).Add(*genesisTime), + ChainID: *chainID, + ConsensusParams: &ConsensusParams{ + Block: BlockParams{ MaxBytes: 10000000, MaxGas: 100000, TimeIotaMs: 1000, }, - Evidence: types.EvidenceParams{ - MaxAge: 1000, + Evidence: EvidenceParams{ + MaxAgeNumBlocks: 1000, }, - Validator: types.ValidatorParams{ + Validator: ValidatorParams{ PubKeyTypes: []string{types.ABCIPubKeyTypeEd25519}, }, }, @@ -69,4 +92,174 @@ func main() { if err := genesis.SaveAs("genesis.json"); err != nil { panic(err) } + + fmt.Println("OK") + fmt.Println(sha3.Sum512([]byte(fmt.Sprintf("%v", genesis)))) +} + +type GenesisDoc struct { + GenesisTime time.Time `json:"genesis_time"` + ChainID string `json:"chain_id"` + ConsensusParams *ConsensusParams `json:"consensus_params,omitempty"` + Validators []GenesisValidator `json:"validators,omitempty"` + AppHash common.HexBytes `json:"app_hash"` + AppState json.RawMessage `json:"app_state,omitempty"` +} +type ConsensusParams struct { + Block BlockParams `json:"block"` + Evidence EvidenceParams `json:"evidence"` + Validator ValidatorParams `json:"validator"` +} +type BlockParams struct { + MaxBytes int64 `json:"max_bytes"` + MaxGas int64 `json:"max_gas"` + // Minimum time increment between consecutive blocks (in milliseconds) + // Not exposed to the application. + TimeIotaMs int64 `json:"time_iota_ms"` +} +type GenesisValidator struct { + Address Address `json:"address"` + PubKey crypto.PubKey `json:"pub_key"` + Power int64 `json:"power"` + Name string `json:"name"` +} +type Address = crypto.Address +type EvidenceParams struct { + MaxAgeNumBlocks int64 `json:"max_age_num_blocks"` // only accept new evidence more recent than this + MaxAgeDuration time.Duration `json:"max_age_duration"` +} +type ValidatorParams struct { + PubKeyTypes []string `json:"pub_key_types"` +} + +const ( + MaxChainIDLen = 50 +) + +func (genDoc *GenesisDoc) ValidateAndComplete() error { + if genDoc.ChainID == "" { + return errors.New("genesis doc must include non-empty chain_id") + } + if len(genDoc.ChainID) > MaxChainIDLen { + return errors.Errorf("chain_id in genesis doc is too long (max: %d)", MaxChainIDLen) + } + + if genDoc.ConsensusParams == nil { + genDoc.ConsensusParams = DefaultConsensusParams() + } else if err := genDoc.ConsensusParams.Validate(); err != nil { + return err + } + + for i, v := range genDoc.Validators { + if v.Power == 0 { + return errors.Errorf("the genesis file cannot contain validators with no voting power: %v", v) + } + if len(v.Address) > 0 && !bytes.Equal(v.PubKey.Address(), v.Address) { + return errors.Errorf("incorrect address for validator %v in the genesis file, should be %v", v, v.PubKey.Address()) + } + if len(v.Address) == 0 { + genDoc.Validators[i].Address = v.PubKey.Address() + } + } + + if genDoc.GenesisTime.IsZero() { + genDoc.GenesisTime = tmtime.Now() + } + + return nil +} +func DefaultConsensusParams() *ConsensusParams { + return &ConsensusParams{ + DefaultBlockParams(), + DefaultEvidenceParams(), + DefaultValidatorParams(), + } +} +func DefaultBlockParams() BlockParams { + return BlockParams{ + MaxBytes: 22020096, // 21MB + MaxGas: -1, + TimeIotaMs: 1000, // 1s + } +} + +// DefaultEvidenceParams Params returns a default EvidenceParams. +func DefaultEvidenceParams() EvidenceParams { + return EvidenceParams{ + MaxAgeNumBlocks: 100000, // 27.8 hrs at 1block/s + MaxAgeDuration: 48 * time.Hour, + } +} + +// DefaultValidatorParams returns a default ValidatorParams, which allows +// only ed25519 pubkeys. +func DefaultValidatorParams() ValidatorParams { + return ValidatorParams{[]string{types.ABCIPubKeyTypeEd25519}} +} + +const ( + ABCIPubKeyTypeEd25519 = "ed25519" + ABCIPubKeyTypeSr25519 = "sr25519" + ABCIPubKeyTypeSecp256k1 = "secp256k1" + + MaxBlockSizeBytes = 104857600 +) + +var ABCIPubKeyTypesToAminoNames = map[string]string{ + ABCIPubKeyTypeEd25519: ed25519.PubKeyAminoName, + ABCIPubKeyTypeSr25519: "tendermint/PubKeySr25519", + ABCIPubKeyTypeSecp256k1: secp256k1.PubKeyAminoName, +} + +func (params *ConsensusParams) Validate() error { + if params.Block.MaxBytes <= 0 { + return errors.Errorf("block.MaxBytes must be greater than 0. Got %d", + params.Block.MaxBytes) + } + if params.Block.MaxBytes > MaxBlockSizeBytes { + return errors.Errorf("block.MaxBytes is too big. %d > %d", + params.Block.MaxBytes, MaxBlockSizeBytes) + } + + if params.Block.MaxGas < -1 { + return errors.Errorf("block.MaxGas must be greater or equal to -1. Got %d", + params.Block.MaxGas) + } + + if params.Block.TimeIotaMs <= 0 { + return errors.Errorf("block.TimeIotaMs must be greater than 0. Got %v", + params.Block.TimeIotaMs) + } + + if params.Evidence.MaxAgeNumBlocks <= 0 { + return errors.Errorf("evidenceParams.MaxAgeNumBlocks must be greater than 0. Got %d", + params.Evidence.MaxAgeNumBlocks) + } + + if params.Evidence.MaxAgeDuration <= 0 { + return errors.Errorf("evidenceParams.MaxAgeDuration must be grater than 0 if provided, Got %v", + params.Evidence.MaxAgeDuration) + } + + if len(params.Validator.PubKeyTypes) == 0 { + return errors.New("len(Validator.PubKeyTypes) must be greater than 0") + } + + // Check if keyType is a known ABCIPubKeyType + for i := 0; i < len(params.Validator.PubKeyTypes); i++ { + keyType := params.Validator.PubKeyTypes[i] + if _, ok := ABCIPubKeyTypesToAminoNames[keyType]; !ok { + return errors.Errorf("params.Validator.PubKeyTypes[%d], %s, is an unknown pubkey type", + i, keyType) + } + } + + return nil +} +func (genDoc *GenesisDoc) SaveAs(file string) error { + genDocBytes, err := types.GetCodec().MarshalJSONIndent(genDoc, "", " ") + if err != nil { + return err + } + return ioutil.WriteFile(file, genDocBytes, 0644) } diff --git a/cmd/export/types11/types.go b/cmd/export/types11/types.go new file mode 100644 index 000000000..ce500efc6 --- /dev/null +++ b/cmd/export/types11/types.go @@ -0,0 +1,93 @@ +package types11 + +import "sync" + +type AppState struct { + Note string `json:"note"` + StartHeight uint64 `json:"start_height"` + Validators []Validator `json:"validators,omitempty"` + Candidates []Candidate `json:"candidates,omitempty"` + Accounts []Account `json:"accounts,omitempty"` + Coins []Coin `json:"coins,omitempty"` + FrozenFunds []FrozenFund `json:"frozen_funds,omitempty"` + UsedChecks []UsedCheck `json:"used_checks,omitempty"` + MaxGas uint64 `json:"max_gas"` + TotalSlashed string `json:"total_slashed"` +} + +type Validator struct { + TotalBipStake string `json:"total_bip_stake"` + PubKey Pubkey `json:"pub_key"` + AccumReward string `json:"accum_reward"` + AbsentTimes *BitArray `json:"absent_times"` +} + +type Candidate struct { + RewardAddress Address `json:"reward_address"` + OwnerAddress Address `json:"owner_address"` + TotalBipStake string `json:"total_bip_stake"` + PubKey Pubkey `json:"pub_key"` + Commission uint `json:"commission"` + Stakes []Stake `json:"stakes"` + Status byte `json:"status"` +} + +type Stake struct { + Owner Address `json:"owner"` + Coin CoinSymbol `json:"coin"` + Value string `json:"value"` + BipValue string `json:"bip_value"` +} + +type Coin struct { + Name string `json:"name"` + Symbol CoinSymbol `json:"symbol"` + Volume string `json:"volume"` + Crr uint `json:"crr"` + Reserve string `json:"reserve"` + MaxSupply string `json:"max_supply"` +} + +type FrozenFund struct { + Height uint64 `json:"height"` + Address Address `json:"address"` + CandidateKey *Pubkey `json:"candidate_key,omitempty"` + Coin CoinSymbol `json:"coin"` + Value string `json:"value"` +} + +type UsedCheck string + +type Account struct { + Address Address `json:"address"` + Balance []Balance `json:"balance"` + Nonce uint64 `json:"nonce"` + MultisigData *Multisig `json:"multisig_data,omitempty"` +} + +type Balance struct { + Coin CoinSymbol `json:"coin"` + Value string `json:"value"` +} + +type Multisig struct { + Weights []uint `json:"weights"` + Threshold uint `json:"threshold"` + Addresses []Address `json:"addresses"` +} + +type Pubkey [32]byte + +const CoinSymbolLength = 10 + +type CoinSymbol [CoinSymbolLength]byte + +type BitArray struct { + mtx sync.Mutex + Bits uint `json:"bits"` // NOTE: persisted via reflect, must be exported + Elems []uint64 `json:"elems"` // NOTE: persisted via reflect, must be exported +} + +const AddressLength = 20 + +type Address [AddressLength]byte diff --git a/core/state/statedb.go b/core/state/statedb.go index 0a63c791f..2e6902605 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -4,6 +4,7 @@ import ( "encoding/hex" "fmt" "github.com/MinterTeam/go-amino" + "github.com/MinterTeam/minter-go-node/cmd/export/types11" "github.com/MinterTeam/minter-go-node/cmd/utils" "github.com/MinterTeam/minter-go-node/core/rewards" "github.com/MinterTeam/minter-go-node/core/types" @@ -1908,6 +1909,132 @@ func (s *StateDB) Export(currentHeight uint64) types.AppState { return appState } +func (s *StateDB) Export11(currentHeight uint64) types11.AppState { + consertPubKey10to11 := func(pubkey types.Pubkey) (res types11.Pubkey) { + copy(res[:], pubkey) + return + } + + appState := types11.AppState{} + + s.iavl.Iterate(func(key []byte, value []byte) bool { + // export accounts + if key[0] == addressPrefix[0] { + account := s.GetOrNewStateObject(types.BytesToAddress(key[1:])) + + balance := make([]types11.Balance, len(account.Balances().Data)) + i := 0 + for coin, value := range account.Balances().Data { + balance[i] = types11.Balance{ + Coin: [10]byte(coin), + Value: value.String(), + } + i++ + } + + acc := types11.Account{ + Address: [20]byte(account.address), + Balance: balance, + Nonce: account.data.Nonce, + } + + if account.IsMultisig() { + addresses := make([]types11.Address, 0, len(account.data.MultisigData.Addresses)) + for _, address := range account.data.MultisigData.Addresses { + addresses = append(addresses, [20]byte(address)) + } + acc.MultisigData = &types11.Multisig{ + Weights: account.data.MultisigData.Weights, + Threshold: account.data.MultisigData.Threshold, + Addresses: addresses, + } + } + + appState.Accounts = append(appState.Accounts, acc) + } + + // export coins + if key[0] == coinPrefix[0] { + coin := s.GetStateCoin(types.StrToCoinSymbol(string(key[1:]))) + + appState.Coins = append(appState.Coins, types11.Coin{ + Name: coin.Name(), + Symbol: [10]byte(coin.Symbol()), + Volume: coin.Volume().String(), + Crr: coin.Crr(), + Reserve: coin.ReserveBalance().String(), + MaxSupply: big.NewInt(0).Exp(big.NewInt(10), big.NewInt(15+18), nil).String(), + }) + } + + // export used checks + if key[0] == usedCheckPrefix[0] { + appState.UsedChecks = append(appState.UsedChecks, types11.UsedCheck(fmt.Sprintf("%x", key[1:]))) + } + + // export frozen funds + if key[0] == frozenFundsPrefix[0] { + height := binary.BigEndian.Uint64(key[1:]) + frozenFunds := s.GetStateFrozenFunds(height) + + for _, frozenFund := range frozenFunds.List() { + pubKey10to11 := consertPubKey10to11(frozenFund.CandidateKey) + appState.FrozenFunds = append(appState.FrozenFunds, types11.FrozenFund{ + Height: height - uint64(currentHeight), + Address: [20]byte(frozenFund.Address), + CandidateKey: &pubKey10to11, + Coin: [10]byte(frozenFund.Coin), + Value: frozenFund.Value.String(), + }) + } + } + + return false + }) + + candidates := s.getStateCandidates() + for _, candidate := range candidates.data { + var stakes []types11.Stake + for _, s := range candidate.Stakes { + stakes = append(stakes, types11.Stake{ + Owner: [20]byte(s.Owner), + Coin: [10]byte(s.Coin), + Value: s.Value.String(), + BipValue: s.BipValue.String(), + }) + } + + appState.Candidates = append(appState.Candidates, types11.Candidate{ + RewardAddress: [20]byte(candidate.RewardAddress), + OwnerAddress: [20]byte(candidate.OwnerAddress), + TotalBipStake: candidate.TotalBipStake.String(), + PubKey: consertPubKey10to11(candidate.PubKey), + Commission: candidate.Commission, + Stakes: stakes, + Status: candidate.Status, + }) + } + + vals := s.getStateValidators() + for _, val := range vals.data { + appState.Validators = append(appState.Validators, types11.Validator{ + TotalBipStake: val.TotalBipStake.String(), + PubKey: consertPubKey10to11(val.PubKey), + AccumReward: val.AccumReward.String(), + AbsentTimes: &types11.BitArray{ + Bits: val.AbsentTimes.Bits, + Elems: val.AbsentTimes.Elems, + }, + }) + } + + appState.MaxGas = s.GetMaxGas() + appState.StartHeight = s.height + appState.TotalSlashed = s.GetTotalSlashed().String() + + return appState +} + func (s *StateDB) Import(appState types.AppState) { s.SetMaxGas(appState.MaxGas) s.setTotalSlashed(appState.TotalSlashed) From 940e838c2adbfaebc478be7aabf8c49bab704171 Mon Sep 17 00:00:00 2001 From: klim0v Date: Tue, 28 Jan 2020 13:45:03 +0300 Subject: [PATCH 02/10] fixes and recover --- cmd/export/main.go | 8 +++++++- cmd/export/readme.md | 8 ++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) create mode 100644 cmd/export/readme.md diff --git a/cmd/export/main.go b/cmd/export/main.go index dc12f58f2..50f6d5c83 100644 --- a/cmd/export/main.go +++ b/cmd/export/main.go @@ -29,6 +29,11 @@ var ( ) func main() { + defer func() { + if r := recover(); r != nil { + fmt.Println("Recovered in f", r) + } + }() required := []string{"height", "chain_id", "genesis_time"} flag.Parse() seen := make(map[string]bool) @@ -75,6 +80,7 @@ func main() { }, Evidence: EvidenceParams{ MaxAgeNumBlocks: 1000, + MaxAgeDuration: 24 * time.Hour, //todo }, Validator: ValidatorParams{ PubKeyTypes: []string{types.ABCIPubKeyTypeEd25519}, @@ -94,7 +100,7 @@ func main() { } fmt.Println("OK") - fmt.Println(sha3.Sum512([]byte(fmt.Sprintf("%v", genesis)))) + fmt.Println(fmt.Sprintf("%x", sha3.Sum512([]byte(fmt.Sprintf("%v", genesis))))) } type GenesisDoc struct { diff --git a/cmd/export/readme.md b/cmd/export/readme.md new file mode 100644 index 000000000..6bb4adafc --- /dev/null +++ b/cmd/export/readme.md @@ -0,0 +1,8 @@ +# Export Command + +Command to convert the genesis file from version 1.0 to 1.1 +More details will be received later. +#####Example +```bash +./export --height=4350342 --chain_id=1 --genesis_time=1580206824s +``` \ No newline at end of file From fb10e8e4dab964e45cbb4c598acd3cdd9d0e143e Mon Sep 17 00:00:00 2001 From: klim0v Date: Tue, 28 Jan 2020 14:38:30 +0300 Subject: [PATCH 03/10] fixes --- cmd/export/types11/types.go | 76 ++++++++++++++++++++++++++++++++++++- core/state/statedb.go | 22 +++++------ 2 files changed, 86 insertions(+), 12 deletions(-) diff --git a/cmd/export/types11/types.go b/cmd/export/types11/types.go index ce500efc6..dbda8fff1 100644 --- a/cmd/export/types11/types.go +++ b/cmd/export/types11/types.go @@ -1,6 +1,11 @@ package types11 -import "sync" +import ( + "bytes" + "encoding/hex" + "github.com/MinterTeam/minter-go-node/hexutil" + "sync" +) type AppState struct { Note string `json:"note"` @@ -82,6 +87,29 @@ const CoinSymbolLength = 10 type CoinSymbol [CoinSymbolLength]byte +func (c CoinSymbol) String() string { return string(bytes.Trim(c[:], "\x00")) } +func (c CoinSymbol) Bytes() []byte { return c[:] } + +func (c CoinSymbol) MarshalJSON() ([]byte, error) { + + buffer := bytes.NewBufferString("\"") + buffer.WriteString(c.String()) + buffer.WriteString("\"") + + return buffer.Bytes(), nil +} + +func (c *CoinSymbol) UnmarshalJSON(input []byte) error { + *c = StrToCoinSymbol(string(input[1 : len(input)-1])) + return nil +} + +func StrToCoinSymbol(s string) CoinSymbol { + var symbol CoinSymbol + copy(symbol[:], []byte(s)) + return symbol +} + type BitArray struct { mtx sync.Mutex Bits uint `json:"bits"` // NOTE: persisted via reflect, must be exported @@ -91,3 +119,49 @@ type BitArray struct { const AddressLength = 20 type Address [AddressLength]byte + +func BytesToAddress(b []byte) Address { + var a Address + a.SetBytes(b) + return a +} +func (a *Address) SetBytes(b []byte) { + if len(b) > len(a) { + b = b[len(b)-AddressLength:] + } + copy(a[AddressLength-len(b):], b) +} +func (a Address) String() string { + return a.Hex() +} +func (a Address) Hex() string { + return "Mx" + hex.EncodeToString(a[:]) +} + +// MarshalText returns the hex representation of a. +func (a Address) MarshalText() ([]byte, error) { + return hexutil.Bytes(a[:]).MarshalText() +} + +// UnmarshalText parses a hash in hex syntax. +func (a *Address) UnmarshalText(input []byte) error { + return hexutil.UnmarshalFixedText("Address", input, a[:]) +} + +func (a *Address) Unmarshal(input []byte) error { + copy(a[:], input) + return nil +} + +// +//func (a Address) MarshalJSON() ([]byte, error) { +// return []byte(fmt.Sprintf("\"%s\"", a.String())), nil +//} +func (a Address) MarshalJSON() ([]byte, error) { + + buffer := bytes.NewBufferString("\"") + buffer.WriteString(a.String()) + buffer.WriteString("\"") + + return buffer.Bytes(), nil +} diff --git a/core/state/statedb.go b/core/state/statedb.go index 2e6902605..d64b6851c 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -1925,15 +1925,15 @@ func (s *StateDB) Export11(currentHeight uint64) types11.AppState { balance := make([]types11.Balance, len(account.Balances().Data)) i := 0 for coin, value := range account.Balances().Data { + balance[i] = types11.Balance{ - Coin: [10]byte(coin), + Coin: types11.StrToCoinSymbol(coin.String()), Value: value.String(), } i++ } - acc := types11.Account{ - Address: [20]byte(account.address), + Address: types11.BytesToAddress(account.address.Bytes()), Balance: balance, Nonce: account.data.Nonce, } @@ -1941,7 +1941,7 @@ func (s *StateDB) Export11(currentHeight uint64) types11.AppState { if account.IsMultisig() { addresses := make([]types11.Address, 0, len(account.data.MultisigData.Addresses)) for _, address := range account.data.MultisigData.Addresses { - addresses = append(addresses, [20]byte(address)) + addresses = append(addresses, types11.BytesToAddress(address.Bytes())) } acc.MultisigData = &types11.Multisig{ Weights: account.data.MultisigData.Weights, @@ -1959,7 +1959,7 @@ func (s *StateDB) Export11(currentHeight uint64) types11.AppState { appState.Coins = append(appState.Coins, types11.Coin{ Name: coin.Name(), - Symbol: [10]byte(coin.Symbol()), + Symbol: types11.StrToCoinSymbol(coin.Symbol().String()), Volume: coin.Volume().String(), Crr: coin.Crr(), Reserve: coin.ReserveBalance().String(), @@ -1981,9 +1981,9 @@ func (s *StateDB) Export11(currentHeight uint64) types11.AppState { pubKey10to11 := consertPubKey10to11(frozenFund.CandidateKey) appState.FrozenFunds = append(appState.FrozenFunds, types11.FrozenFund{ Height: height - uint64(currentHeight), - Address: [20]byte(frozenFund.Address), + Address: types11.BytesToAddress(frozenFund.Address.Bytes()), CandidateKey: &pubKey10to11, - Coin: [10]byte(frozenFund.Coin), + Coin: types11.StrToCoinSymbol(frozenFund.Coin.String()), Value: frozenFund.Value.String(), }) } @@ -1997,16 +1997,16 @@ func (s *StateDB) Export11(currentHeight uint64) types11.AppState { var stakes []types11.Stake for _, s := range candidate.Stakes { stakes = append(stakes, types11.Stake{ - Owner: [20]byte(s.Owner), - Coin: [10]byte(s.Coin), + Owner: types11.BytesToAddress(s.Owner.Bytes()), + Coin: types11.StrToCoinSymbol(s.Coin.String()), Value: s.Value.String(), BipValue: s.BipValue.String(), }) } appState.Candidates = append(appState.Candidates, types11.Candidate{ - RewardAddress: [20]byte(candidate.RewardAddress), - OwnerAddress: [20]byte(candidate.OwnerAddress), + RewardAddress: types11.BytesToAddress(candidate.RewardAddress.Bytes()), + OwnerAddress: types11.BytesToAddress(candidate.OwnerAddress.Bytes()), TotalBipStake: candidate.TotalBipStake.String(), PubKey: consertPubKey10to11(candidate.PubKey), Commission: candidate.Commission, From 171a7d0e4bbc5aa8fecb28b17af716ab3a93ea02 Mon Sep 17 00:00:00 2001 From: klim0v Date: Tue, 28 Jan 2020 14:55:04 +0300 Subject: [PATCH 04/10] fixes --- cmd/export/types11/types.go | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/cmd/export/types11/types.go b/cmd/export/types11/types.go index dbda8fff1..eaddbe0a1 100644 --- a/cmd/export/types11/types.go +++ b/cmd/export/types11/types.go @@ -3,6 +3,7 @@ package types11 import ( "bytes" "encoding/hex" + "fmt" "github.com/MinterTeam/minter-go-node/hexutil" "sync" ) @@ -153,15 +154,6 @@ func (a *Address) Unmarshal(input []byte) error { return nil } -// -//func (a Address) MarshalJSON() ([]byte, error) { -// return []byte(fmt.Sprintf("\"%s\"", a.String())), nil -//} func (a Address) MarshalJSON() ([]byte, error) { - - buffer := bytes.NewBufferString("\"") - buffer.WriteString(a.String()) - buffer.WriteString("\"") - - return buffer.Bytes(), nil + return []byte(fmt.Sprintf("\"%s\"", a.String())), nil } From 04a6ac051f6eed7863f1ee862491cd480ab6b8af Mon Sep 17 00:00:00 2001 From: klim0v Date: Tue, 28 Jan 2020 15:13:52 +0300 Subject: [PATCH 05/10] fixes --- cmd/export/types11/types.go | 133 +++++++++++++++++++++++++++++++++++- core/state/statedb.go | 10 +-- 2 files changed, 134 insertions(+), 9 deletions(-) diff --git a/cmd/export/types11/types.go b/cmd/export/types11/types.go index eaddbe0a1..625c0bc48 100644 --- a/cmd/export/types11/types.go +++ b/cmd/export/types11/types.go @@ -2,9 +2,11 @@ package types11 import ( "bytes" + "encoding/binary" "encoding/hex" "fmt" "github.com/MinterTeam/minter-go-node/hexutil" + "strings" "sync" ) @@ -82,8 +84,6 @@ type Multisig struct { Addresses []Address `json:"addresses"` } -type Pubkey [32]byte - const CoinSymbolLength = 10 type CoinSymbol [CoinSymbolLength]byte @@ -117,6 +117,97 @@ type BitArray struct { Elems []uint64 `json:"elems"` // NOTE: persisted via reflect, must be exported } +func (bA *BitArray) getIndex(i uint) bool { + if i >= bA.Bits { + return false + } + return bA.Elems[i/64]&(uint64(1)< 0 +} + +// String returns a string representation of BitArray: BA{}, +// where is a sequence of 'x' (1) and '_' (0). +// The includes spaces and newlines to help people. +// For a simple sequence of 'x' and '_' characters with no spaces or newlines, +// see the MarshalJSON() method. +// Example: "BA{_x_}" or "nil-BitArray" for nil. +func (bA *BitArray) String() string { + return bA.StringIndented("") +} + +// StringIndented returns the same thing as String(), but applies the indent +// at every 10th bit, and twice at every 50th bit. +func (bA *BitArray) StringIndented(indent string) string { + if bA == nil { + return "nil-BitArray" + } + bA.mtx.Lock() + defer bA.mtx.Unlock() + return bA.stringIndented(indent) +} + +func (bA *BitArray) stringIndented(indent string) string { + lines := []string{} + bits := "" + for i := uint(0); i < bA.Bits; i++ { + if bA.getIndex(i) { + bits += "x" + } else { + bits += "_" + } + if i%100 == 99 { + lines = append(lines, bits) + bits = "" + } + if i%10 == 9 { + bits += indent + } + if i%50 == 49 { + bits += indent + } + } + if len(bits) > 0 { + lines = append(lines, bits) + } + return fmt.Sprintf("BA{%v:%v}", bA.Bits, strings.Join(lines, indent)) +} + +// Bytes returns the byte representation of the bits within the bitarray. +func (bA *BitArray) Bytes() []byte { + bA.mtx.Lock() + defer bA.mtx.Unlock() + + numBytes := (bA.Bits + 7) / 8 + bytes := make([]byte, numBytes) + for i := 0; i < len(bA.Elems); i++ { + elemBytes := [8]byte{} + binary.LittleEndian.PutUint64(elemBytes[:], bA.Elems[i]) + copy(bytes[i*8:], elemBytes[:]) + } + return bytes +} + +// MarshalJSON implements json.Marshaler interface by marshaling bit array +// using a custom format: a string of '-' or 'x' where 'x' denotes the 1 bit. +func (bA *BitArray) MarshalJSON() ([]byte, error) { + if bA == nil { + return []byte("null"), nil + } + + bA.mtx.Lock() + defer bA.mtx.Unlock() + + bits := `"` + for i := uint(0); i < bA.Bits; i++ { + if bA.getIndex(i) { + bits += `x` + } else { + bits += `_` + } + } + bits += `"` + return []byte(bits), nil +} + const AddressLength = 20 type Address [AddressLength]byte @@ -157,3 +248,41 @@ func (a *Address) Unmarshal(input []byte) error { func (a Address) MarshalJSON() ([]byte, error) { return []byte(fmt.Sprintf("\"%s\"", a.String())), nil } + +const PubKeyLength = 32 + +type Pubkey [PubKeyLength]byte + +func BytesToPubkey(b []byte) Pubkey { + var p Pubkey + p.SetBytes(b) + return p +} + +func (p *Pubkey) SetBytes(b []byte) { + if len(b) > len(p) { + b = b[len(b)-PubKeyLength:] + } + copy(p[PubKeyLength-len(b):], b) +} + +func (p Pubkey) Bytes() []byte { return p[:] } + +func (p Pubkey) String() string { + return fmt.Sprintf("Mp%x", p[:]) +} + +func (p Pubkey) MarshalText() ([]byte, error) { + return []byte(p.String()), nil +} + +func (p Pubkey) MarshalJSON() ([]byte, error) { + return []byte(fmt.Sprintf("\"%s\"", p.String())), nil +} + +func (p *Pubkey) UnmarshalJSON(input []byte) error { + b, err := hex.DecodeString(string(input)[3 : len(input)-1]) + copy(p[:], b) + + return err +} diff --git a/core/state/statedb.go b/core/state/statedb.go index d64b6851c..a8df678c2 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -1910,10 +1910,6 @@ func (s *StateDB) Export(currentHeight uint64) types.AppState { } func (s *StateDB) Export11(currentHeight uint64) types11.AppState { - consertPubKey10to11 := func(pubkey types.Pubkey) (res types11.Pubkey) { - copy(res[:], pubkey) - return - } appState := types11.AppState{} @@ -1978,7 +1974,7 @@ func (s *StateDB) Export11(currentHeight uint64) types11.AppState { frozenFunds := s.GetStateFrozenFunds(height) for _, frozenFund := range frozenFunds.List() { - pubKey10to11 := consertPubKey10to11(frozenFund.CandidateKey) + pubKey10to11 := types11.BytesToPubkey(frozenFund.CandidateKey) appState.FrozenFunds = append(appState.FrozenFunds, types11.FrozenFund{ Height: height - uint64(currentHeight), Address: types11.BytesToAddress(frozenFund.Address.Bytes()), @@ -2008,7 +2004,7 @@ func (s *StateDB) Export11(currentHeight uint64) types11.AppState { RewardAddress: types11.BytesToAddress(candidate.RewardAddress.Bytes()), OwnerAddress: types11.BytesToAddress(candidate.OwnerAddress.Bytes()), TotalBipStake: candidate.TotalBipStake.String(), - PubKey: consertPubKey10to11(candidate.PubKey), + PubKey: types11.BytesToPubkey(candidate.PubKey), Commission: candidate.Commission, Stakes: stakes, Status: candidate.Status, @@ -2019,7 +2015,7 @@ func (s *StateDB) Export11(currentHeight uint64) types11.AppState { for _, val := range vals.data { appState.Validators = append(appState.Validators, types11.Validator{ TotalBipStake: val.TotalBipStake.String(), - PubKey: consertPubKey10to11(val.PubKey), + PubKey: types11.BytesToPubkey(val.PubKey), AccumReward: val.AccumReward.String(), AbsentTimes: &types11.BitArray{ Bits: val.AbsentTimes.Bits, From 13f1d4c2060502ae1ac4f0ad28b33e0880580d0e Mon Sep 17 00:00:00 2001 From: Daniil Lashin <3121312+danil-lashin@users.noreply.github.com> Date: Wed, 29 Jan 2020 13:26:24 +0300 Subject: [PATCH 06/10] Refactor --- cmd/export/main.go | 282 +++++++++++--------------------- cmd/export/readme.md | 4 +- cmd/export/types11/tmtypes.go | 190 ++++++++++++++++++++++ cmd/export/types11/types.go | 295 ++-------------------------------- core/state/statedb.go | 127 +-------------- 5 files changed, 312 insertions(+), 586 deletions(-) create mode 100644 cmd/export/types11/tmtypes.go diff --git a/cmd/export/main.go b/cmd/export/main.go index 50f6d5c83..bb1b9fc61 100644 --- a/cmd/export/main.go +++ b/cmd/export/main.go @@ -1,24 +1,21 @@ package main import ( - "bytes" + "crypto/sha256" "encoding/json" "flag" "fmt" "github.com/MinterTeam/go-amino" + "github.com/MinterTeam/minter-go-node/cmd/export/types11" "github.com/MinterTeam/minter-go-node/cmd/utils" "github.com/MinterTeam/minter-go-node/core/state" - "github.com/pkg/errors" - "github.com/tendermint/tendermint/crypto" - "github.com/tendermint/tendermint/crypto/ed25519" - "github.com/tendermint/tendermint/crypto/secp256k1" + mtypes "github.com/MinterTeam/minter-go-node/core/types" "github.com/tendermint/tendermint/libs/common" "github.com/tendermint/tendermint/libs/db" "github.com/tendermint/tendermint/types" - tmtime "github.com/tendermint/tendermint/types/time" - "golang.org/x/crypto/sha3" - "io/ioutil" + "io" "log" + "os" "time" ) @@ -28,14 +25,22 @@ var ( genesisTime = flag.Duration("genesis_time", 0, "genesis_time") ) +const ( + genesisPath = "genesis.json" + + maxSupply = "1000000000000000000000000000000000" +) + func main() { defer func() { if r := recover(); r != nil { - fmt.Println("Recovered in f", r) + fmt.Println("Error occurred", r) } }() - required := []string{"height", "chain_id", "genesis_time"} + flag.Parse() + + required := []string{"height", "chain_id", "genesis_time"} seen := make(map[string]bool) flag.Visit(func(f *flag.Flag) { seen[f.Name] = true }) for _, req := range required { @@ -59,34 +64,96 @@ func main() { panic(err) } - cdc := amino.NewCodec() + oldState := currentState.Export(*height) + newState := types11.AppState{ + Note: oldState.Note, + StartHeight: oldState.StartHeight, + MaxGas: oldState.MaxGas, + TotalSlashed: oldState.TotalSlashed.String(), + } + + for _, account := range oldState.Accounts { + newState.Accounts = append(newState.Accounts, mtypes.Account{ + Address: account.Address, + Balance: account.Balance, + Nonce: account.Nonce, + MultisigData: account.MultisigData, + }) + } + + for _, coin := range oldState.Coins { + newState.Coins = append(newState.Coins, types11.Coin{ + Name: coin.Name, + Symbol: coin.Symbol, + Volume: coin.Volume.String(), + Crr: coin.Crr, + Reserve: coin.ReserveBalance.String(), + MaxSupply: maxSupply, + }) + } + + for _, check := range oldState.UsedChecks { + newState.UsedChecks = append(newState.UsedChecks, check) + } + + for _, ff := range oldState.FrozenFunds { + newState.FrozenFunds = append(newState.FrozenFunds, mtypes.FrozenFund{ + Height: ff.Height, + Address: ff.Address, + CandidateKey: ff.CandidateKey, + Coin: ff.Coin, + Value: ff.Value, + }) + } + + for _, candidate := range oldState.Candidates { + newState.Candidates = append(newState.Candidates, mtypes.Candidate{ + RewardAddress: candidate.RewardAddress, + OwnerAddress: candidate.OwnerAddress, + TotalBipStake: candidate.TotalBipStake, + PubKey: candidate.PubKey, + Commission: candidate.Commission, + Stakes: candidate.Stakes, + Status: candidate.Status, + }) + } - jsonBytes, err := cdc.MarshalJSONIndent(currentState.Export11(*height), "", " ") + for _, validator := range oldState.Validators { + newState.Validators = append(newState.Validators, mtypes.Validator{ + TotalBipStake: validator.TotalBipStake, + PubKey: validator.PubKey, + AccumReward: validator.AccumReward, + AbsentTimes: validator.AbsentTimes, + }) + } + + cdc := amino.NewCodec() + jsonBytes, err := cdc.MarshalJSONIndent(newState, "", " ") if err != nil { panic(err) } - appHash := [32]byte{} + appHash := currentState.Hash() // Compose Genesis - genesis := GenesisDoc{ + genesis := types11.GenesisDoc{ GenesisTime: time.Unix(0, 0).Add(*genesisTime), ChainID: *chainID, - ConsensusParams: &ConsensusParams{ - Block: BlockParams{ + ConsensusParams: &types11.ConsensusParams{ + Block: types11.BlockParams{ MaxBytes: 10000000, MaxGas: 100000, TimeIotaMs: 1000, }, - Evidence: EvidenceParams{ + Evidence: types11.EvidenceParams{ MaxAgeNumBlocks: 1000, - MaxAgeDuration: 24 * time.Hour, //todo + MaxAgeDuration: 24 * time.Hour, }, - Validator: ValidatorParams{ + Validator: types11.ValidatorParams{ PubKeyTypes: []string{types.ABCIPubKeyTypeEd25519}, }, }, - AppHash: appHash[:], + AppHash: appHash, AppState: json.RawMessage(jsonBytes), } @@ -95,177 +162,24 @@ func main() { panic(err) } - if err := genesis.SaveAs("genesis.json"); err != nil { + if err := genesis.SaveAs(genesisPath); err != nil { panic(err) } - fmt.Println("OK") - fmt.Println(fmt.Sprintf("%x", sha3.Sum512([]byte(fmt.Sprintf("%v", genesis))))) -} - -type GenesisDoc struct { - GenesisTime time.Time `json:"genesis_time"` - ChainID string `json:"chain_id"` - ConsensusParams *ConsensusParams `json:"consensus_params,omitempty"` - Validators []GenesisValidator `json:"validators,omitempty"` - AppHash common.HexBytes `json:"app_hash"` - AppState json.RawMessage `json:"app_state,omitempty"` -} -type ConsensusParams struct { - Block BlockParams `json:"block"` - Evidence EvidenceParams `json:"evidence"` - Validator ValidatorParams `json:"validator"` -} -type BlockParams struct { - MaxBytes int64 `json:"max_bytes"` - MaxGas int64 `json:"max_gas"` - // Minimum time increment between consecutive blocks (in milliseconds) - // Not exposed to the application. - TimeIotaMs int64 `json:"time_iota_ms"` -} -type GenesisValidator struct { - Address Address `json:"address"` - PubKey crypto.PubKey `json:"pub_key"` - Power int64 `json:"power"` - Name string `json:"name"` -} -type Address = crypto.Address -type EvidenceParams struct { - MaxAgeNumBlocks int64 `json:"max_age_num_blocks"` // only accept new evidence more recent than this - MaxAgeDuration time.Duration `json:"max_age_duration"` -} -type ValidatorParams struct { - PubKeyTypes []string `json:"pub_key_types"` -} - -const ( - MaxChainIDLen = 50 -) - -func (genDoc *GenesisDoc) ValidateAndComplete() error { - if genDoc.ChainID == "" { - return errors.New("genesis doc must include non-empty chain_id") - } - if len(genDoc.ChainID) > MaxChainIDLen { - return errors.Errorf("chain_id in genesis doc is too long (max: %d)", MaxChainIDLen) - } - - if genDoc.ConsensusParams == nil { - genDoc.ConsensusParams = DefaultConsensusParams() - } else if err := genDoc.ConsensusParams.Validate(); err != nil { - return err - } - - for i, v := range genDoc.Validators { - if v.Power == 0 { - return errors.Errorf("the genesis file cannot contain validators with no voting power: %v", v) - } - if len(v.Address) > 0 && !bytes.Equal(v.PubKey.Address(), v.Address) { - return errors.Errorf("incorrect address for validator %v in the genesis file, should be %v", v, v.PubKey.Address()) - } - if len(v.Address) == 0 { - genDoc.Validators[i].Address = v.PubKey.Address() - } - } - - if genDoc.GenesisTime.IsZero() { - genDoc.GenesisTime = tmtime.Now() - } - - return nil -} -func DefaultConsensusParams() *ConsensusParams { - return &ConsensusParams{ - DefaultBlockParams(), - DefaultEvidenceParams(), - DefaultValidatorParams(), - } -} -func DefaultBlockParams() BlockParams { - return BlockParams{ - MaxBytes: 22020096, // 21MB - MaxGas: -1, - TimeIotaMs: 1000, // 1s - } -} - -// DefaultEvidenceParams Params returns a default EvidenceParams. -func DefaultEvidenceParams() EvidenceParams { - return EvidenceParams{ - MaxAgeNumBlocks: 100000, // 27.8 hrs at 1block/s - MaxAgeDuration: 48 * time.Hour, - } -} - -// DefaultValidatorParams returns a default ValidatorParams, which allows -// only ed25519 pubkeys. -func DefaultValidatorParams() ValidatorParams { - return ValidatorParams{[]string{types.ABCIPubKeyTypeEd25519}} + fmt.Printf("Ok\n%x\n", getSha256Hash(genesisPath)) } -const ( - ABCIPubKeyTypeEd25519 = "ed25519" - ABCIPubKeyTypeSr25519 = "sr25519" - ABCIPubKeyTypeSecp256k1 = "secp256k1" - - MaxBlockSizeBytes = 104857600 -) - -var ABCIPubKeyTypesToAminoNames = map[string]string{ - ABCIPubKeyTypeEd25519: ed25519.PubKeyAminoName, - ABCIPubKeyTypeSr25519: "tendermint/PubKeySr25519", - ABCIPubKeyTypeSecp256k1: secp256k1.PubKeyAminoName, -} - -func (params *ConsensusParams) Validate() error { - if params.Block.MaxBytes <= 0 { - return errors.Errorf("block.MaxBytes must be greater than 0. Got %d", - params.Block.MaxBytes) - } - if params.Block.MaxBytes > MaxBlockSizeBytes { - return errors.Errorf("block.MaxBytes is too big. %d > %d", - params.Block.MaxBytes, MaxBlockSizeBytes) - } - - if params.Block.MaxGas < -1 { - return errors.Errorf("block.MaxGas must be greater or equal to -1. Got %d", - params.Block.MaxGas) - } - - if params.Block.TimeIotaMs <= 0 { - return errors.Errorf("block.TimeIotaMs must be greater than 0. Got %v", - params.Block.TimeIotaMs) - } - - if params.Evidence.MaxAgeNumBlocks <= 0 { - return errors.Errorf("evidenceParams.MaxAgeNumBlocks must be greater than 0. Got %d", - params.Evidence.MaxAgeNumBlocks) - } - - if params.Evidence.MaxAgeDuration <= 0 { - return errors.Errorf("evidenceParams.MaxAgeDuration must be grater than 0 if provided, Got %v", - params.Evidence.MaxAgeDuration) +func getSha256Hash(file string) []byte { + f, err := os.Open(file) + if err != nil { + log.Fatal(err) } + defer f.Close() - if len(params.Validator.PubKeyTypes) == 0 { - return errors.New("len(Validator.PubKeyTypes) must be greater than 0") + h := sha256.New() + if _, err := io.Copy(h, f); err != nil { + log.Fatal(err) } - // Check if keyType is a known ABCIPubKeyType - for i := 0; i < len(params.Validator.PubKeyTypes); i++ { - keyType := params.Validator.PubKeyTypes[i] - if _, ok := ABCIPubKeyTypesToAminoNames[keyType]; !ok { - return errors.Errorf("params.Validator.PubKeyTypes[%d], %s, is an unknown pubkey type", - i, keyType) - } - } - - return nil -} -func (genDoc *GenesisDoc) SaveAs(file string) error { - genDocBytes, err := types.GetCodec().MarshalJSONIndent(genDoc, "", " ") - if err != nil { - return err - } - return ioutil.WriteFile(file, genDocBytes, 0644) + return h.Sum(nil) } diff --git a/cmd/export/readme.md b/cmd/export/readme.md index 6bb4adafc..6514418a8 100644 --- a/cmd/export/readme.md +++ b/cmd/export/readme.md @@ -1,8 +1,8 @@ # Export Command Command to convert the genesis file from version 1.0 to 1.1 -More details will be received later. +More details will be disclosed later. #####Example ```bash -./export --height=4350342 --chain_id=1 --genesis_time=1580206824s +./export --height=4350342 --chain_id="minter-mainnnet-2" --genesis_time=1580206824s ``` \ No newline at end of file diff --git a/cmd/export/types11/tmtypes.go b/cmd/export/types11/tmtypes.go new file mode 100644 index 000000000..8abbde036 --- /dev/null +++ b/cmd/export/types11/tmtypes.go @@ -0,0 +1,190 @@ +package types11 + +import ( + "bytes" + "encoding/json" + "github.com/pkg/errors" + "github.com/tendermint/tendermint/crypto" + "github.com/tendermint/tendermint/crypto/ed25519" + "github.com/tendermint/tendermint/crypto/secp256k1" + "github.com/tendermint/tendermint/libs/common" + "github.com/tendermint/tendermint/types" + tmtime "github.com/tendermint/tendermint/types/time" + "io/ioutil" + "time" +) + +type GenesisDoc struct { + GenesisTime time.Time `json:"genesis_time"` + ChainID string `json:"chain_id"` + ConsensusParams *ConsensusParams `json:"consensus_params,omitempty"` + Validators []GenesisValidator `json:"validators,omitempty"` + AppHash common.HexBytes `json:"app_hash"` + AppState json.RawMessage `json:"app_state,omitempty"` +} + +type ConsensusParams struct { + Block BlockParams `json:"block"` + Evidence EvidenceParams `json:"evidence"` + Validator ValidatorParams `json:"validator"` +} + +type BlockParams struct { + MaxBytes int64 `json:"max_bytes"` + MaxGas int64 `json:"max_gas"` + // Minimum time increment between consecutive blocks (in milliseconds) + // Not exposed to the application. + TimeIotaMs int64 `json:"time_iota_ms"` +} + +type GenesisValidator struct { + Address Address `json:"address"` + PubKey crypto.PubKey `json:"pub_key"` + Power int64 `json:"power"` + Name string `json:"name"` +} + +type Address = crypto.Address + +type EvidenceParams struct { + MaxAgeNumBlocks int64 `json:"max_age_num_blocks"` // only accept new evidence more recent than this + MaxAgeDuration time.Duration `json:"max_age_duration"` +} + +type ValidatorParams struct { + PubKeyTypes []string `json:"pub_key_types"` +} + +const ( + MaxChainIDLen = 50 +) + +func (genDoc *GenesisDoc) ValidateAndComplete() error { + if genDoc.ChainID == "" { + return errors.New("genesis doc must include non-empty chain_id") + } + if len(genDoc.ChainID) > MaxChainIDLen { + return errors.Errorf("chain_id in genesis doc is too long (max: %d)", MaxChainIDLen) + } + + if genDoc.ConsensusParams == nil { + genDoc.ConsensusParams = DefaultConsensusParams() + } else if err := genDoc.ConsensusParams.Validate(); err != nil { + return err + } + + for i, v := range genDoc.Validators { + if v.Power == 0 { + return errors.Errorf("the genesis file cannot contain validators with no voting power: %v", v) + } + if len(v.Address) > 0 && !bytes.Equal(v.PubKey.Address(), v.Address) { + return errors.Errorf("incorrect address for validator %v in the genesis file, should be %v", v, v.PubKey.Address()) + } + if len(v.Address) == 0 { + genDoc.Validators[i].Address = v.PubKey.Address() + } + } + + if genDoc.GenesisTime.IsZero() { + genDoc.GenesisTime = tmtime.Now() + } + + return nil +} + +func DefaultConsensusParams() *ConsensusParams { + return &ConsensusParams{ + DefaultBlockParams(), + DefaultEvidenceParams(), + DefaultValidatorParams(), + } +} + +func DefaultBlockParams() BlockParams { + return BlockParams{ + MaxBytes: 22020096, // 21MB + MaxGas: -1, + TimeIotaMs: 1000, // 1s + } +} + +// DefaultEvidenceParams Params returns a default EvidenceParams. +func DefaultEvidenceParams() EvidenceParams { + return EvidenceParams{ + MaxAgeNumBlocks: 100000, // 27.8 hrs at 1block/s + MaxAgeDuration: 48 * time.Hour, + } +} + +// DefaultValidatorParams returns a default ValidatorParams, which allows +// only ed25519 pubkeys. +func DefaultValidatorParams() ValidatorParams { + return ValidatorParams{[]string{types.ABCIPubKeyTypeEd25519}} +} + +const ( + ABCIPubKeyTypeEd25519 = "ed25519" + ABCIPubKeyTypeSr25519 = "sr25519" + ABCIPubKeyTypeSecp256k1 = "secp256k1" + + MaxBlockSizeBytes = 104857600 +) + +var ABCIPubKeyTypesToAminoNames = map[string]string{ + ABCIPubKeyTypeEd25519: ed25519.PubKeyAminoName, + ABCIPubKeyTypeSr25519: "tendermint/PubKeySr25519", + ABCIPubKeyTypeSecp256k1: secp256k1.PubKeyAminoName, +} + +func (params *ConsensusParams) Validate() error { + if params.Block.MaxBytes <= 0 { + return errors.Errorf("block.MaxBytes must be greater than 0. Got %d", + params.Block.MaxBytes) + } + if params.Block.MaxBytes > MaxBlockSizeBytes { + return errors.Errorf("block.MaxBytes is too big. %d > %d", + params.Block.MaxBytes, MaxBlockSizeBytes) + } + + if params.Block.MaxGas < -1 { + return errors.Errorf("block.MaxGas must be greater or equal to -1. Got %d", + params.Block.MaxGas) + } + + if params.Block.TimeIotaMs <= 0 { + return errors.Errorf("block.TimeIotaMs must be greater than 0. Got %v", + params.Block.TimeIotaMs) + } + + if params.Evidence.MaxAgeNumBlocks <= 0 { + return errors.Errorf("evidenceParams.MaxAgeNumBlocks must be greater than 0. Got %d", + params.Evidence.MaxAgeNumBlocks) + } + + if params.Evidence.MaxAgeDuration <= 0 { + return errors.Errorf("evidenceParams.MaxAgeDuration must be grater than 0 if provided, Got %v", + params.Evidence.MaxAgeDuration) + } + + if len(params.Validator.PubKeyTypes) == 0 { + return errors.New("len(Validator.PubKeyTypes) must be greater than 0") + } + + // Check if keyType is a known ABCIPubKeyType + for i := 0; i < len(params.Validator.PubKeyTypes); i++ { + keyType := params.Validator.PubKeyTypes[i] + if _, ok := ABCIPubKeyTypesToAminoNames[keyType]; !ok { + return errors.Errorf("params.Validator.PubKeyTypes[%d], %s, is an unknown pubkey type", + i, keyType) + } + } + + return nil +} +func (genDoc *GenesisDoc) SaveAs(file string) error { + genDocBytes, err := types.GetCodec().MarshalJSONIndent(genDoc, "", " ") + if err != nil { + return err + } + return ioutil.WriteFile(file, genDocBytes, 0644) +} diff --git a/cmd/export/types11/types.go b/cmd/export/types11/types.go index 625c0bc48..00dd4b07a 100644 --- a/cmd/export/types11/types.go +++ b/cmd/export/types11/types.go @@ -1,288 +1,27 @@ package types11 import ( - "bytes" - "encoding/binary" - "encoding/hex" - "fmt" - "github.com/MinterTeam/minter-go-node/hexutil" - "strings" - "sync" + "github.com/MinterTeam/minter-go-node/core/types" ) type AppState struct { - Note string `json:"note"` - StartHeight uint64 `json:"start_height"` - Validators []Validator `json:"validators,omitempty"` - Candidates []Candidate `json:"candidates,omitempty"` - Accounts []Account `json:"accounts,omitempty"` - Coins []Coin `json:"coins,omitempty"` - FrozenFunds []FrozenFund `json:"frozen_funds,omitempty"` - UsedChecks []UsedCheck `json:"used_checks,omitempty"` - MaxGas uint64 `json:"max_gas"` - TotalSlashed string `json:"total_slashed"` -} - -type Validator struct { - TotalBipStake string `json:"total_bip_stake"` - PubKey Pubkey `json:"pub_key"` - AccumReward string `json:"accum_reward"` - AbsentTimes *BitArray `json:"absent_times"` -} - -type Candidate struct { - RewardAddress Address `json:"reward_address"` - OwnerAddress Address `json:"owner_address"` - TotalBipStake string `json:"total_bip_stake"` - PubKey Pubkey `json:"pub_key"` - Commission uint `json:"commission"` - Stakes []Stake `json:"stakes"` - Status byte `json:"status"` -} - -type Stake struct { - Owner Address `json:"owner"` - Coin CoinSymbol `json:"coin"` - Value string `json:"value"` - BipValue string `json:"bip_value"` + Note string `json:"note"` + StartHeight uint64 `json:"start_height"` + Validators []types.Validator `json:"validators,omitempty"` + Candidates []types.Candidate `json:"candidates,omitempty"` + Accounts []types.Account `json:"accounts,omitempty"` + Coins []Coin `json:"coins,omitempty"` + FrozenFunds []types.FrozenFund `json:"frozen_funds,omitempty"` + UsedChecks []types.UsedCheck `json:"used_checks,omitempty"` + MaxGas uint64 `json:"max_gas"` + TotalSlashed string `json:"total_slashed"` } type Coin struct { - Name string `json:"name"` - Symbol CoinSymbol `json:"symbol"` - Volume string `json:"volume"` - Crr uint `json:"crr"` - Reserve string `json:"reserve"` - MaxSupply string `json:"max_supply"` -} - -type FrozenFund struct { - Height uint64 `json:"height"` - Address Address `json:"address"` - CandidateKey *Pubkey `json:"candidate_key,omitempty"` - Coin CoinSymbol `json:"coin"` - Value string `json:"value"` -} - -type UsedCheck string - -type Account struct { - Address Address `json:"address"` - Balance []Balance `json:"balance"` - Nonce uint64 `json:"nonce"` - MultisigData *Multisig `json:"multisig_data,omitempty"` -} - -type Balance struct { - Coin CoinSymbol `json:"coin"` - Value string `json:"value"` -} - -type Multisig struct { - Weights []uint `json:"weights"` - Threshold uint `json:"threshold"` - Addresses []Address `json:"addresses"` -} - -const CoinSymbolLength = 10 - -type CoinSymbol [CoinSymbolLength]byte - -func (c CoinSymbol) String() string { return string(bytes.Trim(c[:], "\x00")) } -func (c CoinSymbol) Bytes() []byte { return c[:] } - -func (c CoinSymbol) MarshalJSON() ([]byte, error) { - - buffer := bytes.NewBufferString("\"") - buffer.WriteString(c.String()) - buffer.WriteString("\"") - - return buffer.Bytes(), nil -} - -func (c *CoinSymbol) UnmarshalJSON(input []byte) error { - *c = StrToCoinSymbol(string(input[1 : len(input)-1])) - return nil -} - -func StrToCoinSymbol(s string) CoinSymbol { - var symbol CoinSymbol - copy(symbol[:], []byte(s)) - return symbol -} - -type BitArray struct { - mtx sync.Mutex - Bits uint `json:"bits"` // NOTE: persisted via reflect, must be exported - Elems []uint64 `json:"elems"` // NOTE: persisted via reflect, must be exported -} - -func (bA *BitArray) getIndex(i uint) bool { - if i >= bA.Bits { - return false - } - return bA.Elems[i/64]&(uint64(1)< 0 -} - -// String returns a string representation of BitArray: BA{}, -// where is a sequence of 'x' (1) and '_' (0). -// The includes spaces and newlines to help people. -// For a simple sequence of 'x' and '_' characters with no spaces or newlines, -// see the MarshalJSON() method. -// Example: "BA{_x_}" or "nil-BitArray" for nil. -func (bA *BitArray) String() string { - return bA.StringIndented("") -} - -// StringIndented returns the same thing as String(), but applies the indent -// at every 10th bit, and twice at every 50th bit. -func (bA *BitArray) StringIndented(indent string) string { - if bA == nil { - return "nil-BitArray" - } - bA.mtx.Lock() - defer bA.mtx.Unlock() - return bA.stringIndented(indent) -} - -func (bA *BitArray) stringIndented(indent string) string { - lines := []string{} - bits := "" - for i := uint(0); i < bA.Bits; i++ { - if bA.getIndex(i) { - bits += "x" - } else { - bits += "_" - } - if i%100 == 99 { - lines = append(lines, bits) - bits = "" - } - if i%10 == 9 { - bits += indent - } - if i%50 == 49 { - bits += indent - } - } - if len(bits) > 0 { - lines = append(lines, bits) - } - return fmt.Sprintf("BA{%v:%v}", bA.Bits, strings.Join(lines, indent)) -} - -// Bytes returns the byte representation of the bits within the bitarray. -func (bA *BitArray) Bytes() []byte { - bA.mtx.Lock() - defer bA.mtx.Unlock() - - numBytes := (bA.Bits + 7) / 8 - bytes := make([]byte, numBytes) - for i := 0; i < len(bA.Elems); i++ { - elemBytes := [8]byte{} - binary.LittleEndian.PutUint64(elemBytes[:], bA.Elems[i]) - copy(bytes[i*8:], elemBytes[:]) - } - return bytes -} - -// MarshalJSON implements json.Marshaler interface by marshaling bit array -// using a custom format: a string of '-' or 'x' where 'x' denotes the 1 bit. -func (bA *BitArray) MarshalJSON() ([]byte, error) { - if bA == nil { - return []byte("null"), nil - } - - bA.mtx.Lock() - defer bA.mtx.Unlock() - - bits := `"` - for i := uint(0); i < bA.Bits; i++ { - if bA.getIndex(i) { - bits += `x` - } else { - bits += `_` - } - } - bits += `"` - return []byte(bits), nil -} - -const AddressLength = 20 - -type Address [AddressLength]byte - -func BytesToAddress(b []byte) Address { - var a Address - a.SetBytes(b) - return a -} -func (a *Address) SetBytes(b []byte) { - if len(b) > len(a) { - b = b[len(b)-AddressLength:] - } - copy(a[AddressLength-len(b):], b) -} -func (a Address) String() string { - return a.Hex() -} -func (a Address) Hex() string { - return "Mx" + hex.EncodeToString(a[:]) -} - -// MarshalText returns the hex representation of a. -func (a Address) MarshalText() ([]byte, error) { - return hexutil.Bytes(a[:]).MarshalText() -} - -// UnmarshalText parses a hash in hex syntax. -func (a *Address) UnmarshalText(input []byte) error { - return hexutil.UnmarshalFixedText("Address", input, a[:]) -} - -func (a *Address) Unmarshal(input []byte) error { - copy(a[:], input) - return nil -} - -func (a Address) MarshalJSON() ([]byte, error) { - return []byte(fmt.Sprintf("\"%s\"", a.String())), nil -} - -const PubKeyLength = 32 - -type Pubkey [PubKeyLength]byte - -func BytesToPubkey(b []byte) Pubkey { - var p Pubkey - p.SetBytes(b) - return p -} - -func (p *Pubkey) SetBytes(b []byte) { - if len(b) > len(p) { - b = b[len(b)-PubKeyLength:] - } - copy(p[PubKeyLength-len(b):], b) -} - -func (p Pubkey) Bytes() []byte { return p[:] } - -func (p Pubkey) String() string { - return fmt.Sprintf("Mp%x", p[:]) -} - -func (p Pubkey) MarshalText() ([]byte, error) { - return []byte(p.String()), nil -} - -func (p Pubkey) MarshalJSON() ([]byte, error) { - return []byte(fmt.Sprintf("\"%s\"", p.String())), nil -} - -func (p *Pubkey) UnmarshalJSON(input []byte) error { - b, err := hex.DecodeString(string(input)[3 : len(input)-1]) - copy(p[:], b) - - return err + Name string `json:"name"` + Symbol types.CoinSymbol `json:"symbol"` + Volume string `json:"volume"` + Crr uint `json:"crr"` + Reserve string `json:"reserve"` + MaxSupply string `json:"max_supply"` } diff --git a/core/state/statedb.go b/core/state/statedb.go index a8df678c2..e0172830f 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -4,6 +4,7 @@ import ( "encoding/hex" "fmt" "github.com/MinterTeam/go-amino" + "github.com/MinterTeam/minter-go-node/cmd/export/newtypes" "github.com/MinterTeam/minter-go-node/cmd/export/types11" "github.com/MinterTeam/minter-go-node/cmd/utils" "github.com/MinterTeam/minter-go-node/core/rewards" @@ -1909,128 +1910,6 @@ func (s *StateDB) Export(currentHeight uint64) types.AppState { return appState } -func (s *StateDB) Export11(currentHeight uint64) types11.AppState { - - appState := types11.AppState{} - - s.iavl.Iterate(func(key []byte, value []byte) bool { - // export accounts - if key[0] == addressPrefix[0] { - account := s.GetOrNewStateObject(types.BytesToAddress(key[1:])) - - balance := make([]types11.Balance, len(account.Balances().Data)) - i := 0 - for coin, value := range account.Balances().Data { - - balance[i] = types11.Balance{ - Coin: types11.StrToCoinSymbol(coin.String()), - Value: value.String(), - } - i++ - } - acc := types11.Account{ - Address: types11.BytesToAddress(account.address.Bytes()), - Balance: balance, - Nonce: account.data.Nonce, - } - - if account.IsMultisig() { - addresses := make([]types11.Address, 0, len(account.data.MultisigData.Addresses)) - for _, address := range account.data.MultisigData.Addresses { - addresses = append(addresses, types11.BytesToAddress(address.Bytes())) - } - acc.MultisigData = &types11.Multisig{ - Weights: account.data.MultisigData.Weights, - Threshold: account.data.MultisigData.Threshold, - Addresses: addresses, - } - } - - appState.Accounts = append(appState.Accounts, acc) - } - - // export coins - if key[0] == coinPrefix[0] { - coin := s.GetStateCoin(types.StrToCoinSymbol(string(key[1:]))) - - appState.Coins = append(appState.Coins, types11.Coin{ - Name: coin.Name(), - Symbol: types11.StrToCoinSymbol(coin.Symbol().String()), - Volume: coin.Volume().String(), - Crr: coin.Crr(), - Reserve: coin.ReserveBalance().String(), - MaxSupply: big.NewInt(0).Exp(big.NewInt(10), big.NewInt(15+18), nil).String(), - }) - } - - // export used checks - if key[0] == usedCheckPrefix[0] { - appState.UsedChecks = append(appState.UsedChecks, types11.UsedCheck(fmt.Sprintf("%x", key[1:]))) - } - - // export frozen funds - if key[0] == frozenFundsPrefix[0] { - height := binary.BigEndian.Uint64(key[1:]) - frozenFunds := s.GetStateFrozenFunds(height) - - for _, frozenFund := range frozenFunds.List() { - pubKey10to11 := types11.BytesToPubkey(frozenFund.CandidateKey) - appState.FrozenFunds = append(appState.FrozenFunds, types11.FrozenFund{ - Height: height - uint64(currentHeight), - Address: types11.BytesToAddress(frozenFund.Address.Bytes()), - CandidateKey: &pubKey10to11, - Coin: types11.StrToCoinSymbol(frozenFund.Coin.String()), - Value: frozenFund.Value.String(), - }) - } - } - - return false - }) - - candidates := s.getStateCandidates() - for _, candidate := range candidates.data { - var stakes []types11.Stake - for _, s := range candidate.Stakes { - stakes = append(stakes, types11.Stake{ - Owner: types11.BytesToAddress(s.Owner.Bytes()), - Coin: types11.StrToCoinSymbol(s.Coin.String()), - Value: s.Value.String(), - BipValue: s.BipValue.String(), - }) - } - - appState.Candidates = append(appState.Candidates, types11.Candidate{ - RewardAddress: types11.BytesToAddress(candidate.RewardAddress.Bytes()), - OwnerAddress: types11.BytesToAddress(candidate.OwnerAddress.Bytes()), - TotalBipStake: candidate.TotalBipStake.String(), - PubKey: types11.BytesToPubkey(candidate.PubKey), - Commission: candidate.Commission, - Stakes: stakes, - Status: candidate.Status, - }) - } - - vals := s.getStateValidators() - for _, val := range vals.data { - appState.Validators = append(appState.Validators, types11.Validator{ - TotalBipStake: val.TotalBipStake.String(), - PubKey: types11.BytesToPubkey(val.PubKey), - AccumReward: val.AccumReward.String(), - AbsentTimes: &types11.BitArray{ - Bits: val.AbsentTimes.Bits, - Elems: val.AbsentTimes.Elems, - }, - }) - } - - appState.MaxGas = s.GetMaxGas() - appState.StartHeight = s.height - appState.TotalSlashed = s.GetTotalSlashed().String() - - return appState -} - func (s *StateDB) Import(appState types.AppState) { s.SetMaxGas(appState.MaxGas) s.setTotalSlashed(appState.TotalSlashed) @@ -2275,3 +2154,7 @@ func (s *StateDB) Height() uint64 { func (s *StateDB) DB() dbm.DB { return s.db } + +func (s *StateDB) Hash() []byte { + return s.iavl.Hash() +} From f32860c1a82302402f2e139c68e24edd6b821eb3 Mon Sep 17 00:00:00 2001 From: Daniil Lashin <3121312+danil-lashin@users.noreply.github.com> Date: Wed, 29 Jan 2020 13:29:59 +0300 Subject: [PATCH 07/10] Fix --- core/state/statedb.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/core/state/statedb.go b/core/state/statedb.go index e0172830f..6a37bbbd4 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -4,8 +4,6 @@ import ( "encoding/hex" "fmt" "github.com/MinterTeam/go-amino" - "github.com/MinterTeam/minter-go-node/cmd/export/newtypes" - "github.com/MinterTeam/minter-go-node/cmd/export/types11" "github.com/MinterTeam/minter-go-node/cmd/utils" "github.com/MinterTeam/minter-go-node/core/rewards" "github.com/MinterTeam/minter-go-node/core/types" From 4de5098b28056c59167e9750d698b3e18c85daea Mon Sep 17 00:00:00 2001 From: Daniil Lashin <3121312+danil-lashin@users.noreply.github.com> Date: Wed, 29 Jan 2020 13:57:15 +0300 Subject: [PATCH 08/10] Sort balances --- core/state/statedb.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/state/statedb.go b/core/state/statedb.go index 6a37bbbd4..765a9d008 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -1803,10 +1803,10 @@ func (s *StateDB) Export(currentHeight uint64) types.AppState { balance := make([]types.Balance, len(account.Balances().Data)) i := 0 - for coin, value := range account.Balances().Data { + for _, coin := range account.Balances().getCoins() { balance[i] = types.Balance{ Coin: coin, - Value: value, + Value: account.Balance(coin), } i++ } From 75b980a91bb7bf33b9fdbedba286b7ba88551c2b Mon Sep 17 00:00:00 2001 From: Daniil Lashin <3121312+danil-lashin@users.noreply.github.com> Date: Wed, 29 Jan 2020 14:08:09 +0300 Subject: [PATCH 09/10] Fix validator struct --- cmd/export/main.go | 6 +++--- cmd/export/types11/types.go | 9 ++++++++- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/cmd/export/main.go b/cmd/export/main.go index bb1b9fc61..fa859058c 100644 --- a/cmd/export/main.go +++ b/cmd/export/main.go @@ -119,10 +119,10 @@ func main() { } for _, validator := range oldState.Validators { - newState.Validators = append(newState.Validators, mtypes.Validator{ - TotalBipStake: validator.TotalBipStake, + newState.Validators = append(newState.Validators, types11.Validator{ + TotalBipStake: validator.TotalBipStake.String(), PubKey: validator.PubKey, - AccumReward: validator.AccumReward, + AccumReward: validator.AccumReward.String(), AbsentTimes: validator.AbsentTimes, }) } diff --git a/cmd/export/types11/types.go b/cmd/export/types11/types.go index 00dd4b07a..893b60b16 100644 --- a/cmd/export/types11/types.go +++ b/cmd/export/types11/types.go @@ -7,7 +7,7 @@ import ( type AppState struct { Note string `json:"note"` StartHeight uint64 `json:"start_height"` - Validators []types.Validator `json:"validators,omitempty"` + Validators []Validator `json:"validators,omitempty"` Candidates []types.Candidate `json:"candidates,omitempty"` Accounts []types.Account `json:"accounts,omitempty"` Coins []Coin `json:"coins,omitempty"` @@ -17,6 +17,13 @@ type AppState struct { TotalSlashed string `json:"total_slashed"` } +type Validator struct { + TotalBipStake string `json:"total_bip_stake"` + PubKey types.Pubkey `json:"pub_key"` + AccumReward string `json:"accum_reward"` + AbsentTimes *types.BitArray `json:"absent_times"` +} + type Coin struct { Name string `json:"name"` Symbol types.CoinSymbol `json:"symbol"` From d721e5a99d38654ddb13258fb5c6c4e52448b474 Mon Sep 17 00:00:00 2001 From: klim0v Date: Wed, 29 Jan 2020 14:39:57 +0300 Subject: [PATCH 10/10] refactor --- cmd/export/main.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/export/main.go b/cmd/export/main.go index fa859058c..ec075ccdb 100644 --- a/cmd/export/main.go +++ b/cmd/export/main.go @@ -133,7 +133,7 @@ func main() { panic(err) } - appHash := currentState.Hash() + appHash := [32]byte{} // Compose Genesis genesis := types11.GenesisDoc{ @@ -153,7 +153,7 @@ func main() { PubKeyTypes: []string{types.ABCIPubKeyTypeEd25519}, }, }, - AppHash: appHash, + AppHash: appHash[:], AppState: json.RawMessage(jsonBytes), }